Use external linter instead of Python yamllint module (#309)

README.md

@@ -1,6 +1,6 @@
 # ADI MAX78000/MAX78002 Model Training and Synthesis
 
-September 19, 2023
+September 20, 2023
 
 ADI’s MAX78000/MAX78002 project is comprised of five repositories:
 
@@ -2288,6 +2288,9 @@ The following table describes the most important command line arguments for `ai8
 
 The [quick-start guide](https://github.com/MaximIntegratedAI/MaximAI_Documentation/blob/master/Guides/YAML%20Quickstart.md) provides a short overview of the purpose and structure of the YAML network description file.
 
+If `yamllint` is installed and available in the shell path, it is automatically run against the configuration file and all warnings and errors are reported.
+*Note: The name of the linter can be changed using the `--yamllint` command line argument.*
+
 The following is a detailed guide into all supported configuration options.
 
 An example network description for the ai85net5 architecture and MNIST is shown below:
@@ -2357,7 +2360,7 @@ To generate an RTL simulation for the same network and sample data in the direct
 
 Network descriptions are written in YAML (see <https://en.wikipedia.org/wiki/YAML>). There are two sections in each file — global statements and a sequence of layer descriptions.
 
-*Note: The network loader automatically checks the configuration file for syntax errors and prints warnings for non-fatal errors. To perform the same checks manually, run:* `yamllint configfile.yaml`
+*Note: The network loader automatically checks the configuration file for syntax errors and prints warnings for non-fatal errors if `yamllint` is installed in the shell search path. To perform the same checks manually, run:* `yamllint configfile.yaml` (to use a different linter, specify `--yamllint mylinter`).
 
 #### Purpose of the YAML Network Description
 

izer/commandline.py

@@ -460,6 +460,8 @@ def get_parser() -> argparse.Namespace:
                        help='version check update interval (hours), default = 24')
     group.add_argument('--upstream', metavar='REPO', default="MaximIntegratedAI/ai8x-synthesis",
                        help='GitHub repository name for update checking')
+    group.add_argument('--yamllint', metavar='S', default='yamllint',
+                       help='name of linter for YAML files (default: yamllint)')
 
     args = parser.parse_args()
 

izer/izer.py

@@ -69,7 +69,7 @@ def main():
     commandline.set_state(args)
 
     # Load configuration file
-    cfg, cfg_layers, params = yamlcfg.parse(args.config_file, args.skip_yaml_layers)
+    cfg, cfg_layers, params = yamlcfg.parse(args.config_file, args.skip_yaml_layers, args.yamllint)
     state.layer_name = params['layer_name']
 
     # If not using test data, load weights and biases

izer/yamlcfg.py

@@ -8,11 +8,9 @@
 YAML Configuration Routines
 """
 import os
+import subprocess
 
 import yaml
-import yamllint
-import yamllint.config
-import yamllint.linter
 
 from . import devices, names, op, state
 from . import tornadocnn as tc
@@ -58,12 +56,14 @@ def construct_mapping(self, node, deep=False):
 def parse(
         config_file,
         skip_layers=0,
+        linter=None,
 ):  # pylint: disable=too-many-branches
     """
     Configure network parameters from the YAML configuration file `config_file`.
     `max_conv` can be set to force an early termination of the parser.
     The function returns both YAML dictionary, the length of the processor map,
     as well as a settings dictionary.
+    if `linter` is set, try to run it against the YAML file and display the output.
     """
 
     def error_exit(message, sequence):
@@ -74,15 +74,31 @@ def error_exit(message, sequence):
 
     print(f'Reading {config_file} to configure network...')
 
-    # Run yamllint first
     if not os.path.exists(config_file):
         eprint(f'YAML configuration file {config_file} does not exist!')
 
-    yaml_config = yamllint.config.YamlLintConfig('extends: relaxed')
-    with open(config_file, mode='r', encoding='utf-8') as cfg_file:
-        for p in yamllint.linter.run(cfg_file, yaml_config):
-            eprint(f'{config_file} line {p.line}, col {p.column}: {p.desc}',
-                   error=p.level == 'error')
+    # Run linter first if configured
+    if linter:
+        try:
+            proc = subprocess.run(
+                [linter, config_file],
+                shell=False,
+                text=True,
+                capture_output=True,
+                timeout=5.0,
+                check=False,
+            )
+            errors = proc.stderr.splitlines()
+            for i, line in enumerate(errors):
+                eprint(line, exit_code=None if i < len(errors) - 1 else 1)
+            warnings = proc.stdout.splitlines()
+            for w in warnings:
+                if ' error ' in w:
+                    eprint('  ' + w)
+                elif w != '':
+                    wprint(w)
+        except FileNotFoundError:
+            wprint(f'Cannot run "{linter}" linter to check {config_file}')
 
     # Load configuration file
     with open(config_file, mode='r', encoding='utf-8') as cfg_file:

networks/ai87-facedet-tinierssd.yaml

@@ -1,3 +1,4 @@
+---
 # For AI87
 # CHW configuration for FaceDetection
 # Parallel Model
@@ -33,7 +34,7 @@ layers:
 
   # Layer 2: backbone_conv3
   - out_offset: 0x3000
-    in_offset: 0x0000 #0x4000
+    in_offset: 0x0000  # 0x4000
     processors: 0xffffffff00000000
     output_processors: 0xffffffffffffffff
     operation: conv2d

requirements.txt

@@ -5,7 +5,6 @@ scipy>=1.3.0
 torch==1.8.1
 pytest~=4.6.4
 onnx>=1.9.0
-yamllint>=1.26.1,<1.27
 GitPython>=3.1.18
 PyGithub>=1.55
 rich>=11.2.0