tools/downloader: put the model tools into a Python package

A package is needed so that we can make them installable via pip later.
I don't think you can install individual modules (and even if you could,
the names `downloader`, etc., are way too generic and would pollute the global
namespace).

I also used this refactoring to rename `common` to `_common`, to signify that
it's an internal module.

`open_model_zoo` is a namespace package, so that if we end up creating other
installable OMZ functionality (like a model API or something), we can put it
in a different subpackage of `open_model_zoo`. `open_model_zoo.model_tools`,
however, is not a namespace package, since there should be no reason to extend
it.

CONTRIBUTING.md

@@ -76,7 +76,7 @@ Description of the model. Must match with the description from the model [docume
 
 **`task_type`**
 
-[Model task type](tools/downloader/README.md#model-information-dumper-usage). If there is no task type of your model, add a new one to the list `KNOWN_TASK_TYPES` of the [tools/downloader/common.py](tools/downloader/common.py) file.
+[Model task type](tools/downloader/README.md#model-information-dumper-usage). If there is no task type of your model, add a new one to the list `KNOWN_TASK_TYPES` of the [`open_model_zoo.model_tools._common`](tools/downloader/src/open_model_zoo/model_tools/_common.py) module.
 
 **`files`**
 

tools/downloader/common.py → tools/downloader/src/open_model_zoo/model_tools/_common.py

@@ -34,7 +34,7 @@
 import yaml
 
 DOWNLOAD_TIMEOUT = 5 * 60
-OMZ_ROOT = Path(__file__).resolve().parents[2]
+OMZ_ROOT = Path(__file__).resolve().parents[5]
 MODEL_ROOT = OMZ_ROOT / 'models'
 
 # make sure to update the documentation if you modify these

tools/downloader/converter.py → tools/downloader/src/open_model_zoo/model_tools/converter.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2019 Intel Corporation
+# Copyright (c) 2019-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,11 +19,11 @@
 
 from pathlib import Path
 
-import common
+from open_model_zoo.model_tools import _common
 
 
 def run_pre_convert(reporter, model, output_dir, args):
-    script = common.MODEL_ROOT / model.subdirectory / 'pre-convert.py'
+    script = _common.MODEL_ROOT / model.subdirectory / 'pre-convert.py'
     if not script.exists():
         return True
 
@@ -35,7 +33,7 @@ def run_pre_convert(reporter, model, output_dir, args):
     cmd = [str(args.python), '--', str(script), '--',
         str(args.download_dir / model.subdirectory), str(output_dir / model.subdirectory)]
 
-    reporter.print('Pre-convert command: {}', common.command_string(cmd))
+    reporter.print('Pre-convert command: {}', _common.command_string(cmd))
     reporter.print(flush=True)
 
     success = True if args.dry_run else reporter.job_context.subprocess(cmd)
@@ -47,11 +45,15 @@ def convert_to_onnx(reporter, model, output_dir, args, template_variables):
     reporter.print_section_heading('{}Converting {} to ONNX',
         '(DRY RUN) ' if args.dry_run else '', model.name)
 
+    converter_path = Path(__file__).absolute().parent / \
+        'internal_scripts' / model.converter_to_onnx
+
     conversion_to_onnx_args = [string.Template(arg).substitute(template_variables)
                                for arg in model.conversion_to_onnx_args]
-    cmd = [str(args.python), str(Path(__file__).absolute().parent / model.converter_to_onnx), *conversion_to_onnx_args]
 
-    reporter.print('Conversion to ONNX command: {}', common.command_string(cmd))
+    cmd = [str(args.python), '--', str(converter_path), *conversion_to_onnx_args]
+
+    reporter.print('Conversion to ONNX command: {}', _common.command_string(cmd))
     reporter.print(flush=True)
 
     success = True if args.dry_run else reporter.job_context.subprocess(cmd)
@@ -113,14 +115,14 @@ def main():
     extra_mo_args = args.extra_mo_args or []
 
     if args.precisions is None:
-        requested_precisions = common.KNOWN_PRECISIONS
+        requested_precisions = _common.KNOWN_PRECISIONS
     else:
         requested_precisions = set(args.precisions.split(','))
-        unknown_precisions = requested_precisions - common.KNOWN_PRECISIONS
+        unknown_precisions = requested_precisions - _common.KNOWN_PRECISIONS
         if unknown_precisions:
             sys.exit('Unknown precisions specified: {}.'.format(', '.join(sorted(unknown_precisions))))
 
-    models = common.load_models_from_args(parser, args)
+    models = _common.load_models_from_args(parser, args)
 
     output_dir = args.download_dir if args.output_dir is None else args.output_dir
 
@@ -144,7 +146,7 @@ def convert(reporter, model):
         model_format = model.framework
 
         template_variables = {
-            'config_dir': common.MODEL_ROOT / model.subdirectory,
+            'config_dir': _common.MODEL_ROOT / model.subdirectory,
             'conv_dir': output_dir / model.subdirectory,
             'dl_dir': args.download_dir / model.subdirectory,
             'mo_dir': mo_path.parent,
@@ -171,7 +173,7 @@ def convert(reporter, model):
             reporter.print_section_heading('{}Converting {} to IR ({})',
                 '(DRY RUN) ' if args.dry_run else '', model.name, model_precision)
 
-            reporter.print('Conversion command: {}', common.command_string(mo_cmd))
+            reporter.print('Conversion command: {}', _common.command_string(mo_cmd))
 
             if not args.dry_run:
                 reporter.print(flush=True)
@@ -183,13 +185,13 @@ def convert(reporter, model):
 
         return True
 
-    reporter = common.Reporter(common.DirectOutputContext())
+    reporter = _common.Reporter(_common.DirectOutputContext())
 
     if args.jobs == 1 or args.dry_run:
         results = [convert(reporter, model) for model in models]
     else:
-        results = common.run_in_parallel(args.jobs,
-            lambda context, model: convert(common.Reporter(context), model),
+        results = _common.run_in_parallel(args.jobs,
+            lambda context, model: convert(_common.Reporter(context), model),
             models)
 
     failed_models = [model.name for model, successful in zip(models, results) if not successful]

tools/downloader/downloader.py → tools/downloader/src/open_model_zoo/model_tools/downloader.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python3
-
 """
- Copyright (c) 2018 Intel Corporation
+ Copyright (c) 2018-2021 Intel Corporation
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -32,7 +30,7 @@
 
 from pathlib import Path
 
-import common
+from open_model_zoo.model_tools import _common
 
 CHUNK_SIZE = 1 << 15 if sys.stdout.isatty() else 1 << 20
 
@@ -246,7 +244,7 @@ def download_model(reporter, args, cache, session_factory, requested_precisions,
     for model_file in model.files:
         if len(model_file.name.parts) == 2:
             p = model_file.name.parts[0]
-            if p in common.KNOWN_PRECISIONS and p not in requested_precisions:
+            if p in _common.KNOWN_PRECISIONS and p not in requested_precisions:
                 continue
 
         model_file_reporter = reporter.with_event_context(model=model.name, model_file=model_file.name.as_posix())
@@ -327,7 +325,7 @@ def main():
     parser.add_argument('--print_all', action='store_true', help='print all available models')
     parser.add_argument('--precisions', metavar='PREC[,PREC...]',
                         help='download only models with the specified precisions (actual for DLDT networks); specify one or more of: '
-                             + ','.join(common.KNOWN_PRECISIONS))
+                             + ','.join(_common.KNOWN_PRECISIONS))
     parser.add_argument('-o', '--output_dir', type=Path, metavar='DIR',
         default=Path.cwd(), help='path where to save models')
     parser.add_argument('--cache_dir', type=Path, metavar='DIR',
@@ -344,22 +342,22 @@ def main():
     args = parser.parse_args()
 
     def make_reporter(context):
-        return common.Reporter(context,
+        return _common.Reporter(context,
             enable_human_output=args.progress_format == 'text',
             enable_json_output=args.progress_format == 'json')
 
-    reporter = make_reporter(common.DirectOutputContext())
+    reporter = make_reporter(_common.DirectOutputContext())
 
     cache = NullCache() if args.cache_dir is None else DirCache(args.cache_dir)
-    models = common.load_models_from_args(parser, args)
+    models = _common.load_models_from_args(parser, args)
 
     failed_models = set()
 
     if args.precisions is None:
-        requested_precisions = common.KNOWN_PRECISIONS
+        requested_precisions = _common.KNOWN_PRECISIONS
     else:
         requested_precisions = set(args.precisions.split(','))
-        unknown_precisions = requested_precisions - common.KNOWN_PRECISIONS
+        unknown_precisions = requested_precisions - _common.KNOWN_PRECISIONS
         if unknown_precisions:
             sys.exit('Unknown precisions specified: {}.'.format(', '.join(sorted(unknown_precisions))))
 
@@ -369,7 +367,7 @@ def make_reporter(context):
             results = [download_model(reporter, args, cache, session_factory, requested_precisions, model)
                 for model in models]
         else:
-            results = common.run_in_parallel(args.jobs,
+            results = _common.run_in_parallel(args.jobs,
                 lambda context, model: download_model(
                     make_reporter(context), args, cache, session_factory, requested_precisions, model),
                 models)

tools/downloader/info_dumper.py → tools/downloader/src/open_model_zoo/model_tools/info_dumper.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2019 Intel Corporation
+# Copyright (c) 2019-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +18,7 @@
 
 from pathlib import Path
 
-import common
+from open_model_zoo.model_tools import _common
 
 def to_info(model):
     return {
@@ -44,7 +42,7 @@ def main():
     parser.add_argument('--print_all', action='store_true', help='print all available models')
     args = parser.parse_args()
 
-    models = common.load_models_from_args(parser, args)
+    models = _common.load_models_from_args(parser, args)
 
     json.dump(list(map(to_info, models)), sys.stdout, indent=4)
     print() # add a final newline

tools/downloader/quantizer.py → tools/downloader/src/open_model_zoo/model_tools/quantizer.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2020 Intel Corporation
+# Copyright (c) 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,7 +22,7 @@
 
 import yaml
 
-import common
+from open_model_zoo.model_tools import _common
 
 DEFAULT_POT_CONFIG_BASE = {
     'compression': {
@@ -40,12 +38,12 @@
     },
 }
 
-DATASET_DEFINITIONS_PATH = common.OMZ_ROOT / 'tools/accuracy_checker/dataset_definitions.yml'
+DATASET_DEFINITIONS_PATH = _common.OMZ_ROOT / 'tools/accuracy_checker/dataset_definitions.yml'
 
 def quantize(reporter, model, precision, args, output_dir, pot_path, pot_env):
-    input_precision = common.KNOWN_QUANTIZED_PRECISIONS[precision]
+    input_precision = _common.KNOWN_QUANTIZED_PRECISIONS[precision]
 
-    pot_config_base_path = common.MODEL_ROOT / model.subdirectory / 'quantization.yml'
+    pot_config_base_path = _common.MODEL_ROOT / model.subdirectory / 'quantization.yml'
 
     try:
         with pot_config_base_path.open('rb') as pot_config_base_file:
@@ -55,7 +53,7 @@ def quantize(reporter, model, precision, args, output_dir, pot_path, pot_env):
 
     pot_config_paths = {
         'engine': {
-            'config': str(common.MODEL_ROOT/ model.subdirectory / 'accuracy-check.yml'),
+            'config': str(_common.MODEL_ROOT/ model.subdirectory / 'accuracy-check.yml'),
         },
         'model': {
             'model': str(args.model_dir / model.subdirectory / input_precision / (model.name + '.xml')),
@@ -90,9 +88,9 @@ def quantize(reporter, model, precision, args, output_dir, pot_path, pot_env):
         '--output-dir={}'.format(pot_output_dir),
     ]
 
-    reporter.print('Quantization command: {}', common.command_string(pot_cmd))
+    reporter.print('Quantization command: {}', _common.command_string(pot_cmd))
     reporter.print('Quantization environment: {}',
-        ' '.join('{}={}'.format(k, common.quote_arg(v))
+        ' '.join('{}={}'.format(k, _common.quote_arg(v))
             for k, v in sorted(pot_env.items())))
 
     success = True
@@ -145,22 +143,22 @@ def main():
             sys.exit('Unable to locate Post-Training Optimization Toolkit. '
                 + 'Use --pot or run setupvars.sh/setupvars.bat from the OpenVINO toolkit.')
 
-    models = common.load_models_from_args(parser, args)
+    models = _common.load_models_from_args(parser, args)
 
     # We can't mark it as required, because it's not required when --print_all is specified.
     # So we have to check it manually.
     if not args.dataset_dir:
         sys.exit('--dataset_dir must be specified.')
 
     if args.precisions is None:
-        requested_precisions = common.KNOWN_QUANTIZED_PRECISIONS.keys()
+        requested_precisions = _common.KNOWN_QUANTIZED_PRECISIONS.keys()
     else:
         requested_precisions = set(args.precisions.split(','))
-        unknown_precisions = requested_precisions - common.KNOWN_QUANTIZED_PRECISIONS.keys()
+        unknown_precisions = requested_precisions - _common.KNOWN_QUANTIZED_PRECISIONS.keys()
         if unknown_precisions:
             sys.exit('Unknown precisions specified: {}.'.format(', '.join(sorted(unknown_precisions))))
 
-    reporter = common.Reporter(common.DirectOutputContext())
+    reporter = _common.Reporter(_common.DirectOutputContext())
 
     output_dir = args.output_dir or args.model_dir