diff --git a/.ci/ipas_default.config b/.ci/ipas_default.config new file mode 100644 index 00000000..bdb8fefb --- /dev/null +++ b/.ci/ipas_default.config @@ -0,0 +1,409 @@ + +### Bandit config file generated from: +# './bandit/bandit/cli/config_generator.py --out ipas_default.config' + +### This config may optionally select a subset of tests to run or skip by +### filling out the 'tests' and 'skips' lists given below. If no tests are +### specified for inclusion then it is assumed all tests are desired. The skips +### set will remove specific tests from the include set. This can be controlled +### using the -t/-s CLI options. Note that the same test ID should not appear +### in both 'tests' and 'skips', this would be nonsensical and is detected by +### Bandit at runtime. + +# Available tests: +# B101 : assert_used +# B102 : exec_used +# B103 : set_bad_file_permissions +# B104 : hardcoded_bind_all_interfaces +# B105 : hardcoded_password_string +# B106 : hardcoded_password_funcarg +# B107 : hardcoded_password_default +# B108 : hardcoded_tmp_directory +# B110 : try_except_pass +# B112 : try_except_continue +# B201 : flask_debug_true +# B301 : pickle +# B302 : marshal +# B303 : md5 +# B304 : ciphers +# B305 : cipher_modes +# B306 : mktemp_q +# B307 : eval +# B308 : mark_safe +# B310 : urllib_urlopen +# B311 : random +# B312 : telnetlib +# B313 : xml_bad_cElementTree +# B314 : xml_bad_ElementTree +# B315 : xml_bad_expatreader +# B316 : xml_bad_expatbuilder +# B317 : xml_bad_sax +# B318 : xml_bad_minidom +# B319 : xml_bad_pulldom +# B320 : xml_bad_etree +# B321 : ftplib +# B323 : unverified_context +# B324 : hashlib_new_insecure_functions +# B401 : import_telnetlib +# B402 : import_ftplib +# B403 : import_pickle +# B404 : import_subprocess +# B405 : import_xml_etree +# B406 : import_xml_sax +# B407 : import_xml_expat +# B408 : import_xml_minidom +# B409 : import_xml_pulldom +# B410 : import_lxml +# B411 : import_xmlrpclib +# B412 : import_httpoxy +# B413 : import_pycrypto +# B501 : request_with_no_cert_validation +# B502 : ssl_with_bad_version +# B503 : ssl_with_bad_defaults +# B504 : ssl_with_no_version +# B505 : weak_cryptographic_key +# B506 : yaml_load +# B507 : ssh_no_host_key_verification +# B601 : paramiko_calls +# B602 : subprocess_popen_with_shell_equals_true +# B603 : subprocess_without_shell_equals_true +# B604 : any_other_function_with_shell_equals_true +# B605 : start_process_with_a_shell +# B606 : start_process_with_no_shell +# B607 : start_process_with_partial_path +# B608 : hardcoded_sql_expressions +# B609 : linux_commands_wildcard_injection +# B610 : django_extra_used +# B611 : django_rawsql_used +# B701 : jinja2_autoescape_false +# B702 : use_of_mako_templates +# B703 : django_mark_safe + +# (optional) list included test IDs here, eg '[B101, B406]': +# IPAS Required Checkers. Do not disable these +# Additional checkers may be added if desired +tests: + [ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B320', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B410', 'B411', 'B412', 'B413'] + +# (optional) list skipped test IDs here, eg '[B101, B406]': +# The following checkers are not required but be added to tests list if desired +skips: + [ 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', 'B108', 'B110', 'B112', 'B201', 'B501', 'B502', 'B503', 'B504', 'B505', 'B506', 'B507', 'B601', 'B602', 'B603', 'B604', 'B605', 'B606', 'B607', 'B608', 'B609', 'B610', 'B611', 'B701', 'B702', 'B703'] + + +# Added to exclude some path which are not actual source code for this project +exclude_dirs: [ + '.tox/', + '.vscode/', + '.git/', + 'build/', +] + +### (optional) plugin settings - some test plugins require configuration data +### that may be given here, per-plugin. All bandit test plugins have a built in +### set of sensible defaults and these will be used if no configuration is +### provided. It is not necessary to provide settings for every (or any) plugin +### if the defaults are acceptable. + +any_other_function_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +assert_used: + skips: [] +hardcoded_tmp_directory: + tmp_dirs: + - /tmp + - /var/tmp + - /dev/shm +linux_commands_wildcard_injection: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +ssl_with_bad_defaults: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +ssl_with_bad_version: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 + - PROTOCOL_TLSv1 + - SSLv3_METHOD + - TLSv1_METHOD +start_process_with_a_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_no_shell: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +start_process_with_partial_path: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_popen_with_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +subprocess_without_shell_equals_true: + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - subprocess.run +try_except_continue: + check_typed_exception: false +try_except_pass: + check_typed_exception: false +weak_cryptographic_key: + weak_key_size_dsa_high: 1024 + weak_key_size_dsa_medium: 2048 + weak_key_size_ec_high: 160 + weak_key_size_ec_medium: 224 + weak_key_size_rsa_high: 1024 + weak_key_size_rsa_medium: 2048 diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml new file mode 100644 index 00000000..1b806bf3 --- /dev/null +++ b/.github/workflows/code_scan.yml @@ -0,0 +1,69 @@ +name: Security Code Scan + +on: + schedule: + # every UTC 7PM from Mon to Fri + - cron: "0 19 * * 1-5" + push: + branches: + - releases/* + workflow_dispatch: # run on request (no need for PR) + +# Declare default permissions as read only. +permissions: read-all + +jobs: + Trivy-Scan: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set up Python + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + with: + python-version: "3.10" + - name: Install dependencies + run: | + pip install . + pip freeze > requirements.txt + - name: Run Trivy security scan + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: fs + scan-ref: requirements.txt + output: trivy-scan-results.txt + - name: Run Trivy spdx scan + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: fs + scan-ref: requirements.txt + format: spdx-json + output: trivy-scan-results.spdx.json + - name: Upload Trivy scan results + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: trivy-scan-results + path: trivy-scan-results.* + # Use always() to always run this step to publish scan results when there are test failures + if: ${{ always() }} + + Bandit-Scan: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set up Python + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + with: + python-version: "3.10" + - name: Install dependencies + run: pip install bandit + - name: Bandit Scanning + run: bandit -r -c .ci/ipas_default.config . -f txt -o bandit-scan-results.txt + - name: Upload Bandit artifact + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: bandit-scan-results + path: bandit-scan-results.txt + # Use always() to always run this step to publish scan results when there are test failures + if: ${{ always() }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..5de85eb1 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,41 @@ +name: E2E test + +on: + schedule: + # every UTC 7PM from Mon to Fri + - cron: "0 19 * * 1-5" + workflow_dispatch: # run on request (no need for PR) + +# Declare default permissions as read only. +permissions: read-all + +jobs: + Pre-Merge-Checks: + uses: ./.github/workflows/pre_merge.yml + + E2E-Test: + runs-on: ubuntu-22.04 + needs: Pre-Merge-Checks + timeout-minutes: 120 + # This is what will cancel the job concurrency + concurrency: + group: ${{ github.workflow }}-E2E-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Install Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.10" + - name: Install tox + run: python -m pip install tox==4.4.6 + - name: Run E2E Test + run: tox -vv -e val-py310 -- tests/e2e --csv=.tox/val-py310/e2e-test.csv + - name: Upload artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: e2e-test-results + path: .tox/val-py310/*.csv + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index eaf3ed28..1027194f 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -4,7 +4,7 @@ on: push: branches: - develop - - releases/** + - releases/* pull_request: types: - opened @@ -61,22 +61,22 @@ jobs: - name: Install tox run: python -m pip install tox==4.4.6 - name: Run unit test - run: tox -vv -e pytest-${{ matrix.tox-env }} -- tests/unit --csv=.tox/pytest-${{ matrix.tox-env }}/unit-test.csv - --cov=openvino_xai --cov-report term --cov-report xml:.tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml + run: tox -vv -e dev-${{ matrix.tox-env }} -- tests/unit --csv=.tox/dev-${{ matrix.tox-env }}/unit-test.csv + --cov=openvino_xai --cov-report term --cov-report xml:.tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml - name: Upload artifacts uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: unit-test-results-${{ matrix.tox-env }} path: | - .tox/pytest-${{ matrix.tox-env }}/*.csv - .tox/pytest-${{ matrix.tox-env }}/*.xml + .tox/dev-${{ matrix.tox-env }}/*.csv + .tox/dev-${{ matrix.tox-env }}/*.xml # Use always() to always run this step to publish test results when there are test failures if: ${{ always() }} - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: - files: .tox/pytest-${{ matrix.tox-env }}/unit-test-coverage.xml - flags: pytest-${{ matrix.tox-env }} + files: .tox/dev-${{ matrix.tox-env }}/unit-test-coverage.xml + flags: dev-${{ matrix.tox-env }} Integration-Test: runs-on: ubuntu-22.04 @@ -96,11 +96,11 @@ jobs: - name: Install tox run: python -m pip install tox==4.4.6 - name: Run Integration Test - run: tox -vv -e pytest-py310 -- tests/integration --csv=.tox/pytest-py310/intg-test.csv + run: tox -vv -e dev-py310 -- tests/integration --csv=.tox/dev-py310/intg-test.csv - name: Upload artifacts uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: intg-test-results - path: .tox/pytest-py310/*.csv + path: .tox/dev-py310/*.csv # Use always() to always run this step to publish test results when there are test failures if: ${{ always() }} diff --git a/.gitignore b/.gitignore index 3ff03347..cedb1e45 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ otx_models/ .mypy_cache .ruff_cache .coverage +.tox *.jpg *.jpeg diff --git a/openvino_xai/common/utils.py b/openvino_xai/common/utils.py index 29a5f2ab..4f2bd8b5 100644 --- a/openvino_xai/common/utils.py +++ b/openvino_xai/common/utils.py @@ -39,6 +39,7 @@ def has_xai(model: ov.Model) -> bool: return False +# Not a part of product def retrieve_otx_model(data_dir: str | Path, model_name: str, dir_url=None) -> None: destination_folder = Path(data_dir) / "otx_models" os.makedirs(destination_folder, exist_ok=True) @@ -50,7 +51,7 @@ def retrieve_otx_model(data_dir: str | Path, model_name: str, dir_url=None) -> N for post_fix in ["xml", "bin"]: if not os.path.isfile(os.path.join(destination_folder, model_name + f".{post_fix}")): - urlretrieve( + urlretrieve( # nosec B310 f"{dir_url}/{snapshot_file}.{post_fix}", f"{destination_folder}/{model_name}.{post_fix}", ) diff --git a/pyproject.toml b/pyproject.toml index 4ac86b64..7a20280b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dev = [ "pre-commit==3.7.0", "addict", ] -dev_timm = [ +val = [ "timm==0.9.5", "onnx==1.14.1", ] diff --git a/tests/integration/test_classification_timm.py b/tests/e2e/test_classification_timm.py similarity index 100% rename from tests/integration/test_classification_timm.py rename to tests/e2e/test_classification_timm.py diff --git a/tox.ini b/tox.ini index 4c265e21..2e439416 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,12 @@ skip_install = true commands = pre-commit run --all-files -[testenv:pytest-{py310, py311}] -extras = dev,dev_timm +[testenv:dev-{py310, py311}] +extras = dev +commands = + pytest -ra --showlocals {posargs:tests/} + +[testenv:val-{py310, py311}] +extras = dev,val commands = pytest -ra --showlocals {posargs:tests/}