Merge branch 'master' into nashez/jit_less_emitter

openvinotoolkit · Nov 6, 2024 · b73067e · b73067e
2 parents 8590e10 + 7666e39
commit b73067e
Show file tree

Hide file tree

Showing 754 changed files with 18,721 additions and 13,657 deletions.
diff --git a/.github/actions/handle_docker/action.yml b/.github/actions/handle_docker/action.yml
@@ -27,25 +27,28 @@ runs:
   using: 'composite'
   steps:
     - name: Checkout head
-      uses: actions/checkout@v4
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
 
     - name: Checkout base
-      uses: actions/checkout@v4
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       with:
         ref: ${{ github.base_ref || github.event.merge_group.base_ref }}
         sparse-checkout: ${{ inputs.dockerfiles_root_dir }}/docker_tag
         path: base
 
     - name: Install Python dependencies
-      uses: py-actions/py-dependency-install@v4
+      uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0
       with:
         path: "${{ github.action_path }}/requirements.txt"
         update-setuptools: "false"
         update-wheel: "false"
 
     - name: Set up Docker Buildx
       id: buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
+      with:
+        driver-opts: |
+          image=${{ inputs.registry }}/dockerio/moby/buildkit:buildx-stable-1
 
     - name: Handle docker images
       id: handle_images
@@ -62,7 +65,7 @@ runs:
         --base_tag_file "base/${{ inputs.dockerfiles_root_dir }}/docker_tag" \
         --docker_env_changed "${{ fromJSON(inputs.changed_components).docker_env }}" \
         --dockerfiles_changed "${{ fromJSON(inputs.changed_components).dockerfiles }}" \
-        --docker_builder "${{ steps.buildx.outputs.name}}" \
+        --docker_builder "${{ steps.buildx.outputs.name }}" \
         --repo "${{ github.repository }}" \
         --ref_name "${{ github.ref_name }}" \
         $([[ -n $pr ]] && echo "--pr $pr" || echo '-s ${{ github.sha }}') \

diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml
@@ -56,7 +56,7 @@ runs:
 
     - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }}
       name: Setup Python ${{ inputs.version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
       with:
         python-version: ${{ inputs.version }}
       env:

diff --git a/.github/actions/smart-ci/action.yml b/.github/actions/smart-ci/action.yml
@@ -30,7 +30,6 @@ inputs:
   components_config_schema:
     description: "Path to the schema file for components configuration"
     required: false
-    default: ".github/actions/smart-ci/components_schema.yml"
   labeler_config:
     description: "Path to labeler configuration file"
     required: false
@@ -66,7 +65,7 @@ runs:
   using: "composite"
   steps:
     - name: Wait for labeler to finish
-      uses: lewagon/[email protected].1
+      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc # v1.3.4
       if: ${{ github.event_name == 'pull_request' }}
       with:
         ref: ${{ github.event.pull_request.head.sha }}
@@ -75,13 +74,13 @@ runs:
         wait-interval: 10
 
     - name: checkout components file
-      uses: actions/checkout@v4
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
       with:
         sparse-checkout: .github/components.yml
         sparse-checkout-cone-mode: false
 
     - name: Install Python dependencies
-      uses: py-actions/py-dependency-install@v4
+      uses: py-actions/py-dependency-install@30aa0023464ed4b5b116bd9fbdab87acf01a484e # v4.1.0
       with:
         path: "${{ github.action_path }}/requirements.txt"
         update-setuptools: "false"
@@ -101,11 +100,12 @@ runs:
           -f "${{ inputs.ref_name }}" \
           -p "${{ inputs.component_pattern }}" \
           -c "${{ inputs.components_config }}" \
-          -m "${{ inputs.components_config_schema }}" \
+          -m "${{ inputs.components_config_schema || env.DEFAULT_CONFIG_SCHEMA }}" \
           -l "${{ inputs.labeler_config }}" \
           --enable_for_org "${{ inputs.enable_for_org }}" \
           --skip-when-only-listed-labels-set "${{ inputs.skip_when_only_listed_labels_set }}" \
           --skip-when-only-listed-files-changed "${{ inputs.skip_when_only_listed_files_changed }}"
       shell: bash
       env:
         GITHUB_TOKEN: ${{ inputs.repo_token }}
+        DEFAULT_CONFIG_SCHEMA: "${{ github.action_path }}/components_schema.yml"
diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag
@@ -1 +1 @@
-pr-26993
+pr-27384
diff --git a/.github/workflows/android_x64.yml b/.github/workflows/android_x64.yml
@@ -135,6 +135,7 @@ jobs:
             -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
             -DENABLE_LTO=ON \
             -DENABLE_PYTHON=OFF \
+            -DENABLE_TESTS=ON \
             -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_GENAI_REPO }} \
             -S ${OPENVINO_REPO} \
             -B ${BUILD_DIR}

diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml
@@ -29,7 +29,7 @@ jobs:
           packages: graphviz texlive liblua5.2-0 libclang1-9 libclang-cpp9
           version: 3.0
 
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         id: cp310
         with:
           python-version: '3.10'
@@ -41,7 +41,7 @@ jobs:
       - name: Install python dependencies
         run: |
           python3 -m pip install -r docs/requirements.txt
-          (cd docs/openvino_sphinx_theme && python3 setup.py install)
+          (cd docs/openvino_sphinx_theme && python3 -m pip install .)
           python3 -m pip install docs/openvino_custom_sphinx_sitemap
 
       - name: Download and install doxygen

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: Setup python
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: '3.10.10'
           architecture: 'x64'

diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -25,15 +25,67 @@ env:
   TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
 
 jobs:
+
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg'
+
+  Docker:
+    needs: Smart_CI
+    runs-on: aks-linux-4-cores-16gb-docker-build
+    container:
+      image: openvinogithubactions.azurecr.io/docker_build:0.2
+      volumes:
+        - /mount:/mount
+    outputs:
+      images: "${{ steps.handle_docker.outputs.images }}"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: ./.github/actions/handle_docker
+        id: handle_docker
+        with:
+          images: |
+            ov_build/ubuntu_20_04_x64
+          registry: 'openvinogithubactions.azurecr.io'
+          dockerfiles_root_dir: '.github/dockerfiles'
+          changed_components: ${{ needs.smart_ci.outputs.changed_components }}
+
   Build:
+    needs: Docker
     timeout-minutes: 150
     defaults:
       run:
         shell: bash
     runs-on: aks-linux-16-cores-32gb
     if: ${{ github.repository_owner == 'openvinotoolkit' }}
     container:
-      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}
+      volumes:
+        - /mount:/mount
+      options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
     env:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       CMAKE_BUILD_TYPE: 'Release'
@@ -43,13 +95,16 @@ jobs:
       OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
       BUILD_DIR: /__w/openvino/openvino/openvino_build
       COVERITY_TOOL_DIR: /__w/openvino/openvino/coverity_tool
+      CMAKE_CXX_COMPILER_LAUNCHER: sccache
+      CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
+      SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt
+      SCCACHE_LOG: warn
+      SCCACHE_AZURE_KEY_PREFIX: coverity_lin
+      SCCACHE_CACHE_SIZE: 50G
 
     steps:
-      - name: Install git
-        run: |
-          apt-get update
-          apt-get install --assume-yes --no-install-recommends git ca-certificates
-
       - name: Clone OpenVINO
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -86,7 +141,9 @@ jobs:
       #
       # Build
       #
-
+      - name: Clean sccache stats
+        run: ${SCCACHE_PATH} --zero-stats
+
       - name: CMake configure - OpenVINO
         run: |
           cmake \
@@ -108,12 +165,13 @@ jobs:
 
       - name: Cmake build - OpenVINO with Coverity
         run: ${COVERITY_TOOL_DIR}/cov-analysis*/bin/cov-build --dir ${BUILD_DIR}/cov-int cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
+
+      - name: Show sccache stats
+        run: ${SCCACHE_PATH} --show-stats
 
       - name: Pack Artefacts
-        run: |
-          pushd ${BUILD_DIR}
-            tar -cvf - cov-int | pigz > openvino.tgz
-          popd
+        run: tar -cvf - cov-int | pigz > openvino.tgz
+        working-directory: ${{ env.BUILD_DIR }}
 
       - name: Submit artefacts
         run: |

diff --git a/.github/workflows/job_gpu_tests.yml b/.github/workflows/job_gpu_tests.yml
@@ -74,7 +74,7 @@ jobs:
           DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
           TZ: "Europe/London" # to prevent tzdata from waiting user input
       - name: Setup Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

diff --git a/.github/workflows/job_openvino_js.yml b/.github/workflows/job_openvino_js.yml
@@ -52,7 +52,7 @@ jobs:
 
       - name: Setup Node ${{ env.NODE_VERSION }}
         if: runner.os != 'Linux' # Node is already installed in the Docker image
-        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
         with:
           node-version: ${{ env.NODE_VERSION }}
 

diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml
@@ -111,10 +111,7 @@ jobs:
         run: |
           # To enable pytest parallel features
           python3 -m pip install pytest-xdist[psutil]
-          # For torchvision to OpenVINO preprocessing converter
-          python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt
-
-          # TODO: replace with Python API tests requirements
+          python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt
           python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt
 
       #
@@ -158,6 +155,9 @@ jobs:
 
       - name: Install Python Layer tests dependencies
         run: |
+          # For torchvision to OpenVINO preprocessing converter
+          python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt
+
           # layer test requirements
           python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt
 
@@ -267,7 +267,7 @@ jobs:
         if: ${{ fromJSON(inputs.affected-components).Python_API.test }}
         run: |
           python3 -m pip uninstall -y numpy
-          python3 -m pip install "numpy>=2.0.0,<2.1.0"
+          python3 -m pip install "numpy>=2.0.0,<2.2.0"
           python3 -m pip install -r ${INSTALL_TEST_DIR}/bindings/python/requirements_test.txt
           # for 'template' extension
           export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH

diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml
@@ -20,12 +20,15 @@ on:
         description: 'Components that are affected by changes in the commit defined by the Smart CI Action'
         type: string
         required: true
+      python-version:
+        description: 'Python version to setup. E.g., "3.11"'
+        type: string
+        required: true
 
 permissions: read-all
 
 env:
   PIP_CACHE_PATH: /mount/caches/pip/linux
-  PYTHON_VERSION: '3.11'
   TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
 
 jobs:
@@ -63,10 +66,10 @@ jobs:
         if: runner.os == 'macOS'
         run: brew install pigz
 
-      - name: Setup Python ${{ env.PYTHON_VERSION }}
+      - name: Setup Python ${{ inputs.python-version }}
         uses: ./.github/actions/setup_python
         with:
-          version: ${{ env.PYTHON_VERSION }}
+          version: ${{ inputs.python-version }}
           pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }}
           should-setup-pip-paths: ${{ runner.os == 'Linux' }}
           self-hosted-runner: ${{ runner.os == 'Linux' }}

diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml
@@ -146,6 +146,7 @@ jobs:
       shell: bash
       container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"]}'
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
+      python-version: '3.11'
     if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
 
   CXX_Unit_Tests:

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -258,6 +258,7 @@ jobs:
       runner: 'macos-13'
       shell: bash
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
+      python-version: '3.11'
     if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
 
   CXX_Unit_Tests:

diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml
@@ -258,6 +258,7 @@ jobs:
       runner: 'macos-13-xlarge'
       shell: bash
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
+      python-version: '3.11'
     if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
 
   CXX_Unit_Tests:

diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml
@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Python
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: '3.10'
 

diff --git a/.github/workflows/ovc.yml b/.github/workflows/ovc.yml
@@ -22,7 +22,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Python
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: '3.10'
 

diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml
@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Python
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: '3.9'
 

diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml
@@ -491,6 +491,7 @@ jobs:
       shell: bash
       container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_x64 }}", "volumes": ["/mount:/mount"]}'
       affected-components: ${{ needs.smart_ci.outputs.affected_components }}
+      python-version: '3.11'
     if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
 
   iGPU: