From 20c6b437c42f5adf5dc7761d40783dc78c36ad0f Mon Sep 17 00:00:00 2001 From: atobisze Date: Thu, 17 Oct 2024 08:24:54 +0200 Subject: [PATCH 1/2] WIP using cache * Use Docker cache more for CI * Build ovms_test in parallel to ovms --- .bazelrc | 17 +++++++++++++- Dockerfile.redhat | 20 ++++++++-------- Dockerfile.ubuntu | 24 ++++++++++---------- Makefile | 14 ++++++++---- src/BUILD | 58 ++++++++++++++++++++++++----------------------- 5 files changed, 77 insertions(+), 56 deletions(-) diff --git a/.bazelrc b/.bazelrc index ba93a08b22..f648fd38ae 100644 --- a/.bazelrc +++ b/.bazelrc @@ -78,7 +78,7 @@ build --cxxopt=-fstack-clash-protection build --cxxopt=-Wformat build --cxxopt=-Wformat-security build --cxxopt=-Werror=format-security - +build --cxxopt=-DOVMS_DUMP_TO_FILE=0 # Adding "--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" creates parity with TF # compilation options. It also addresses memory use due to # copy-on-write semantics of std::strings of the older ABI. @@ -97,3 +97,18 @@ build --cxxopt=-DOVMS_DUMP_TO_FILE=0 # very strict in sandboxing test --test_env PYTHONPATH=/opt/intel/openvino/python:/ovms/bazel-bin/src/python/binding test --test_env LD_LIBRARY_PATH=/opt/opencv/lib/:/opt/intel/openvino/runtime/lib/intel64/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ +# genai hardcodes path during build time so without this so is not visible +test --test_env OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so + +build --remote_cache=http://10.123.246.16:9666 +build --remote_upload_local_results=true +build:mp_on_py_on --define=MEDIAPIPE_DISABLE=0 +build:mp_on_py_on --define=PYTHON_DISABLE=0 +build:mp_on_py_on --disk_cache=/root/.cache/bazel_mp_on_py_on +build:mp_on_py_off --define=MEDIAPIPE_DISABLE=0 +build:mp_on_py_off --define=PYTHON_DISABLE=1 +build:mp_on_py_off --disk_cache=/root/.cache/bazel_mp_on_py_off +build:mp_off_py_off --define=MEDIAPIPE_DISABLE=1 +build:mp_off_py_off --define=PYTHON_DISABLE=1 +build:mp_off_py_off --disk_cache=/root/.cache/bazel_mp_off_py_off +#build --remote_header=x-build-event-log=DEBUG diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 9f40dcd237..abd3b80f0a 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -176,7 +176,7 @@ ARG ov_use_binary=0 ARG DLDT_PACKAGE_URL ARG TEMP_DIR=/tmp/openvino_installer ARG CMAKE_BUILD_TYPE=Release -ARG debug_bazel_flags="--strip=always --define MEDIAPIPE_DISABLE=0 --define PYTHON_DISABLE=0 --//:distro=redhat" +ARG debug_bazel_flags="--strip=always --config=mp_on_py_on --//:distro=redhat" # hadolint ignore=DL3003 RUN if [[ "$NVIDIA" == "1" ]] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_contrib_org/openvino_contrib.git /openvino_contrib && cd /openvino_contrib && git checkout $ov_contrib_branch && git submodule update --init --recursive @@ -189,7 +189,7 @@ RUN dnf install -y http://vault.centos.org/centos/8-stream/PowerTools/x86_64/os/ dnf clean all # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; pip3 install --no-cache-dir -r /openvino/src/bindings/python/wheel/requirements-dev.txt +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; pip3 install --no-cache-dir -r /openvino/src/bindings/python/wheel/requirements-dev.txt WORKDIR /openvino/build RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DENABLE_SAMPLES=0 -DENABLE_INTEL_NPU=OFF -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " .. RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; make --jobs=$JOBS @@ -201,8 +201,8 @@ RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; \ ln -s /usr/local/runtime /opt/intel/openvino && \ ln -s /openvino/scripts/setupvars/setupvars.sh /opt/intel/openvino/setupvars.sh && \ ln -s /opt/intel/openvino /opt/intel/openvino_2024 -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; mkdir -p /opt/intel/openvino && cp -r /openvino/bin/intel64/Release/python /opt/intel/openvino/ -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; cp -r /openvino/tools/ovc/* /opt/intel/openvino/python +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; mkdir -p /opt/intel/openvino && cp -r /openvino/bin/intel64/Release/python /opt/intel/openvino/ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; cp -r /openvino/tools/ovc/* /opt/intel/openvino/python ################## END OF OPENVINO SOURCE BUILD ###################### ################### TAKE OPENVINO FROM A BINARY RELEASE - buildarg ov_use_binary=1 (DEFAULT) ########## @@ -285,7 +285,7 @@ COPY yarn.lock /ovms/ COPY package.json /ovms/ # prebuild dependencies before copying sources -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies +RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest # hadolint ignore=DL3059 RUN cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt @@ -301,7 +301,7 @@ COPY src/ /ovms/src/ WORKDIR /ovms/src/example/SampleCpuExtension/ RUN make -RUN if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; \ +RUN if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ mkdir -p /opt/intel/openvino/python/openvino-2024.5.dist-info && \ echo $'Metadata-Version: 1.0\nName: openvino\nVersion: 2024.5' > /opt/intel/openvino/python/openvino-2024.5.dist-info/METADATA ENV PYTHONPATH=/opt/intel/openvino/python:/ovms/bazel-bin/src/python/binding @@ -332,7 +332,7 @@ RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //src:release_custom_nodes # OVMS # hadolint ignore=DL3059 -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms +RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms //src:ovms_test # hadolint ignore=DL3059 RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //src:libsampleloader.so @@ -354,7 +354,7 @@ RUN rm -Rf /etc/entitlement /etc/rhsm/ca FROM $BUILD_IMAGE as capi-build # C api shared library -ARG CAPI_FLAGS="--strip=always --define MEDIAPIPE_DISABLE=1 --define PYTHON_DISABLE=1 --//:distro=redhat" +ARG CAPI_FLAGS="--strip=always --config=mp_off_py_off --//:distro=redhat" ARG JOBS=40 RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared @@ -384,7 +384,7 @@ WORKDIR / ARG BASE_OS=redhat ARG ov_use_binary=0 ARG FUZZER_BUILD=0 -ARG debug_bazel_flags="--strip=always --define MEDIAPIPE_DISABLE=0 --define PYTHON_DISABLE=0 --//:distro=redhat" +ARG debug_bazel_flags="--strip=always --config=mp_on_py_on --//:distro=redhat" COPY --from=capi-build /ovms_release/lib/libovms_shared.so /ovms_release/lib/ COPY create_package.sh / RUN ./create_package.sh @@ -426,7 +426,7 @@ RUN if [ -f /usr/bin/dnf ] ; then export DNF_TOOL=dnf ; echo -e "max_parallel_do source install_gpu_drivers.sh && rm -rf ./install_gpu_drivers.sh; \ fi ; \ # For image with Python enabled install Python library - if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then \ + if ! [[ $debug_bazel_flags == *"py_off"* ]]; then \ $DNF_TOOL install -y python39-libs --setopt=install_weak_deps=0 --nodocs; \ fi ; \ $DNF_TOOL install -y shadow-utils; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 1466bd84b9..73a896c3c7 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -95,7 +95,7 @@ ARG BASE_IMAGE ENV DEBIAN_FRONTEND=noninteractive SHELL ["/bin/bash", "-xo", "pipefail", "-c"] -ARG debug_bazel_flags="--strip=always --define MEDIAPIPE_DISABLE=0 --define PYTHON_DISABLE=0 --//:distro=ubuntu" +ARG debug_bazel_flags="--strip=always --config=mp_on_py_on --//:distro=ubuntu" RUN apt-get update && apt-get install --no-install-recommends -y \ libgflags-dev \ @@ -156,10 +156,10 @@ RUN if [[ "$NVIDIA" == "1" ]] ; then true ; else exit 0 ; fi ; git clone https:/ # Build OpenVINO and nGraph (OV dependency) with D_GLIBCXX_USE_CXX11_ABI=0 or 1 # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/$ov_source_org/openvino.git /openvino && cd /openvino && git checkout $ov_source_branch && git submodule update --init --recursive -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; pip install --no-cache-dir -r /openvino/src/bindings/python/wheel/requirements-dev.txt +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; pip install --no-cache-dir -r /openvino/src/bindings/python/wheel/requirements-dev.txt WORKDIR /openvino/build -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DENABLE_PYTHON=ON -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " -DENABLE_CPPLINT=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. -RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " -DENABLE_CPPLINT=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DENABLE_PYTHON=ON -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " -DENABLE_CPPLINT=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"py_off=1"* ]]; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " -DENABLE_CPPLINT=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; make --jobs=$JOBS RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; make install RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; \ @@ -167,8 +167,8 @@ RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; \ ln -s /usr/local/runtime /opt/intel/openvino && \ ln -s /openvino/scripts/setupvars/setupvars.sh /opt/intel/openvino/setupvars.sh && \ ln -s /opt/intel/openvino /opt/intel/openvino_2024 -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; mkdir -p /opt/intel/openvino && cp -r /openvino/bin/intel64/${CMAKE_BUILD_TYPE}/python /opt/intel/openvino/ -RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; cp -r /openvino/tools/ovc/* /opt/intel/openvino/python +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; mkdir -p /opt/intel/openvino && cp -r /openvino/bin/intel64/${CMAKE_BUILD_TYPE}/python /opt/intel/openvino/ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; cp -r /openvino/tools/ovc/* /opt/intel/openvino/python ################## END OF OPENVINO SOURCE BUILD ###################### @@ -286,8 +286,8 @@ COPY *\.bzl /ovms/ COPY yarn.lock /ovms/ COPY package.json /ovms/ -# prebuild dependencies before copying sources -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies +# prebuild dependencies before copying sources & test dependencies for caching +RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest COPY src/ /ovms/src/ @@ -303,7 +303,7 @@ COPY demos/benchmark/cpp/synthetic_client_async_benchmark.cpp demos/image_classi WORKDIR /ovms/src/example/SampleCpuExtension/ RUN make -RUN if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; \ +RUN if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ mkdir -p /opt/intel/openvino/python/openvino-2024.5.dist-info && \ echo $'Metadata-Version: 1.0\nName: openvino\nVersion: 2024.5' > /opt/intel/openvino/python/openvino-2024.5.dist-info/METADATA @@ -337,7 +337,7 @@ RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_f # OVMS # hadolint ignore=DL3059 -RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms; fi; +RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms //src:ovms_test; fi; # hadolint ignore=DL3059 RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so; fi; @@ -357,7 +357,7 @@ FROM $BUILD_IMAGE as capi-build SHELL ["/bin/bash", "-xo", "pipefail", "-c"] # C api shared library # hadolint ignore=DL3059 -ARG CAPI_FLAGS="--strip=always --define MEDIAPIPE_DISABLE=1 --define PYTHON_DISABLE=1 --//:distro=ubuntu" +ARG CAPI_FLAGS="--strip=always --config mp_off_py_off --//:distro=ubuntu" ARG JOBS=40 ARG FUZZER_BUILD=0 RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared @@ -458,7 +458,7 @@ COPY --from=build /usr/local/lib/python3.*/dist-packages/jinja2-3.1.4.dist-info COPY --from=build /usr/local/lib/python3.*/dist-packages/markupsafe /ovms/python_deps/markupsafe # For image with Python enabled install Python library -RUN if ! [[ $debug_bazel_flags == *"PYTHON_DISABLE=1"* ]]; then true ; else exit 0 ; fi ; \ +RUN if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ apt-get update -y ; \ if [[ "$BASE_IMAGE" == *"22.04"* ]] ; then python_version=3.10; else python_version=3.8; fi; \ apt-get install libpython$python_version --no-install-recommends -y && \ diff --git a/Makefile b/Makefile index 05db29f32d..5e1c8510fe 100644 --- a/Makefile +++ b/Makefile @@ -99,10 +99,14 @@ else DISABLE_MEDIAPIPE_PARAMS = " --define MEDIAPIPE_DISABLE=0" endif -ifeq ($(PYTHON_DISABLE),1) - DISABLE_PYTHON_PARAMS = " --define PYTHON_DISABLE=1" +ifeq ($(MEDIAPIPE_DISABLE),1) + DISABLE_PARAMS = " --config=mp_off_py_off" else - DISABLE_PYTHON_PARAMS = " --define PYTHON_DISABLE=0" + ifeq ($(PYTHON_DISABLE),1) + DISABLE_PARAMS = " --config=mp_on_py_off" + else + DISABLE_PARAMS = " --config=mp_on_py_on" + endif endif FUZZER_BUILD_PARAMS ?= "" @@ -136,8 +140,8 @@ else ifeq ($(findstring redhat,$(BASE_OS)),redhat) else $(error BASE_OS must be either ubuntu or redhat) endif -CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --define MEDIAPIPE_DISABLE=1 --define PYTHON_DISABLE=1"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) -BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_MEDIAPIPE_PARAMS)$(DISABLE_PYTHON_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) +CAPI_FLAGS = "--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)" --config=mp_off_py_off"$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) +BAZEL_DEBUG_FLAGS="--strip=$(STRIP)"$(BAZEL_DEBUG_BUILD_FLAGS)$(DISABLE_PARAMS)$(FUZZER_BUILD_PARAMS)$(OV_TRACING_PARAMS)$(TARGET_DISTRO_PARAMS) # Option to Override release image. # Release image OS *must have* glibc version >= glibc version on BASE_OS: diff --git a/src/BUILD b/src/BUILD index 82033516d4..6983ea26e7 100644 --- a/src/BUILD +++ b/src/BUILD @@ -22,12 +22,14 @@ load("//:common_settings.bzl", COPTS_ADJUSTED = COMMON_STATIC_LIBS_COPTS + select({ "//conditions:default": [], "//:fuzzer_build" : COMMON_FUZZER_COPTS, -}) + select({ - "//conditions:default": ["-DPYTHON_DISABLE=1"], - "//:not_disable_python" : ["-DPYTHON_DISABLE=0"], -}) + select({ - "//conditions:default": ["-DMEDIAPIPE_DISABLE=1"], - "//:not_disable_mediapipe" : ["-DMEDIAPIPE_DISABLE=0"], +}) +COPTS_PYTHON = select({ + "//conditions:default": ["-DPYTHON_DISABLE=1"], + "//:not_disable_python" : ["-DPYTHON_DISABLE=0"], +}) +COPTS_MEDIAPIPE = select({ + "//conditions:default": ["-DMEDIAPIPE_DISABLE=1"], + "//:not_disable_mediapipe" : ["-DMEDIAPIPE_DISABLE=0"], }) LINKOPTS_ADJUSTED = COMMON_STATIC_LIBS_LINKOPTS + select({ @@ -146,7 +148,6 @@ cc_shared_library( load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") - mediapipe_proto_library( name = "ovmscalculatoroptions_proto", # ovmscalculatoroptions_cc_proto - just mediapipe stuff with mediapipe_proto_library adding nonvisible target srcs = ["mediapipe_calculators/ovmscalculator.proto",], @@ -399,7 +400,7 @@ cc_library( }), visibility = ["//visibility:public",], local_defines = COMMON_LOCAL_DEFINES, - copts = COPTS_ADJUSTED, + copts = COPTS_ADJUSTED + COPTS_PYTHON + COPTS_MEDIAPIPE, linkopts = LINKOPTS_ADJUSTED + ["-lOpenCL"], # TODO make as a direct dependency data = select({ "//:not_disable_python": [ @@ -769,7 +770,7 @@ cc_library( ], visibility = ["//visibility:public"], local_defines = COMMON_LOCAL_DEFINES, - copts = COPTS_ADJUSTED, + copts = COPTS_ADJUSTED + COPTS_MEDIAPIPE, linkopts = LINKOPTS_ADJUSTED, ) @@ -1466,24 +1467,6 @@ cc_test( "test/summator/1/summator.bin", "test/scalar/1/saved_model.xml", "test/scalar/1/saved_model.bin", - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - "-lssl", - ], - deps = [ - ":openvino_tests", - ":openvino_remote_tensors_tests", - ":inferencerequest_test", - ":libtest_environment", - ":libtest_gpuenvironment", - ":test_utils", - "//src:ovms_lib", - "//src:libovmsfilesystemfactory", - "//src:custom_nodes_common_lib", "//src:libsampleloader.so", "//src:lib_node_mock.so", "//src:lib_node_missing_implementation.so", @@ -1498,7 +1481,26 @@ cc_test( "//src:libcustom_node_image_transformation.so", "//src:libcustom_node_add_one.so", "//src:libcustom_node_horizontal_ocr.so", - "@com_google_googletest//:gtest", + + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + "-lssl", + ], + deps = [ + ":openvino_tests", + ":openvino_remote_tensors_tests", + ":inferencerequest_test", + ":libtest_environment", + ":libtest_gpuenvironment", + ":test_utils", + "//src:ovms_lib", + "//src:libovmsfilesystemfactory", + "//src:custom_nodes_common_lib", + "@com_google_googletest//:gtest", ] + select({ "//conditions:default": [ "//src/test/mediapipe/calculators:mediapipe_test_calculators", From 8d3bb29cc721cab89e94645991fda146f1c6f522 Mon Sep 17 00:00:00 2001 From: atobisze Date: Wed, 23 Oct 2024 10:27:05 +0200 Subject: [PATCH 2/2] Remove copies --- WORKSPACE | 4 +- src/BUILD | 78 +++++++++++++++++++++---- src/embeddings/BUILD | 1 + src/embeddings/embeddings_calculator.cc | 62 ++++++++++++++++---- src/json_parser.cpp | 2 +- src/json_parser.hpp | 1 - src/model.cpp | 1 + src/modelconfig.cpp | 1 - src/modelconfig.hpp | 2 +- src/schema.cpp | 1 + src/schema.hpp | 3 +- src/serialization.hpp | 4 +- src/test/schema_test.cpp | 1 + 13 files changed, 129 insertions(+), 32 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index face6700ca..897b32f2a2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -124,7 +124,7 @@ http_archive( git_repository( name = "mediapipe", remote = "https://github.com/openvinotoolkit/mediapipe", - commit = "9407697e8a18eebea664bab27d217a06bfa237fd", # Support ov::string in ovms model api adapter (#90) + commit = "5d060825975114cecfb04d53ed60c1d68b636a73", # Support getting output metadata in model api adapter ) # DEV mediapipe 1 source - adjust local repository path for build @@ -419,7 +419,7 @@ new_git_repository( name = "model_api", remote = "https:///github.com/openvinotoolkit/model_api/", build_file = "@_model-api//:BUILD", - commit = "eb9fcfb1e1eebc047ff144707f76203b132e1aa6" # master Jun 24 15:02:17 2024 [cpp] Fix num classes check + commit = "28eed6279d5ebc20f41f76daea9be66681704968" # Support getting output metadata in model api adapter ) new_local_repository( diff --git a/src/BUILD b/src/BUILD index 6983ea26e7..b3a0c25912 100644 --- a/src/BUILD +++ b/src/BUILD @@ -250,10 +250,6 @@ cc_library( "get_model_metadata_impl.hpp", "global_sequences_viewer.hpp", "global_sequences_viewer.cpp", - "layout.cpp", - "layout.hpp", - "layout_configuration.cpp", - "layout_configuration.hpp", "grpc_utils.cpp", "grpc_utils.hpp", "grpcservermodule.cpp", @@ -272,8 +268,6 @@ cc_library( "model.hpp", "modelchangesubscription.cpp", "modelchangesubscription.hpp", - "modelconfig.cpp", - "modelconfig.hpp", "modelmanager.cpp", "modelmanager.hpp", "modelinstance.cpp", @@ -299,8 +293,6 @@ cc_library( "rest_parser.hpp", "rest_utils.cpp", "rest_utils.hpp", - "schema.hpp", - "schema.cpp", "serialization.cpp", "serialization.hpp", "servablemanagermodule.cpp", @@ -321,7 +313,6 @@ cc_library( "tensorinfo.hpp", "tensor_utils.hpp", "threadsafequeue.hpp", - "timer.hpp", "version.hpp", "tensor_conversion.hpp", "tensor_conversion.cpp", @@ -356,6 +347,7 @@ cc_library( }) + [ "cpp_headers", "libmodelconfigjsonparser", + "libovmsmodelconfig", "libovmscapibuffer", "libovmscapi_utils", "libovmscapiinferenceparameter", @@ -371,6 +363,10 @@ cc_library( "libovmsmetrics", "libovmsprecision", "libovmsshape", + "libovmsschema", + "libovmslayout", + "libovmslayout_configuration", + "libovmstimer", "libovmstfs_grpc", "//src:libovmsprofiler", "libovmsstatus", @@ -1029,6 +1025,26 @@ cc_library( copts = COPTS_ADJUSTED, linkopts = LINKOPTS_ADJUSTED, ) +cc_library( + name = "libovmsmodelconfig", + hdrs = ["modelconfig.hpp",], + srcs = ["modelconfig.cpp",], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", # TODO split into parser + "libovmsfilesystem", + "libovmslayout_configuration", + "libovmsmodelversioning", + "libovmsschema", + "libovmsshape", + "libovmsstatus", + "libovmsstring_utils", + "libmodelconfigjsonparser", + ], + visibility = ["//visibility:public"], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, +) cc_library( name = "libovmsshape", hdrs = ["shape.hpp",], @@ -1045,6 +1061,48 @@ cc_library( copts = COPTS_ADJUSTED, linkopts = LINKOPTS_ADJUSTED, ) +cc_library( + name = "libovmstimer", + hdrs = ["timer.hpp",], + deps = [ + ], + visibility = ["//visibility:public"], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, +) +cc_library( + name = "libovmslayout", + hdrs = ["layout.hpp",], + srcs = ["layout.cpp",], + deps = [ + "libovmsshape", + "@linux_openvino//:openvino", + "ovms_header", + "libovmslogging", + "libovmsstatus", + ], + visibility = ["//visibility:public"], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, +) +cc_library( + name = "libovmslayout_configuration", + hdrs = ["layout_configuration.hpp",], + srcs = ["layout_configuration.cpp",], + deps = [ + "libovmslayout", + "@linux_openvino//:openvino", + "ovms_header", + "libovmsstatus", + "libovmsstring_utils", + ], + visibility = ["//visibility:public"], + local_defines = COMMON_LOCAL_DEFINES, + copts = COPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED, +) cc_library( name = "libovmsmodelversion", hdrs = [ @@ -1186,7 +1244,7 @@ cc_library( deps = ["cpp_headers", "@com_github_tencent_rapidjson//:rapidjson", "@linux_openvino//:openvino", - "@com_github_gabime_spdlog//:spdlog"], + "libovmslogging"], visibility = ["//visibility:public"], local_defines = COMMON_LOCAL_DEFINES, copts = COPTS_ADJUSTED, diff --git a/src/embeddings/BUILD b/src/embeddings/BUILD index ef5b656afd..5ea9fbe54b 100644 --- a/src/embeddings/BUILD +++ b/src/embeddings/BUILD @@ -48,6 +48,7 @@ cc_library( "//src:httppayload", "//src:libovmslogging", "//src:libovmsprofiler", + "//src:libovmstimer", "embeddings_calculator_cc_proto", ], visibility = ["//visibility:public"], diff --git a/src/embeddings/embeddings_calculator.cc b/src/embeddings/embeddings_calculator.cc index c2599fe716..a6e2fbc2b3 100644 --- a/src/embeddings/embeddings_calculator.cc +++ b/src/embeddings/embeddings_calculator.cc @@ -131,6 +131,7 @@ class EmbeddingsCalculator : public CalculatorBase { // Automatically deduce tokenizer input name std::vector tokenizerInputNames = tokenizer_session->getInputNames(); std::vector embeddingsInputNames = embeddings_session->getInputNames(); + std::vector embeddingsOutputNames = embeddings_session->getOutputNames(); RET_CHECK(tokenizerInputNames.size() == 1); const std::string& tokenizerInputName = tokenizerInputNames.at(0); SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Tokenizer input name detected: {}", tokenizerInputName); @@ -141,20 +142,41 @@ class EmbeddingsCalculator : public CalculatorBase { ov::Shape{input_strings.size()}, input_strings.data()}; - ::InferenceOutput embeddingsOutputMap; + ; + std::unique_ptr<::InferenceOutput> embeddingsOutputMap; try { ::InferenceOutput tokenizerOutputMap = tokenizer_session->infer(tokenizerInputMap); ::InferenceInput embeddingsInputMap; // Check if tokenizer produced at least the number of outputs as there are inputs in embedding model RET_CHECK(tokenizerOutputMap.size() >= embeddingsInputNames.size()); + ov::Shape outShape; + bool first = true; for (const auto& embeddingsInputName : embeddingsInputNames) { auto it = tokenizerOutputMap.find(embeddingsInputName); RET_CHECK(it != tokenizerOutputMap.end()); SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Embedding model input {} is connected with matching tokenizer output", embeddingsInputName); embeddingsInputMap[embeddingsInputName] = it->second; + for (auto& d : embeddingsInputMap[embeddingsInputName].get_shape()) { + if (first) + outShape.emplace_back(d); + } + first = false; + } + // XYZ here extract output with pbtxt specified name then use its last dimension and first two dimension of specific input + std::string outputName{"token_embeddings"}; // choose 3d output + ov::PartialShape outPShape = embeddings_session->getOutputShape(outputName); + try { + outShape.emplace_back(outPShape[2].get_length()); + } catch (std::exception& e) { + LOG(ERROR) << "Failed to get 2nd dimension of output" << outputName; + RET_CHECK(false); } - embeddingsOutputMap = embeddings_session->infer(embeddingsInputMap); - RET_CHECK(embeddingsOutputMap.size() > 0); + + // creating output + ov::Tensor outputTensor(ov::element::f32, outShape); + embeddingsInputMap.emplace(std::string("token_embeddings"), std::move(outputTensor)); + embeddingsOutputMap = std::make_unique(std::move(embeddings_session->infer(embeddingsInputMap))); + RET_CHECK(embeddingsOutputMap->size() > 0); } catch (const std::exception& e) { LOG(INFO) << "Caught exception from session infer():" << e.what(); RET_CHECK(false); @@ -163,13 +185,13 @@ class EmbeddingsCalculator : public CalculatorBase { RET_CHECK(false); } - ov::Tensor embeddingsTensor; - if (embeddingsOutputMap.size() == 2) { // GTE + const ov::Tensor* embeddingsTensorPtr{nullptr}; + if (embeddingsOutputMap->size() == 2) { // GTE // Search by number of dimensions, should be 3 bool found = false; - for (const auto& [name, tensor] : embeddingsOutputMap) { + for (const auto& [name, tensor] : *embeddingsOutputMap) { if (tensor.get_shape().size() == 3) { - embeddingsTensor = tensor; + embeddingsTensorPtr = &tensor; SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Multiple embedding model outputs found, 3-dim output with name {} will be used", name); found = true; break; @@ -177,10 +199,11 @@ class EmbeddingsCalculator : public CalculatorBase { } RET_CHECK(found); } else { // BGE - RET_CHECK(embeddingsOutputMap.size() == 1); - embeddingsTensor = embeddingsOutputMap.begin()->second; - SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Single embedding model output found with name {}", embeddingsOutputMap.begin()->first); + RET_CHECK(embeddingsOutputMap->size() == 1); + embeddingsTensorPtr = &embeddingsOutputMap->begin()->second; + SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Single embedding model output found with name {}", embeddingsOutputMap->begin()->first); } + const ov::Tensor& embeddingsTensor = *embeddingsTensorPtr; RET_CHECK(embeddingsTensor.get_shape().size() == 3); RET_CHECK(embeddingsTensor.get_shape()[0] == input_strings.size()); @@ -199,23 +222,39 @@ class EmbeddingsCalculator : public CalculatorBase { bool normalize = options.normalize_embeddings(); // TODO: mean pooling - ov::Shape outputShape = embeddingsTensor.get_shape(); + const ov::Shape& outputShape = embeddingsTensor.get_shape(); size_t batchSize = outputShape[0]; for (size_t i = 0; i < batchSize; i++) { size_t stride = i * outputShape[1] * outputShape[2]; std::vector data(reinterpret_cast(embeddingsTensor.data()) + stride, reinterpret_cast(embeddingsTensor.data()) + stride + outputShape[2]); + float* dataPtr = reinterpret_cast(embeddingsTensor.data()) + stride; + float* dataPtrEnd = dataPtr + outputShape[2]; writer.StartObject(); writer.String("object"); writer.String("embedding"); writer.String("embedding"); if (normalize) { double square_sum = std::inner_product(data.begin(), data.end(), data.begin(), double(0.0)); + double square_sum2 = std::inner_product(dataPtr, dataPtrEnd, dataPtr, double(0.0)); + if (square_sum != square_sum2) { + RET_CHECK(false); + } double denom = std::max(std::sqrt(square_sum), double(1e-12)); + double denom2 = std::max(std::sqrt(square_sum2), double(1e-12)); std::transform(data.begin(), data.end(), data.begin(), [denom](auto& element) { return element / denom; }); + std::transform(dataPtr, dataPtrEnd, dataPtr, + [denom2](auto& element) { return element / denom2; }); + if (denom != denom2) { + RET_CHECK(false); + } } if (isBase64) { std::string_view sv(reinterpret_cast(data.data()), data.size() * sizeof(float)); + std::string_view sv2(reinterpret_cast(dataPtr), outputShape[2] * sizeof(float)); + if (sv.size() != sv2.size()) { + RET_CHECK(false); + } std::string escaped; absl::Base64Escape(sv, &escaped); writer.String(escaped.c_str()); @@ -230,7 +269,6 @@ class EmbeddingsCalculator : public CalculatorBase { writer.Int(i); writer.EndObject(); } - writer.EndArray(); writer.EndObject(); cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string(buffer.GetString()), timestamp); diff --git a/src/json_parser.cpp b/src/json_parser.cpp index 44b2c4f21d..7b4b97ee3d 100644 --- a/src/json_parser.cpp +++ b/src/json_parser.cpp @@ -20,8 +20,8 @@ #include #include -#include +#include "logging.hpp" #include "status.hpp" namespace ovms { diff --git a/src/json_parser.hpp b/src/json_parser.hpp index 08937652e7..7b87cfe99c 100644 --- a/src/json_parser.hpp +++ b/src/json_parser.hpp @@ -47,5 +47,4 @@ class JsonParser { */ static Status parsePluginConfig(std::string command, plugin_config_t& pluginConfig); }; - } // namespace ovms diff --git a/src/model.cpp b/src/model.cpp index e539c3bb69..d57964d401 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -29,6 +29,7 @@ #include "logging.hpp" #include "modelinstance.hpp" #include "statefulmodelinstance.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index 87b8fafe07..a840c2437b 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include "filesystem.hpp" #include "json_parser.hpp" diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index f4c80b2e6c..6232c15e2f 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -30,7 +30,7 @@ #include "layout_configuration.hpp" #include "modelversion.hpp" #include "shape.hpp" -#include "status.hpp" +#include "status.hpp" // TODO fwd dec namespace ovms { class ModelVersionPolicy; diff --git a/src/schema.cpp b/src/schema.cpp index 06aed6a0b7..b1590d9ce8 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -25,6 +25,7 @@ #include #include "logging.hpp" +#include "status.hpp" namespace ovms { const std::string DAG_DEFINITIONS = R"( diff --git a/src/schema.hpp b/src/schema.hpp index d5f386e5dc..c117071f80 100644 --- a/src/schema.hpp +++ b/src/schema.hpp @@ -19,9 +19,8 @@ #include -#include "status.hpp" - namespace ovms { +class Status; extern const std::string MODELS_CONFIG_SCHEMA; extern const std::string MODEL_CONFIG_DEFINITION2; extern const char* MODELS_MAPPING_SCHEMA; diff --git a/src/serialization.hpp b/src/serialization.hpp index 6bc3cf7682..22b68e87db 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -383,8 +383,8 @@ Status serializePredictResponse( bufferType, std::nullopt, // TODO TBD copyBuffer); - SPDLOG_TRACE("Serialized output with name:{} for servable name:{}; version:{} with no buffer copy", - outputName, response->getServableName(), response->getServableVersion()); + SPDLOG_TRACE("Serialized output with name:{}; for servable name:{}; version:{}; with buffer copy:{}", + outputName, response->getServableName(), response->getServableVersion(), copyBuffer); ++outputId; } return StatusCode::OK; diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index 3c303c3960..aab14c2aa7 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -17,6 +17,7 @@ #include #include "../schema.hpp" +#include "../status.hpp" TEST(SchemaTest, PipelineConfigMatchingSchema) { const char* pipelineConfigMatchingSchema = R"(