diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 565aeb3cbecd8..6e83e2b52fdbe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -42,7 +42,6 @@ common/tier4_traffic_light_rviz_plugin/** satoshi.ota@tier4.jp common/tier4_vehicle_rviz_plugin/** yukihiro.saito@tier4.jp common/traffic_light_recognition_marker_publisher/** shumpei.wakabayashi@tier4.jp takeshi.miura@tier4.jp tomoya.kimura@tier4.jp common/traffic_light_utils/** kotaro.uetake@tier4.jp satoshi.ota@tier4.jp shunsuke.miura@tier4.jp -common/tvm_utility/** ambroise.vincent@arm.com xinyu.wang@tier4.jp control/autoware_autonomous_emergency_braking/** daniel.sanchez@tier4.jp kyoichi.sugahara@tier4.jp mamoru.sobue@tier4.jp takamasa.horibe@tier4.jp tomoya.kimura@tier4.jp control/autoware_collision_detector/** go.sakayori@tier4.jp kyoichi.sugahara@tier4.jp tomohito.ando@tier4.jp control/autoware_control_validator/** kyoichi.sugahara@tier4.jp makoto.kurihara@tier4.jp mamoru.sobue@tier4.jp takamasa.horibe@tier4.jp takayuki.murooka@tier4.jp diff --git a/common/.pages b/common/.pages index e0425488ad59b..bba56f6dab486 100644 --- a/common/.pages +++ b/common/.pages @@ -25,9 +25,6 @@ nav: - 'Butterworth Filter': common/signal_processing/documentation/ButterworthFilter - 'autoware_universe_utils': common/autoware_universe_utils - 'traffic_light_utils': common/traffic_light_utils - - 'TVM Utility': - - 'Introduction': common/tvm_utility - - 'YOLOv2 Tiny Example': common/tvm_utility/tvm-utility-yolo-v2-tiny-tests - 'RVIZ2 Plugins': - 'autoware_perception_rviz_plugin': common/autoware_perception_rviz_plugin - 'autoware_overlay_rviz_plugin': common/autoware_overlay_rviz_plugin/autoware_overlay_rviz_plugin diff --git a/common/tvm_utility/.gitignore b/common/tvm_utility/.gitignore deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/common/tvm_utility/CHANGELOG.rst b/common/tvm_utility/CHANGELOG.rst deleted file mode 100644 index f0838352ec660..0000000000000 --- a/common/tvm_utility/CHANGELOG.rst +++ /dev/null @@ -1,146 +0,0 @@ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Changelog for package tvm_utility -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -0.38.0 (2024-11-08) -------------------- -* unify package.xml version to 0.37.0 -* fix(tvm_utility): add virtual destructor to tvm_utility (`#7759 `_) - * fix(tvm_utility): add virtual destructor to tvm_utility - * add override - --------- -* fix(tvm_utility): fix selfAssignment warnings (`#7561 `_) -* fix(tvm_utility): fix warning of negativeContainerIndex (`#6924 `_) -* Contributors: Ryuta Kambe, Yukinari Hisaki, Yutaka Kondo - -0.26.0 (2024-04-03) -------------------- -* build(tvm_utility): remove download logic from CMake and update documentation (`#4923 `_) - * add include tier4_autoware_utils and dependency - * remove downloading logic from Cmake, update documentation - * build(tvm_utility): remove downloading logic from Cmake, update documentation - * style(pre-commit): autofix - * build(tvm_utility): fix lint_cmake error - * build(tvm_utility): format warning message - * build(tvm_utility): add logic to work with autoware_data folder, add nn config header and test image - * style(pre-commit): autofix - * style(pre-commit): autofix - * build(tvm_utility): refactor, update InferenceEngineTVM constructor - * style(pre-commit): autofix - * build(tvm_utility): add lightweight model and test with it - * build(tvm_utility): make building yolo_v2_tiny disable by default - * build(tvm_utility): remove test artifact for yolo_v2_tiny - * build(tvm_utility): update docs - * build(tvm_utility): update docs - * style(pre-commit): autofix - * build(tvm_utility): update namespace in abs_model test - * build(tvm_utility): rewrite yolo_v2_tiny as example - * build(tvm_utility): clean comments in yolo_v2_tiny example main.cpp - * build(tvm_utility): add launch file for yolo_v2_tiny example - * build(tvm_utility): update yolo_v2_tiny example readme - * style(pre-commit): autofix - * build(tvm_utility): add model for arm based systems, need to be tested on device - * style(pre-commit): autofix - * style(pre-commit): autofix - * build(tvm_utility): update config header for arm - * style(pre-commit): autofix - * build(tvm_utility): remove debug output - * build(tvm_utility): add find_package conditional section - * build(tvm_utility): fix lint_cmake errors - * build(tvm_utility): remove coping model files during build - * build(tvm_utility): update readme with new data folder structure - * build(tvm_utility): fix spell check warnings - * style(pre-commit): autofix - * build(tvm_utility): add no model files guard to get_neural_network - * style(pre-commit): autofix - * build(tvm_utility): set back default paths in config headers - * build(tvm_utility): add param file, update launch file - * build(tvm_utility): add schema file, update node name - * style(pre-commit): autofix - * build(tvm_utility): fix json-schema-check - * build(tvm_utility): fix json-schema-check - * style(pre-commit): autofix - * build(tvm_utility): add parameter table to example readme - * build(tvm_utility): fix typo-error in description of schema.json - * style(pre-commit): autofix - * buiild(tvm_utility): fix spell-check warning and typo - --------- - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> -* refactor: fix spell-check (`#4289 `_) - * refactor: fix spell-check - * fix spell-check - * fix typo - * Fix obvious typo, shortened words - * Fix obvious typo, shortened words, in common directory - * add cspell ignore - * Update perception/tensorrt_classifier/CMakeLists.txt - Co-authored-by: Yusuke Muramatsu - --------- - Co-authored-by: Yusuke Muramatsu -* build: mark autoware_cmake as (`#3616 `_) - * build: mark autoware_cmake as - with , autoware_cmake is automatically exported with ament_target_dependencies() (unecessary) - * style(pre-commit): autofix - * chore: fix pre-commit errors - --------- - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> - Co-authored-by: Kenji Miyake -* refactor(lidar_apollo_segmentation_tvm/_nodes): remove autoware_auto_common dependency (`#3136 `_) -* docs: comply with mkdocs style (`#3378 `_) - refactor(doc): comply with mkdocs style - Change from the Autoware.Auto style of package documentation to one that - gets parsed better by the current way of generating the documentation. - Issue-Id: SCM-5887 - Change-Id: I898d00eda921ac73373067036c582a7bbc192cce - Co-authored-by: Xinyu Wang <93699235+angry-crab@users.noreply.github.com> -* feat(tvm_utility, lidar_centerpoint_tvm): tvm pipeline extension (`#2468 `_) - * add scatter - * two stage pipeline - * ci(pre-commit): autofix - * fix build - * ci(pre-commit): autofix - * remove unecessary code - * apply changes from network_node refactor - * ci(pre-commit): autofix - * unpack tar file first and add maintainer - * fix pre commit error - * remove comment - * fix cmake download, remove scatter cpp and rename tvm function - * ci(pre-commit): autofix - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> -* refactor(tvm_utility): redefine network node and inference config (`#2467 `_) - * redefine network node and inference config - * accomondate network node changes - * ci(pre-commit): autofix - * revert device type and id - * cmake setting - * data type and modelzoo version - Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> -* fix(tvm_utility): copy test result to CPU (`#2414 `_) - Also remove dependency to autoware_auto_common. - Issue-Id: SCM-5401 - Change-Id: I83b859742df2f2ff7df1d0bd2d287bfe0aa04c3d - Co-authored-by: Xinyu Wang <93699235+angry-crab@users.noreply.github.com> -* fix(lidar_apollo_segmentation_tvm, tvm_utility): fixed output dim and pipeline (`#2185 `_) -* feat(ModelZoo): model descriptors versioning (`#2158 `_) - Introduce a version field to the interface. - Make use of it in the lidar_apollo_segmentation_tvm packages and the - yolo_v2 example of tvm_utility. - Issue-Id: SCM-4000 - Change-Id: I6d666f886b0a9f01128bfa4cf30e189d23f3481e - Co-authored-by: Xinyu Wang <93699235+angry-crab@users.noreply.github.com> -* feat(ModelZoo): rework prebuilt assets management (`#1880 `_) - ModelZoo artifacts changed from a single archive to one archive per - model/backend combination. It allows users to only download needed - archives, but prevents keeping the current design. - Change from all models being handled by the "neural_networks_provider" - package to models being downloaded by packages that need them. - Leverage the newly added versioning of the prebuilt models. - Fix the NN check of the "nodes" package. - Issue-Id: SCM-3999 - Change-Id: I1df9007f5bf446a8b50e38c4fd98e9e3a8d2550f - Co-authored-by: Xinyu Wang <93699235+angry-crab@users.noreply.github.com> -* docs(tvm_utility): fix broken link (`#1670 `_) -* feat(tvm_utility): port tvm_utility (`#893 `_) - Co-authored-by: Luca Foschiani -* Contributors: Alexey Panferov, Ambroise Vincent, Kenji Miyake, Luca Foschiani, M. Fatih Cırıt, Shunsuke Miura, Vincent Richard, Xinyu Wang diff --git a/common/tvm_utility/CMakeLists.txt b/common/tvm_utility/CMakeLists.txt deleted file mode 100644 index c0a0d7385f615..0000000000000 --- a/common/tvm_utility/CMakeLists.txt +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2021-2022 Arm Limited and Contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 3.14) -project(tvm_utility) - -include("${PROJECT_NAME}-extras.cmake") - -find_package(autoware_cmake REQUIRED) -autoware_package() - -# Library - -set(TVM_UTILITY_NODE_LIB_HEADERS - "include/${PROJECT_NAME}/pipeline.hpp" -) - -ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS}) -set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) - -set(BUILD_EXAMPLE OFF CACHE BOOL "enable build yolo_v2_tiny") - -if(BUILD_TESTING OR BUILD_EXAMPLE) - find_package(OpenCV REQUIRED) - set(tvm_runtime_DIR ${tvm_vendor_DIR}) - find_package(tvm_runtime CONFIG REQUIRED) - # Get target backend - set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend") -endif() - -if(BUILD_TESTING) - # compile each folder inside test/ as a test case - find_package(ament_cmake_gtest REQUIRED) - - set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts") - file(GLOB TEST_CASES test/*) - foreach(TEST_FOLDER ${TEST_CASES}) - if(NOT IS_DIRECTORY ${TEST_FOLDER}) - continue() - endif() - # the folder name becomes the test case name - file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER}) - # Get neural network. - set(NN_DEPENDENCY "") - get_neural_network(${TEST_CASE_NAME}_${CMAKE_SYSTEM_PROCESSOR} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY) - - if(NOT NN_DEPENDENCY STREQUAL "") - # add all cpp files in the folder to the target - file(GLOB TEST_CASE_SOURCES ${TEST_FOLDER}/*.cpp) - ament_add_gtest(${TEST_CASE_NAME} ${TEST_CASE_SOURCES}) - ament_target_dependencies(${TEST_CASE_NAME} - "ament_index_cpp" - "tvm_vendor" - ) - add_dependencies(${TEST_CASE_NAME} ${NN_DEPENDENCY}) - - target_link_libraries("${TEST_CASE_NAME}" - "${OpenCV_LIBRARIES}" - "${tvm_runtime_LIBRARIES}" - ) - - target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC - "${OpenCV_INCLUDE_DIRS}" - "${tvm_utility_FOUND_INCLUDE_DIRS}" - "data/models/${TEST_CASE_NAME}_${CMAKE_SYSTEM_PROCESSOR}" - "include" - ) - - # Install test-specific files - if(IS_DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}) - install(DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}/ - DESTINATION ${CMAKE_BINARY_DIR}/${TEST_CASE_NAME}_artifacts - ) - endif() - - else() - message(WARNING "No model is generated for ${TEST_FOLDER}, skipping test") - endif() - - endforeach() -endif() - -if(BUILD_EXAMPLE) - # compile each folder inside example/ as an example - find_package(rclcpp REQUIRED) - - set(EXAMPLE_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts") - file(GLOB EXAMPLE_CASES example/*) - foreach(EXAMPLE_FOLDER ${EXAMPLE_CASES}) - if(NOT IS_DIRECTORY ${EXAMPLE_FOLDER}) - continue() - endif() - # the folder name becomes the example name - file(RELATIVE_PATH EXAMPLE_NAME ${CMAKE_CURRENT_LIST_DIR}/example ${EXAMPLE_FOLDER}) - # Get neural network. - set(NN_DEPENDENCY "") - get_neural_network(${EXAMPLE_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY) - - if(NOT NN_DEPENDENCY STREQUAL "") - if(EXAMPLE_NAME STREQUAL "yolo_v2_tiny" AND - NOT EXISTS ${EXAMPLE_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg) - message(WARNING "Missing image artifact for yolo_v2_tiny, skipping example") - continue() - endif() - # add all cpp files in the folder to the target - file(GLOB EXAMPLE_SOURCES ${EXAMPLE_FOLDER}/*.cpp) - ament_auto_add_executable(${EXAMPLE_NAME} ${EXAMPLE_SOURCES}) - ament_target_dependencies(${EXAMPLE_NAME} - "ament_index_cpp" - "tvm_vendor" - "rclcpp" - ) - add_dependencies(${EXAMPLE_NAME} ${NN_DEPENDENCY}) - - target_link_libraries("${EXAMPLE_NAME}" - "${OpenCV_LIBRARIES}" - "${tvm_runtime_LIBRARIES}" - ) - - target_include_directories("${EXAMPLE_NAME}" SYSTEM PUBLIC - "${OpenCV_INCLUDE_DIRS}" - "${tvm_utility_FOUND_INCLUDE_DIRS}" - "data/models" - "include" - ) - - else() - message(WARNING "No model is generated for ${EXAMPLE_FOLDER} example") - endif() - - endforeach() -endif() - -list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake") -# ament_auto_package() -ament_auto_package( - INSTALL_TO_SHARE - launch - config - artifacts) diff --git a/common/tvm_utility/README.md b/common/tvm_utility/README.md deleted file mode 100644 index 4751428353886..0000000000000 --- a/common/tvm_utility/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# TVM Utility - -This is the design document for the `tvm_utility` package. For instructions on how to build the tests for YOLOv2 Tiny, -see the [YOLOv2 Tiny Example Pipeline](tvm-utility-yolo-v2-tiny-tests.md). -For information about where to store test artifacts see the [TVM Utility Artifacts](artifacts/README.md). - -## Purpose / Use cases - -A set of c++ utilities to help build a TVM based machine learning inference pipeline. The library contains a pipeline -class which helps building the pipeline and a number of utility functions that are common in machine learning. - -## Design - -The Pipeline Class is a standardized way to write an inference pipeline. The pipeline class contains 3 different stages: -the pre-processor, the inference engine and the post-processor. The TVM implementation of an inference engine stage is -provided. - -### API - -The pre-processor and post-processor need to be implemented by the user before instantiating the pipeline. You can see example -usage in the example pipeline at `test/yolo_v2_tiny`. - -Each stage in the pipeline has a `schedule` function which takes input data as a parameter and return the output data. -Once the pipeline object is created, `pipeline.schedule` is called to run the pipeline. - -```{cpp} -int main() { - create_subscription("points_raw", - rclcpp::QoS{1}, [this](const sensor_msgs::msg::PointCloud2::SharedPtr msg) - {pipeline.schedule(msg);}); -} -``` - -#### Version checking - -The `InferenceEngineTVM::version_check` function can be used to check the version of the neural network in use against the range of earliest to latest supported versions. - -The `InferenceEngineTVM` class holds the latest supported version, which needs to be updated when the targeted version changes; after having tested the effect of the version change on the packages dependent on this one. - -The earliest supported version depends on each package making use of the inference, and so should be defined (and maintained) in those packages. - -#### Models - -Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to build proper external dependency. - -### Error detection and handling - -`std::runtime_error` should be thrown whenever an error is encountered. It should be populated with an appropriate text -error description. - -### Neural Networks Provider - -The neural networks are compiled as part of the -[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket. - -The `get_neural_network` function creates an abstraction for the artifact management. -Users should check if model configuration header file is under "data/user/${MODEL_NAME}/". Otherwise, nothing happens and compilation of the package will be skipped. - -The structure inside of the source directory of the package making use of the function is as follow: - -```{text} -. -├── data -│ └── models -│ ├── ${MODEL 1} -│ │ └── inference_engine_tvm_config.hpp -│ ├── ... -│ └── ${MODEL ...} -│ └── ... -``` - -The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages should be available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp". -Dependent packages can use the cmake `add_dependencies` function with the name provided in the `DEPENDENCY` output parameter of `get_neural_network` to ensure this file is created before it gets used. - -The other `deploy_*` files are installed to "models/${MODEL_NAME}/" under the `share` directory of the package. - -The other model files should be stored in autoware_data folder under package folder with the structure: - -```{text} -$HOME/autoware_data -| └──${package} -| └──models -| ├── ${MODEL 1} -| | ├── deploy_graph.json -| | ├── deploy_lib.so -| | └── deploy_param.params -| ├── ... -| └── ${MODEL ...} -| └── ... -``` - -#### Inputs / Outputs - -Outputs: - -- `get_neural_network` cmake function; create proper external dependency for a package with use of the model provided by the user. - -In/Out: - -- The `DEPENDENCY` argument of `get_neural_network` can be checked for the outcome of the function. - It is an empty string when the neural network wasn't provided by the user. - -## Security considerations - -### Pipeline - -Both the input and output are controlled by the same actor, so the following security concerns are out-of-scope: - -- Spoofing -- Tampering - -Leaking data to another actor would require a flaw in TVM or the host operating system that allows arbitrary memory to -be read, a significant security flaw in itself. This is also true for an external actor operating the pipeline early: -only the object that initiated the pipeline can run the methods to receive its output. - -A Denial-of-Service attack could make the target hardware unusable for other pipelines but would require being able to -run code on the CPU, which would already allow a more severe Denial-of-Service attack. - -No elevation of privilege is required for this package. - -### Network provider - -The pre-compiled networks are downloaded from an S3 bucket and are under threat of spoofing, -tampering and denial of service. -Spoofing is mitigated by using an https connection. -Mitigations for tampering and denial of service are left to AWS. - -The user-provided networks are installed as they are on the host system. -The user is in charge of securing the files they provide with regard to information disclosure. - -## Future extensions / Unimplemented parts - -Future packages will use tvm_utility as part of the perception stack to run machine learning workloads. - -## Related issues - - diff --git a/common/tvm_utility/artifacts/README.md b/common/tvm_utility/artifacts/README.md deleted file mode 100644 index a2c6affd9a0e7..0000000000000 --- a/common/tvm_utility/artifacts/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# TVM Utility Artifacts - -Place any test artifacts in subdirectories within this directory. - -e.g.: -./artifacts/yolo_v2_tiny diff --git a/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv b/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv deleted file mode 100644 index 1a3666f7e3483..0000000000000 --- a/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv +++ /dev/null @@ -1,5 +0,0 @@ -0.57273, 0.677385f -1.87446, 2.06253f -3.33843, 5.47434f -7.88282, 3.52778f -9.77052, 9.16828f diff --git a/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt b/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt deleted file mode 100644 index 941cb4e139226..0000000000000 --- a/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt +++ /dev/null @@ -1,80 +0,0 @@ -person -bicycle -car -motorcycle -airplane -bus -train -truck -boat -traffic light -fire hydrant -stop sign -parking meter -bench -bird -cat -dog -horse -sheep -cow -elephant -bear -zebra -giraffe -backpack -umbrella -handbag -tie -suitcase -frisbee -skis -snowboard -sports ball -kite -baseball bat -baseball glove -skateboard -surfboard -tennis racket -bottle -wine glass -cup -fork -knife -spoon -bowl -banana -apple -sandwich -orange -broccoli -carrot -hot dog -pizza -donut -cake -chair -couch -potted plant -bed -dining table -toilet -tv -laptop -mouse -remote -keyboard -cell phone -microwave -oven -toaster -sink -refrigerator -book -clock -vase -scissors -teddy bear -hair drier -toothbrush diff --git a/common/tvm_utility/config/yolo_v2_tiny_example.param.yaml b/common/tvm_utility/config/yolo_v2_tiny_example.param.yaml deleted file mode 100644 index b63e4a99f97f2..0000000000000 --- a/common/tvm_utility/config/yolo_v2_tiny_example.param.yaml +++ /dev/null @@ -1,6 +0,0 @@ -/**: - ros__parameters: - image_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/test_image_0.jpg - label_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/labels.txt - anchor_filename: $(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/anchors.csv - data_path: $(env HOME)/autoware_data diff --git a/common/tvm_utility/data/models/abs_model_aarch64/deploy_graph.json b/common/tvm_utility/data/models/abs_model_aarch64/deploy_graph.json deleted file mode 100644 index b226c01747dca..0000000000000 --- a/common/tvm_utility/data/models/abs_model_aarch64/deploy_graph.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "nodes": [ - { - "op": "null", - "name": "a", - "inputs": [] - }, - { - "op": "tvm_op", - "name": "tvmgen_default_fused_abs", - "attrs": { - "num_outputs": "1", - "num_inputs": "1", - "flatten_data": "0", - "func_name": "tvmgen_default_fused_abs", - "hash": "1be44995aa501758" - }, - "inputs": [[0, 0, 0]] - } - ], - "arg_nodes": [0], - "heads": [[1, 0, 0]], - "attrs": { - "dltype": ["list_str", ["float32", "float32"]], - "device_index": ["list_int", [1, 1]], - "storage_id": ["list_int", [0, 1]], - "shape": [ - "list_shape", - [ - [2, 2], - [2, 2] - ] - ] - }, - "node_row_ptr": [0, 1, 2] -} diff --git a/common/tvm_utility/data/models/abs_model_aarch64/deploy_lib.so b/common/tvm_utility/data/models/abs_model_aarch64/deploy_lib.so deleted file mode 100755 index e1ad7cebad734..0000000000000 Binary files a/common/tvm_utility/data/models/abs_model_aarch64/deploy_lib.so and /dev/null differ diff --git a/common/tvm_utility/data/models/abs_model_aarch64/deploy_param.params b/common/tvm_utility/data/models/abs_model_aarch64/deploy_param.params deleted file mode 100644 index 1011def01ed82..0000000000000 Binary files a/common/tvm_utility/data/models/abs_model_aarch64/deploy_param.params and /dev/null differ diff --git a/common/tvm_utility/data/models/abs_model_aarch64/inference_engine_tvm_config.hpp b/common/tvm_utility/data/models/abs_model_aarch64/inference_engine_tvm_config.hpp deleted file mode 100644 index 09c8c0beacebe..0000000000000 --- a/common/tvm_utility/data/models/abs_model_aarch64/inference_engine_tvm_config.hpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tvm_utility/pipeline.hpp" - -#ifndef COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ // NOLINT -#define COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - -namespace model_zoo -{ -namespace inf_test -{ -namespace engine_load -{ -namespace abs_model -{ - -static const tvm_utility::pipeline::InferenceEngineTVMConfig config{ - {0, 0, 0}, // modelzoo_version - - // cspell: ignore mtriple - "abs_model_aarch64", // network_name - "llvm -mtriple=aarch64-linux-gnu", // network_backend - - "deploy_lib.so", // network_module_path - "deploy_graph.json", // network_graph_path - "deploy_param.params", // network_params_path - - // cspell: ignore DLCPU - kDLCPU, // tvm_device_type - 0, // tvm_device_id - - {{"a", kDLFloat, 32, 1, {2, 2}}}, // network_inputs - - {{"output", kDLFloat, 32, 1, {2, 2}}} // network_outputs -}; - -} // namespace abs_model -} // namespace engine_load -} // namespace inf_test -} // namespace model_zoo -#endif // COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_AARCH64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - // NOLINT diff --git a/common/tvm_utility/data/models/abs_model_x86_64/deploy_graph.json b/common/tvm_utility/data/models/abs_model_x86_64/deploy_graph.json deleted file mode 100644 index b226c01747dca..0000000000000 --- a/common/tvm_utility/data/models/abs_model_x86_64/deploy_graph.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "nodes": [ - { - "op": "null", - "name": "a", - "inputs": [] - }, - { - "op": "tvm_op", - "name": "tvmgen_default_fused_abs", - "attrs": { - "num_outputs": "1", - "num_inputs": "1", - "flatten_data": "0", - "func_name": "tvmgen_default_fused_abs", - "hash": "1be44995aa501758" - }, - "inputs": [[0, 0, 0]] - } - ], - "arg_nodes": [0], - "heads": [[1, 0, 0]], - "attrs": { - "dltype": ["list_str", ["float32", "float32"]], - "device_index": ["list_int", [1, 1]], - "storage_id": ["list_int", [0, 1]], - "shape": [ - "list_shape", - [ - [2, 2], - [2, 2] - ] - ] - }, - "node_row_ptr": [0, 1, 2] -} diff --git a/common/tvm_utility/data/models/abs_model_x86_64/deploy_lib.so b/common/tvm_utility/data/models/abs_model_x86_64/deploy_lib.so deleted file mode 100644 index 9a6d02817e048..0000000000000 Binary files a/common/tvm_utility/data/models/abs_model_x86_64/deploy_lib.so and /dev/null differ diff --git a/common/tvm_utility/data/models/abs_model_x86_64/deploy_param.params b/common/tvm_utility/data/models/abs_model_x86_64/deploy_param.params deleted file mode 100644 index 1011def01ed82..0000000000000 Binary files a/common/tvm_utility/data/models/abs_model_x86_64/deploy_param.params and /dev/null differ diff --git a/common/tvm_utility/data/models/abs_model_x86_64/inference_engine_tvm_config.hpp b/common/tvm_utility/data/models/abs_model_x86_64/inference_engine_tvm_config.hpp deleted file mode 100644 index 7a7e3ef97c1b3..0000000000000 --- a/common/tvm_utility/data/models/abs_model_x86_64/inference_engine_tvm_config.hpp +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2021 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tvm_utility/pipeline.hpp" - -#ifndef COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_X86_64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ // NOLINT -#define COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_X86_64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - -namespace model_zoo -{ -namespace inf_test -{ -namespace engine_load -{ -namespace abs_model -{ - -static const tvm_utility::pipeline::InferenceEngineTVMConfig config{ - {0, 0, 0}, // modelzoo_version - - "abs_model_x86_64", // network_name - "llvm", // network_backend - - "deploy_lib.so", // network_module_path - "deploy_graph.json", // network_graph_path - "deploy_param.params", // network_params_path - - // cspell: ignore DLCPU - kDLCPU, // tvm_device_type - 0, // tvm_device_id - - {{"a", kDLFloat, 32, 1, {2, 2}}}, // network_inputs - - {{"output", kDLFloat, 32, 1, {2, 2}}} // network_outputs -}; - -} // namespace abs_model -} // namespace engine_load -} // namespace inf_test -} // namespace model_zoo -#endif // COMMON__TVM_UTILITY__DATA__MODELS__ABS_MODEL_X86_64__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - // NOLINT diff --git a/common/tvm_utility/data/models/yolo_v2_tiny/inference_engine_tvm_config.hpp b/common/tvm_utility/data/models/yolo_v2_tiny/inference_engine_tvm_config.hpp deleted file mode 100644 index 45ff0d8ce33e3..0000000000000 --- a/common/tvm_utility/data/models/yolo_v2_tiny/inference_engine_tvm_config.hpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2021 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tvm_utility/pipeline.hpp" - -#ifndef COMMON__TVM_UTILITY__DATA__MODELS__YOLO_V2_TINY__INFERENCE_ENGINE_TVM_CONFIG_HPP_ // NOLINT -#define COMMON__TVM_UTILITY__DATA__MODELS__YOLO_V2_TINY__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - -namespace model_zoo -{ -namespace perception -{ -namespace camera_obstacle_detection -{ -namespace yolo_v2_tiny -{ -namespace tensorflow_fp32_coco -{ - -static const tvm_utility::pipeline::InferenceEngineTVMConfig config{ - {3, 0, 0}, // modelzoo_version - - "yolo_v2_tiny", // network_name - "llvm", // network_backend - - // cspell: ignore DLCPU - "./deploy_lib.so", // network_module_path - "./deploy_graph.json", // network_graph_path - "./deploy_param.params", // network_params_path - - kDLCPU, // tvm_device_type - 0, // tvm_device_id - - {{"input", kDLFloat, 32, 1, {-1, 416, 416, 3}}}, // network_inputs - - {{"output", kDLFloat, 32, 1, {1, 13, 13, 425}}} // network_outputs -}; - -} // namespace tensorflow_fp32_coco -} // namespace yolo_v2_tiny -} // namespace camera_obstacle_detection -} // namespace perception -} // namespace model_zoo -#endif // COMMON__TVM_UTILITY__DATA__MODELS__YOLO_V2_TINY__INFERENCE_ENGINE_TVM_CONFIG_HPP_ - // NOLINT diff --git a/common/tvm_utility/example/yolo_v2_tiny/main.cpp b/common/tvm_utility/example/yolo_v2_tiny/main.cpp deleted file mode 100644 index 38b3355ca38ba..0000000000000 --- a/common/tvm_utility/example/yolo_v2_tiny/main.cpp +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2021-2022 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tvm_utility/pipeline.hpp" -#include "yolo_v2_tiny/inference_engine_tvm_config.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include - -using model_zoo::perception::camera_obstacle_detection::yolo_v2_tiny::tensorflow_fp32_coco::config; - -namespace tvm_utility -{ -namespace yolo_v2_tiny -{ - -class PreProcessorYoloV2Tiny : public tvm_utility::pipeline::PreProcessor -{ -public: - explicit PreProcessorYoloV2Tiny(tvm_utility::pipeline::InferenceEngineTVMConfig config) - : network_input_width(config.network_inputs[0].node_shape[1]), - network_input_height(config.network_inputs[0].node_shape[2]), - network_input_depth(config.network_inputs[0].node_shape[3]), - network_input_datatype_bytes(config.network_inputs[0].tvm_dtype_bits / 8) - { - // Allocate input variable - std::vector shape_x{1, network_input_width, network_input_height, network_input_depth}; - tvm_utility::pipeline::TVMArrayContainer x{ - shape_x, - config.network_inputs[0].tvm_dtype_code, - config.network_inputs[0].tvm_dtype_bits, - config.network_inputs[0].tvm_dtype_lanes, - config.tvm_device_type, - config.tvm_device_id}; - - output = x; - } - - // The cv::Mat can't be used as an input because it throws an exception when - // passed as a constant reference - tvm_utility::pipeline::TVMArrayContainerVector schedule(const std::string & input) - { - // Read input image - auto image = cv::imread(input, cv::IMREAD_COLOR); - if (!image.data) { - throw std::runtime_error("File " + input + " not found"); - } - - // Compute the ratio for resizing and size for padding - double scale_x = static_cast(image.size().width) / network_input_width; - double scale_y = static_cast(image.size().height) / network_input_height; - double scale = std::max(scale_x, scale_y); - - // Perform padding - if (scale != 1) { - cv::resize(image, image, cv::Size(), 1.0f / scale, 1.0f / scale); - } - - size_t w_pad = network_input_width - image.size().width; - size_t h_pad = network_input_height - image.size().height; - - if (w_pad || h_pad) { - cv::copyMakeBorder( - image, image, h_pad / 2, (h_pad - h_pad / 2), w_pad / 2, (w_pad - w_pad / 2), - cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); - } - - // Convert pixel values from int8 to float32. convert pixel value range from 0 - 255 to 0 - 1. - cv::Mat3f image_3f{}; - image.convertTo(image_3f, CV_32FC3, 1 / 255.0f); - - // cv library uses BGR as a default color format, the network expects the data in RGB format - cv::cvtColor(image_3f, image_3f, cv::COLOR_BGR2RGB); - - TVMArrayCopyFromBytes( - output.getArray(), image_3f.data, - network_input_width * network_input_height * network_input_depth * - network_input_datatype_bytes); - - return {output}; - } - -private: - int64_t network_input_width; - int64_t network_input_height; - int64_t network_input_depth; - int64_t network_input_datatype_bytes; - tvm_utility::pipeline::TVMArrayContainer output; -}; - -class PostProcessorYoloV2Tiny : public tvm_utility::pipeline::PostProcessor> -{ -public: - explicit PostProcessorYoloV2Tiny( - tvm_utility::pipeline::InferenceEngineTVMConfig config, std::string label_filename, - std::string anchor_filename) - : network_output_width(config.network_outputs[0].node_shape[1]), - network_output_height(config.network_outputs[0].node_shape[2]), - network_output_depth(config.network_outputs[0].node_shape[3]), - network_output_datatype_bytes(config.network_outputs[0].tvm_dtype_bits / 8) - { - // Parse human readable names for the classes - std::ifstream label_file{label_filename}; - if (!label_file.good()) { - throw std::runtime_error("unable to open label file:" + label_filename); - } - std::string line{}; - while (std::getline(label_file, line)) { - labels.push_back(line); - } - - // Get anchor values for this network from the anchor file - std::ifstream anchor_file{anchor_filename}; - if (!anchor_file.good()) { - throw std::runtime_error("unable to open anchor file:" + anchor_filename); - } - std::string first{}; - std::string second{}; - while (std::getline(anchor_file, line)) { - std::stringstream line_stream(line); - std::getline(line_stream, first, ','); - std::getline(line_stream, second, ','); - anchors.push_back(std::make_pair(std::atof(first.c_str()), std::atof(second.c_str()))); - } - } - - // Sigmoid function - float sigmoid(float x) { return static_cast(1.0 / (1.0 + std::exp(-x))); } - - std::vector schedule(const tvm_utility::pipeline::TVMArrayContainerVector & input) - { - auto l_h = network_output_width; // Layer height - auto l_w = network_output_height; // Layer width - auto n_classes = labels.size(); // Total number of classes - auto n_anchors = anchors.size(); // Total number of anchors - const uint32_t n_coords = 4; // Number of coordinates in a single anchor box - - // Assert data is stored row-majored in input and the dtype is float - assert(input[0].getArray()->strides == nullptr); - assert(input[0].getArray()->dtype.bits == sizeof(float) * 8); - - // Copy the inference data to CPU memory - std::vector infer( - network_output_width * network_output_height * network_output_depth, 0); - TVMArrayCopyToBytes( - input[0].getArray(), infer.data(), - network_output_width * network_output_height * network_output_depth * - network_output_datatype_bytes); - - // Utility function to return data from y given index - auto get_output_data = [this, infer, n_classes, n_anchors, n_coords]( - auto row_i, auto col_j, auto anchor_k, auto offset) { - auto box_index = (row_i * network_output_height + col_j) * network_output_depth; - auto index = box_index + anchor_k * (n_classes + n_coords + 1); - return infer[index + offset]; - }; - - // Vector used to check if the result is accurate, - // this is also the output of this (schedule) function - std::vector scores_above_threshold{}; - - // Parse results into detections. Loop over each detection cell in the model output - for (decltype(l_w) i = 0; i < l_w; i++) { - for (decltype(l_h) j = 0; j < l_h; j++) { - for (size_t anchor_k = 0; anchor_k < n_anchors; anchor_k++) { - // Compute property index - auto box_p = get_output_data(i, j, anchor_k, 4); - - // Decode the confidence of detection in this anchor box - auto p_0 = sigmoid(box_p); - - // Find maximum probability of all classes - float max_p = 0.0f; - int max_ind = -1; - for (size_t i_class = 0; i_class < n_classes; i_class++) { - auto class_p = get_output_data(i, j, anchor_k, 5 + i_class); - if (max_p < class_p) { - max_p = class_p; - max_ind = i_class; - } - } - - if (max_ind == -1) continue; - - // Decode and copy class probabilities - std::vector class_probabilities{}; - float p_total = 0; - for (size_t i_class = 0; i_class < n_classes; i_class++) { - auto class_p = get_output_data(i, j, anchor_k, 5 + i_class); - class_probabilities.push_back(std::exp(class_p - max_p)); - p_total += class_probabilities[i_class]; - } - - // Find the most likely score - auto max_score = class_probabilities[max_ind] * p_0 / p_total; - - // Draw all detections with high scores - if (max_score > 0.3) { - scores_above_threshold.push_back(max_score); - } - } - } - } - - return scores_above_threshold; - } - -private: - int64_t network_output_width; - int64_t network_output_height; - int64_t network_output_depth; - int64_t network_output_datatype_bytes; - std::vector labels{}; - std::vector> anchors{}; -}; - -} // namespace yolo_v2_tiny -} // namespace tvm_utility - -bool check_near(double expected, double actual, double tolerance) -{ - return fabs(expected - actual) <= tolerance; -} - -int main(int argc, char * argv[]) -{ - // init node to use parameters - rclcpp::init(argc, argv); - auto node = rclcpp::Node::make_shared("yolo_v2_tiny_example"); - // Filename of the image on which to run the inference - node->declare_parameter("image_filename"); - // Name of file containing the human readable names of the classes. One class - // on each line. - node->declare_parameter("label_filename"); - // Name of file containing the anchor values for the network. Each line is one - // anchor. each anchor has 2 comma separated floating point values. - node->declare_parameter("anchor_filename"); - // Packages data and artifacts directory path. - node->declare_parameter("data_path"); - - RCLCPP_INFO(node->get_logger(), "Node started"); - - // Instantiate the pipeline - using PrePT = tvm_utility::yolo_v2_tiny::PreProcessorYoloV2Tiny; - using IET = tvm_utility::pipeline::InferenceEngineTVM; - using PostPT = tvm_utility::yolo_v2_tiny::PostProcessorYoloV2Tiny; - - PrePT PreP{config}; - IET IE{config, "tvm_utility", node->get_parameter("data_path").as_string()}; - PostPT PostP{ - config, node->get_parameter("label_filename").as_string(), - node->get_parameter("anchor_filename").as_string()}; - - tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP); - - // Push data input the pipeline and get the output - auto output = pipeline.schedule(node->get_parameter("image_filename").as_string()); - - // Define reference vector containing expected values, expressed as hexadecimal integers - std::vector int_output{0x3eb64594, 0x3f435656, 0x3ece1600, 0x3e99d381, - 0x3f1cd6bc, 0x3f14f4dd, 0x3ed8065f, 0x3ee9f4fa, - 0x3ec1b5e8, 0x3f4e7c6c, 0x3f136af1}; - - std::vector expected_output(int_output.size()); - - // A memcpy means that the floats in expected_output have a well-defined binary value - for (size_t i = 0; i < int_output.size(); i++) { - memcpy(&expected_output[i], &int_output[i], sizeof(expected_output[i])); - } - - // Test: check if the generated output is equal to the reference - if (expected_output.size() == output.size()) { - RCLCPP_INFO(node->get_logger(), "Model has proper output size"); - } else { - RCLCPP_INFO(node->get_logger(), "Model has unexpected output size"); - } - - for (size_t i = 0; i < output.size(); ++i) { - if (check_near(expected_output[i], output[i], 0.0001)) { - std::cout << "Model has proper output at index: " << i << std::endl; - RCLCPP_INFO(node->get_logger(), "Model has proper output at index: %zu", i); - - } else { - RCLCPP_INFO(node->get_logger(), "Model has unexpected output at index: %zu", i); - } - } - rclcpp::shutdown(); - return 0; -} diff --git a/common/tvm_utility/include/tvm_utility/pipeline.hpp b/common/tvm_utility/include/tvm_utility/pipeline.hpp deleted file mode 100644 index 8c1f22bfcba30..0000000000000 --- a/common/tvm_utility/include/tvm_utility/pipeline.hpp +++ /dev/null @@ -1,414 +0,0 @@ -// Copyright 2021-2022 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TVM_UTILITY__PIPELINE_HPP_ -#define TVM_UTILITY__PIPELINE_HPP_ - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace tvm_utility -{ - -/** - * @brief Possible version status for a neural network. - */ -enum class Version { - OK, - Unknown, - Untested, - Unsupported, -}; - -namespace pipeline -{ - -class TVMArrayContainer -{ -public: - TVMArrayContainer() = default; - - TVMArrayContainer( - std::vector shape, DLDataTypeCode dtype_code, int32_t dtype_bits, int32_t dtype_lanes, - DLDeviceType device_type, int32_t device_id) - { - TVMArrayHandle x{}; - TVMArrayAlloc( - &shape[0], static_cast(shape.size()), dtype_code, dtype_bits, dtype_lanes, device_type, - device_id, &x); - handle_ = std::make_shared(x); - } - - TVMArrayHandle getArray() const { return *handle_.get(); } - -private: - std::shared_ptr handle_{nullptr, [](TVMArrayHandle ptr) { - if (ptr) { - TVMArrayFree(ptr); - } - }}; -}; - -using TVMArrayContainerVector = std::vector; - -/** - * @class PipelineStage - * @brief Base class for all types of pipeline stages. - * - * @tparam InputType The datatype of the input of the pipeline stage. - * @tparam OutputType The datatype of the output from the pipeline stage. - */ -template -class PipelineStage -{ -public: - /** - * @brief Execute the pipeline stage - * - * @param input The data to push into the pipeline stage. The pipeline stage - * should not modify the input data. - * @return The output of the pipeline - */ - virtual OutputType schedule(const InputType & input) = 0; - virtual ~PipelineStage() {} - InputType input_type_indicator_; - OutputType output_type_indicator_; -}; - -/** - * @class PreProcessor - * @brief Pre processor of the inference pipeline. In charge of converting data - * from InputType into TVMArrayContainer format. Any necessary pre processing - * of the data, such as image resizing or padding, should also be done in this - * stage . - * - * @tparam InputType The data type of the input to the pre-processing pipeline - * stage. Usually a ROS message type. - */ -template -class PreProcessor : public PipelineStage -{ -}; - -/** - * @class InferenceEngine - * @brief Pipeline stage in charge of machine learning inference. - */ -class InferenceEngine : public PipelineStage -{ -}; - -/** - * @class PostProcessor - * @brief The post processing stage of the inference pipeline. In charge of - * converting the tensor data from the inference stage into detections in - * OutputType, usually a ROS message format. Thing such as decoding bounding - * boxes, non-maximum-suppression and minimum score filtering should be done in - * this stage. - * - * @tparam OutputType The data type of the output of the inference pipeline. - * Usually a ROS message type. - */ -template -class PostProcessor : public PipelineStage -{ -}; - -/** - * @class Pipeline - * @brief Inference Pipeline. Consists of 3 stages: preprocessor, inference - * stage and postprocessor. - */ -template -class Pipeline -{ - using InputType = decltype(std::declval().input_type_indicator_); - using OutputType = decltype(std::declval().output_type_indicator_); - -public: - /** - * @brief Construct a new Pipeline object - * - * @param pre_processor a PreProcessor object - * @param post_processor a PostProcessor object - * @param inference_engine a InferenceEngine object - */ - Pipeline( - PreProcessorType pre_processor, InferenceEngineType inference_engine, - PostProcessorType post_processor) - : pre_processor_(pre_processor), - inference_engine_(inference_engine), - post_processor_(post_processor) - { - } - - /** - * @brief run the pipeline. Return asynchronously in a callback. - * - * @param input The data to push into the pipeline - * @return The pipeline output - */ - OutputType schedule(const InputType & input) - { - auto input_tensor = pre_processor_.schedule(input); - auto output_tensor = inference_engine_.schedule(input_tensor); - return post_processor_.schedule(output_tensor); - } - -private: - PreProcessorType pre_processor_{}; - InferenceEngineType inference_engine_{}; - PostProcessorType post_processor_{}; -}; - -// NetworkNode -typedef struct -{ - // Node name - std::string node_name; - - // Network data type configurations - DLDataTypeCode tvm_dtype_code; - int32_t tvm_dtype_bits; - int32_t tvm_dtype_lanes; - - // Shape info - std::vector node_shape; -} NetworkNode; - -typedef struct -{ - // Network info - std::array modelzoo_version; - std::string network_name; - std::string network_backend; - - // Network files - std::string network_module_path; - std::string network_graph_path; - std::string network_params_path; - - // Inference hardware configuration - DLDeviceType tvm_device_type; - int32_t tvm_device_id; - - // Network inputs - std::vector network_inputs; - - // Network outputs - std::vector network_outputs; -} InferenceEngineTVMConfig; - -class InferenceEngineTVM : public InferenceEngine -{ -public: - explicit InferenceEngineTVM( - const InferenceEngineTVMConfig & config, const std::string & pkg_name, - const std::string & autoware_data_path = "") - : config_(config) - { - // Get full network path - std::string network_prefix; - if (autoware_data_path == "") { - network_prefix = ament_index_cpp::get_package_share_directory(pkg_name) + "/models/" + - config.network_name + "/"; - } else { - network_prefix = autoware_data_path + "/" + pkg_name + "/models/" + config.network_name + "/"; - } - std::string network_module_path = network_prefix + config.network_module_path; - std::string network_graph_path = network_prefix + config.network_graph_path; - std::string network_params_path = network_prefix + config.network_params_path; - - // Load compiled functions - std::ifstream module(network_module_path); - if (!module.good()) { - throw std::runtime_error( - "File " + network_module_path + " specified in inference_engine_tvm_config.hpp not found"); - } - module.close(); - tvm::runtime::Module mod = tvm::runtime::Module::LoadFromFile(network_module_path); - - // Load json graph - std::ifstream json_in(network_graph_path, std::ios::in); - if (!json_in.good()) { - throw std::runtime_error( - "File " + network_graph_path + " specified in inference_engine_tvm_config.hpp not found"); - } - std::string json_data( - (std::istreambuf_iterator(json_in)), std::istreambuf_iterator()); - json_in.close(); - - // Load parameters from binary file - std::ifstream params_in(network_params_path, std::ios::binary); - if (!params_in.good()) { - throw std::runtime_error( - "File " + network_params_path + " specified in inference_engine_tvm_config.hpp not found"); - } - std::string params_data( - (std::istreambuf_iterator(params_in)), std::istreambuf_iterator()); - params_in.close(); - - // Parameters need to be in TVMByteArray format - TVMByteArray params_arr; - params_arr.data = params_data.c_str(); - params_arr.size = params_data.length(); - - // Create tvm runtime module - tvm::runtime::Module runtime_mod = (*tvm::runtime::Registry::Get("tvm.graph_executor.create"))( - json_data, mod, static_cast(config.tvm_device_type), config.tvm_device_id); - - // Load parameters - auto load_params = runtime_mod.GetFunction("load_params"); - load_params(params_arr); - - // Get set_input function - set_input = runtime_mod.GetFunction("set_input"); - - // Get the function which executes the network - execute = runtime_mod.GetFunction("run"); - - // Get the function to get output data - get_output = runtime_mod.GetFunction("get_output"); - - for (auto & output_config : config.network_outputs) { - output_.push_back(TVMArrayContainer( - output_config.node_shape, output_config.tvm_dtype_code, output_config.tvm_dtype_bits, - output_config.tvm_dtype_lanes, config.tvm_device_type, config.tvm_device_id)); - } - } - - TVMArrayContainerVector schedule(const TVMArrayContainerVector & input) override - { - // Set input(s) - for (uint32_t index = 0; index < input.size(); ++index) { - if (input[index].getArray() == nullptr) { - throw std::runtime_error("input variable is null"); - } - set_input(config_.network_inputs[index].node_name.c_str(), input[index].getArray()); - } - - // Execute the inference - execute(); - - // Get output(s) - for (uint32_t index = 0; index < output_.size(); ++index) { - if (output_[index].getArray() == nullptr) { - throw std::runtime_error("output variable is null"); - } - get_output(index, output_[index].getArray()); - } - return output_; - } - - /** - * @brief Get version information from the config structure and check if there is a mismatch - * between the supported version(s) and the actual version. - * @param[in] version_from Earliest supported model version. - * @return The version status. - */ - Version version_check(const std::array & version_from) const - { - auto x{config_.modelzoo_version[0]}; - auto y{config_.modelzoo_version[1]}; - Version ret{Version::OK}; - - if (x == 0) { - ret = Version::Unknown; - } else if (x > version_up_to[0] || (x == version_up_to[0] && y > version_up_to[1])) { - ret = Version::Untested; - } else if (x < version_from[0] || (x == version_from[0] && y < version_from[1])) { - ret = Version::Unsupported; - } - - return ret; - } - -private: - InferenceEngineTVMConfig config_; - TVMArrayContainerVector output_; - tvm::runtime::PackedFunc set_input; - tvm::runtime::PackedFunc execute; - tvm::runtime::PackedFunc get_output; - // Latest supported model version. - const std::array version_up_to{2, 1, 0}; -}; - -template < - class PreProcessorType, class InferenceEngineType, class TVMScriptEngineType, - class PostProcessorType> -class TowStagePipeline -{ - using InputType = decltype(std::declval().input_type_indicator_); - using OutputType = decltype(std::declval().output_type_indicator_); - -public: - /** - * @brief Construct a new Pipeline object - * - * @param pre_processor a PreProcessor object - * @param post_processor a PostProcessor object - * @param inference_engine a InferenceEngine object - */ - TowStagePipeline( - std::shared_ptr pre_processor, - std::shared_ptr inference_engine_1, - std::shared_ptr tvm_script_engine, - std::shared_ptr inference_engine_2, - std::shared_ptr post_processor) - : pre_processor_(pre_processor), - inference_engine_1_(inference_engine_1), - tvm_script_engine_(tvm_script_engine), - inference_engine_2_(inference_engine_2), - post_processor_(post_processor) - { - } - - /** - * @brief run the pipeline. Return asynchronously in a callback. - * - * @param input The data to push into the pipeline - * @return The pipeline output - */ - OutputType schedule(const InputType & input) - { - auto input_tensor = pre_processor_->schedule(input); - auto output_infer_1 = inference_engine_1_->schedule(input_tensor); - auto output_tvm_script = tvm_script_engine_->schedule(output_infer_1); - auto output_infer_2 = inference_engine_2_->schedule(output_tvm_script); - return post_processor_->schedule(output_infer_2); - } - -private: - std::shared_ptr pre_processor_; - std::shared_ptr inference_engine_1_; - std::shared_ptr tvm_script_engine_; - std::shared_ptr inference_engine_2_; - std::shared_ptr post_processor_; -}; - -} // namespace pipeline -} // namespace tvm_utility -#endif // TVM_UTILITY__PIPELINE_HPP_ diff --git a/common/tvm_utility/launch/yolo_v2_tiny_example.launch.xml b/common/tvm_utility/launch/yolo_v2_tiny_example.launch.xml deleted file mode 100644 index 045a6fc9dfa27..0000000000000 --- a/common/tvm_utility/launch/yolo_v2_tiny_example.launch.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - diff --git a/common/tvm_utility/package.xml b/common/tvm_utility/package.xml deleted file mode 100644 index a1984a84c0bdc..0000000000000 --- a/common/tvm_utility/package.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - tvm_utility - 0.38.0 - - A set of utility functions to help build a machine learning pipeline using - TVM as the inference engine. - - Ambroise Vincent - Xinyu Wang - Apache License 2.0 - - autoware_cmake - - ament_index_cpp - libopenblas-dev - libopencv-dev - tvm_vendor - - ament_lint_auto - autoware_lint_common - - - ament_cmake - - diff --git a/common/tvm_utility/schema/yolo_v2_tiny_example.schema.json b/common/tvm_utility/schema/yolo_v2_tiny_example.schema.json deleted file mode 100644 index 8ee1987f73a62..0000000000000 --- a/common/tvm_utility/schema/yolo_v2_tiny_example.schema.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Parameters for yolo_v2_tiny_example of tvm_utility", - "type": "object", - "definitions": { - "yolo_v2_tiny_example": { - "type": "object", - "properties": { - "image_filename": { - "type": "string", - "default": "$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/test_image_0.jpg", - "description": "Filename of the image on which to run the inference." - }, - "label_filename": { - "type": "string", - "default": "$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/labels.txt", - "description": "Name of file containing the human readable names of the classes. One class on each line." - }, - "anchor_filename": { - "type": "string", - "default": "$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/anchors.csv", - "description": "Name of file containing the anchor values for the network. Each line is one anchor. each anchor has 2 comma separated floating point values." - }, - "data_path": { - "type": "string", - "default": "$(env HOME)/autoware_data", - "description": "Packages data and artifacts directory path." - } - }, - "required": ["image_filename", "label_filename", "anchor_filename", "data_path"] - } - }, - "properties": { - "/**": { - "type": "object", - "properties": { - "ros__parameters": { - "$ref": "#/definitions/yolo_v2_tiny_example" - } - }, - "required": ["ros__parameters"], - "additionalProperties": false - } - }, - "required": ["/**"], - "additionalProperties": false -} diff --git a/common/tvm_utility/test/abs_model/main.cpp b/common/tvm_utility/test/abs_model/main.cpp deleted file mode 100644 index 4bf1a69c0556b..0000000000000 --- a/common/tvm_utility/test/abs_model/main.cpp +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2021-2022 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtest/gtest.h" -#include "tvm_utility/pipeline.hpp" -// file for current arch x86 or arm is chosen in cmake file -#include -#include - -#include -#include -#include -#include -#include - -using model_zoo::inf_test::engine_load::abs_model::config; - -namespace tvm_utility -{ -namespace abs_model -{ - -class PreProcessorLinearModel : public tvm_utility::pipeline::PreProcessor> -{ -public: - explicit PreProcessorLinearModel(tvm_utility::pipeline::InferenceEngineTVMConfig config) - : network_input_a_width(config.network_inputs[0].node_shape[0]), - network_input_a_height(config.network_inputs[0].node_shape[1]), - network_input_datatype_bytes(config.network_inputs[0].tvm_dtype_bits / 8) - { - // Allocate input variable - std::vector shape_a{network_input_a_width, network_input_a_height}; - tvm_utility::pipeline::TVMArrayContainer a{ - shape_a, - config.network_inputs[0].tvm_dtype_code, - config.network_inputs[0].tvm_dtype_bits, - config.network_inputs[0].tvm_dtype_lanes, - config.tvm_device_type, - config.tvm_device_id}; - - output = a; - } - - // The cv::Mat can't be used as an input because it throws an exception when - // passed as a constant reference - tvm_utility::pipeline::TVMArrayContainerVector schedule(const std::vector & input) - { - float input_mat[2][2]; - input_mat[0][0] = input[0]; - input_mat[0][1] = input[1]; - input_mat[1][0] = input[2]; - input_mat[1][1] = input[3]; - - // Create cv::Mat from input array - cv::Mat a_input = cv::Mat(2, 2, CV_32F, &input_mat); - - TVMArrayCopyFromBytes( - output.getArray(), a_input.data, - network_input_a_width * network_input_a_height * network_input_datatype_bytes); - - return {output}; - } - -private: - int64_t network_input_a_width; - int64_t network_input_a_height; - int64_t network_input_datatype_bytes; - tvm_utility::pipeline::TVMArrayContainer output; -}; - -class PostProcessorLinearModel : public tvm_utility::pipeline::PostProcessor> -{ -public: - explicit PostProcessorLinearModel(tvm_utility::pipeline::InferenceEngineTVMConfig config) - : network_output_width(config.network_outputs[0].node_shape[0]), - network_output_height(config.network_outputs[0].node_shape[1]), - network_output_datatype_bytes(config.network_outputs[0].tvm_dtype_bits / 8) - { - } - - std::vector schedule(const tvm_utility::pipeline::TVMArrayContainerVector & input) - { - // Assert data is stored row-majored in input and the dtype is float - assert(input[0].getArray()->strides == nullptr); - assert(input[0].getArray()->dtype.bits == sizeof(float) * 8); - - // Copy the inference data to CPU memory - std::vector infer(network_output_width * network_output_height, 0.0f); - - TVMArrayCopyToBytes( - input[0].getArray(), infer.data(), - network_output_width * network_output_height * network_output_datatype_bytes); - - return infer; - } - -private: - int64_t network_output_width; - int64_t network_output_height; - int64_t network_output_datatype_bytes; -}; - -TEST(PipelineExamples, SimplePipeline) -{ - // // Instantiate the pipeline - using PrePT = PreProcessorLinearModel; - using IET = tvm_utility::pipeline::InferenceEngineTVM; - using PostPT = PostProcessorLinearModel; - - PrePT PreP{config}; - IET IE{config, "tvm_utility"}; - PostPT PostP{config}; - - tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP); - - auto version_status = IE.version_check({2, 0, 0}); - EXPECT_NE(version_status, tvm_utility::Version::Unsupported); - - // create input array - std::vector input_arr{-1., -2., -3., 4.}; - // send it to the model - auto output = pipeline.schedule(input_arr); - - // define vector with expected values - std::vector expected_output{1., 2., 3., 4.}; - - // // Test: check if the generated output is equal to the reference - EXPECT_EQ(expected_output.size(), output.size()) << "Unexpected output size"; - for (size_t i = 0; i < output.size(); ++i) { - EXPECT_NEAR(expected_output[i], output[i], 0.0001) << "at index: " << i; - } -} - -} // namespace abs_model -} // namespace tvm_utility diff --git a/common/tvm_utility/tvm-utility-yolo-v2-tiny-tests.md b/common/tvm_utility/tvm-utility-yolo-v2-tiny-tests.md deleted file mode 100644 index 39bcc640c2147..0000000000000 --- a/common/tvm_utility/tvm-utility-yolo-v2-tiny-tests.md +++ /dev/null @@ -1,53 +0,0 @@ -# YOLOv2 Tiny Example Pipeline - -This is an example implementation of an inference pipeline using the pipeline -framework. This example pipeline executes the -[YOLO V2 Tiny](https://pjreddie.com/darknet/yolov2/) model and decodes its -output. - -## Compiling the Example - - - -1. Check if model was downloaded during the env preparation step by ansible and - models files exist in the folder $HOME/autoware_data/tvm_utility/models/yolo_v2_tiny. - - If not you can download them manually, see [Manual Artifacts Downloading](https://github.com/autowarefoundation/autoware/tree/main/ansible/roles/artifacts). - -2. Download an example image to be used as test input. This image needs to be - saved in the `artifacts/yolo_v2_tiny/` folder. - - ```sh - curl https://raw.githubusercontent.com/pjreddie/darknet/master/data/dog.jpg \ - > artifacts/yolo_v2_tiny/test_image_0.jpg - ``` - -3. Build. - - ```sh - colcon build --packages-up-to tvm_utility --cmake-args -DBUILD_EXAMPLE=ON - ``` - -4. Run. - - ```sh - ros2 launch tvm_utility yolo_v2_tiny_example.launch.xml - ``` - -## Parameters - -| Name | Type | Default Value | Description | -| ----------------- | ------ | ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | -| `image_filename` | string | `$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/test_image_0.jpg` | Filename of the image on which to run the inference. | -| `label_filename` | string | `$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/labels.txt` | Name of file containing the human readable names of the classes. One class on each line. | -| `anchor_filename` | string | `$(find-pkg-share tvm_utility)/artifacts/yolo_v2_tiny/anchors.csv` | Name of file containing the anchor values for the network. Each line is one anchor. each anchor has 2 comma separated floating point values. | -| `data_path` | string | `$(env HOME)/autoware_data` | Packages data and artifacts directory path. | - -## GPU backend - -Vulkan is supported by default by the tvm_vendor package. -It can be selected by setting the `tvm_utility_BACKEND` variable: - -```sh -colcon build --packages-up-to tvm_utility -Dtvm_utility_BACKEND=vulkan -``` diff --git a/common/tvm_utility/tvm_utility-extras.cmake b/common/tvm_utility/tvm_utility-extras.cmake deleted file mode 100644 index 414644c1fe041..0000000000000 --- a/common/tvm_utility/tvm_utility-extras.cmake +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2021-2022 Arm Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Make inference_engine_tvm_config.hpp available under "data/models/${MODEL_NAME}/". -# Install the TVM artifacts to "share/${PROJECT_NAME}/models/". -# Return the name of the custom target in the DEPENDENCY parameter. -# -# :param MODEL_NAME: the name of the targeted neural network -# :type MODEL_NAME: string -# :param MODEL_BACKEND: the name of the targeted backend -# :type MODEL_BACKEND: string -# :param DEPENDENCY: output parameter; name of the ExternalProject top level target -# :type DEPENDENCY: string -# -function(get_neural_network MODEL_NAME MODEL_BACKEND DEPENDENCY) - set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data) - set(EXTERNALPROJECT_NAME ${MODEL_NAME}_${MODEL_BACKEND}) - set(PREPROCESSING "") - - if(IS_DIRECTORY "${DATA_PATH}/models/${MODEL_NAME}") - set(SOURCE_DIR "${DATA_PATH}/models/${MODEL_NAME}") - set(INSTALL_DIRECTORY "${DATA_PATH}/models/${MODEL_NAME}") - if(EXISTS "${DATA_PATH}/models/${MODEL_NAME}/preprocessing_inference_engine_tvm_config.hpp") - set(PREPROCESSING "${DATA_PATH}/models/${MODEL_NAME}/preprocessing_inference_engine_tvm_config.hpp") - endif() - else() - message(WARNING "No model configuration files were provided") - set(${DEPENDENCY} "" PARENT_SCOPE) - return() - endif() - - include(ExternalProject) - externalproject_add(${EXTERNALPROJECT_NAME} - SOURCE_DIR ${SOURCE_DIR} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - BUILD_BYPRODUCTS "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp" - BUILD_BYPRODUCTS ${PREPROCESSING} - INSTALL_COMMAND "" - ) - install( - DIRECTORY ${INSTALL_DIRECTORY} - DESTINATION "share/${PROJECT_NAME}/models/" - USE_SOURCE_PERMISSIONS - ) - - set(${DEPENDENCY} ${EXTERNALPROJECT_NAME} PARENT_SCOPE) - -endfunction()