Merge branch 'master' into ci_test

bmf/demo/face_detect/detect_sample.py

@@ -12,15 +12,14 @@ def main():
         "model_path": "../../models/version-RFB-640.onnx",
         "label_to_frame": 1
     })
-    video_stream = graph.decode({'input_path': "../../files/face_test_small.mp4"})
+    video_stream = graph.decode({'input_path': "../../files/face.mp4"})
     #video_stream = graph.download({
     #    'input_url': 'https://github.com/fromwhzz/test_video/raw/master/face.mp4',
     #    'local_path': '../../files/face_test.mp4'
     #}).decode()
     detect_stream = video_stream['video'].module('onnx_face_detect',
                                                  pre_module=onnx_face_detect)
-    detect_stream[0].encode(None, {"output_path": "../../files/out.mp4"})
-    detect_stream[1].module("upload").run()
+    detect_stream[0].encode(None, {"output_path": "../../files/out.mp4"}).run()
 
 
 if __name__ == '__main__':

bmf/demo/face_detect/nms.py

@@ -0,0 +1,44 @@
+
+import numpy as np
+
+def NMS(bounding_boxes, confidence_score, threshold = 0.5):
+    if len(bounding_boxes) == 0:
+        return [], []
+
+    boxes = np.array(bounding_boxes)
+
+    start_x = boxes[:, 0]
+    start_y = boxes[:, 1]
+    end_x = boxes[:, 2]
+    end_y = boxes[:, 3]
+
+    score = np.array(confidence_score)
+
+    picked_boxes = []
+    picked_score = []
+
+    areas = (end_x - start_x + 1) * (end_y - start_y + 1)
+
+    order = np.argsort(score)
+
+    while order.size > 0:
+        index = order[-1]
+
+        picked_boxes.append(bounding_boxes[index])
+        picked_score.append(confidence_score[index])
+
+        x1 = np.maximum(start_x[index], start_x[order[:-1]])
+        x2 = np.minimum(end_x[index], end_x[order[:-1]])
+        y1 = np.maximum(start_y[index], start_y[order[:-1]])
+        y2 = np.minimum(end_y[index], end_y[order[:-1]])
+
+        w = np.maximum(0.0, x2 - x1 + 1)
+        h = np.maximum(0.0, y2 - y1 + 1)
+        intersection = w * h
+
+        ratio = intersection / (areas[index] + areas[order[:-1]] - intersection)
+
+        left = np.where(ratio < threshold)
+        order = order[left]
+
+    return picked_boxes, picked_score

bmf/demo/face_detect/onnx_face_detect.py

@@ -1,5 +1,6 @@
 import onnxruntime as rt
 import numpy as np
+import bmf.hml.hmp as mp
 import time
 import sys
 if sys.version_info.major == 2:
@@ -10,30 +11,9 @@
 from PIL import Image
 from bmf import Module, Log, LogLevel, InputType, ProcessResult, Packet, Timestamp, scale_av_pts, av_time_base, \
     BmfCallBackType, VideoFrame, AudioFrame
+from nms import NMS
 import threading
 
-
-class DetectResult(object):
-
-    def __init__(self, x1, y1, x2, y2, label, score, pts=0):
-        self.x1_ = x1
-        self.y1_ = y1
-        self.x2_ = x2
-        self.y2_ = y2
-        self.label_ = label
-        self.score_ = score
-
-    def __str__(self):
-        msg = "x1:{},y1:{},x2:{},y2:{},label:{},score:{}".format(
-            self.x1_, self.y1_, self.x2_, self.y2_, self.label_, self.score_)
-        return msg
-
-    def __repr__(self):
-        msg = "x1:{},y1:{},x2:{},y2:{},label:{},score:{}".format(
-            self.x1_, self.y1_, self.x2_, self.y2_, self.label_, self.score_)
-        return msg
-
-
 class onnx_face_detect(Module):
 
     def __init__(self, node=None, option=None):
@@ -106,24 +86,26 @@ def pre_process(self, image_list):
     # transform the onnx reslt to detect result object
     def post_process(self, input_pil_arrays, boxes, scores):
         output_list = []
+        boxes_data = []
+        scores_data = []
         for image_id in range(len(input_pil_arrays)):
 
             image = input_pil_arrays[image_id]
             output_data = []
-            for index in range(len(boxes[0])):
+            for index in range(len(boxes[image_id])):
                 if (scores[image_id][index][1]) > 0.8:
                     box = (boxes[image_id][index])
                     x1 = int(box[0] * image.size[0])
                     y1 = int(box[1] * image.size[1])
                     x2 = int(box[2] * image.size[0])
                     y2 = int(box[3] * image.size[1])
-                    detect_result = DetectResult(x1, y1, x2, y2, 1,
-                                                 scores[image_id][index][1])
-                    output_data.append(detect_result)
-            output_list.append(output_data)
+                    boxes_data.append([x1, y1, x2, y2])
+                    scores_data.append(scores[image_id][index][1])
+
+            nms_boxes, nms_scores = NMS(boxes_data, scores_data)
+            output_list.append(nms_boxes)
         return output_list
 
-    # overlay info to frame
     def label_frame(self, input_frames, input_pil_arrays, detect_result_list):
         from PIL import ImageDraw
         output_frame_list = []
@@ -133,8 +115,8 @@ def label_frame(self, input_frames, input_pil_arrays, detect_result_list):
             for index_box in range(len(detect_result_list[index_frame])):
                 detect_result = detect_result_list[index_frame][index_box]
                 draw.rectangle([
-                    detect_result.x1_, detect_result.y1_, detect_result.x2_,
-                    detect_result.y2_
+                    detect_result[0], detect_result[1], detect_result[2],
+                    detect_result[3]
                 ])
             del draw
 

bmf/demo/face_detect/trt_face_detect.py

@@ -17,28 +17,7 @@
 
 from bmf import *
 import bmf.hml.hmp as mp
-
-
-class DetectResult(object):
-
-    def __init__(self, x1, y1, x2, y2, label, score, pts=0):
-        self.x1_ = x1
-        self.y1_ = y1
-        self.x2_ = x2
-        self.y2_ = y2
-        self.label_ = label
-        self.score_ = score
-
-    def __str__(self):
-        msg = "x1:{},y1:{},x2:{},y2:{},label:{},score:{}".format(
-            self.x1_, self.y1_, self.x2_, self.y2_, self.label_, self.score_)
-        return msg
-
-    def __repr__(self):
-        msg = "x1:{},y1:{},x2:{},y2:{},label:{},score:{}".format(
-            self.x1_, self.y1_, self.x2_, self.y2_, self.label_, self.score_)
-        return msg
-
+from nms import NMS
 
 class trt_face_detect(Module):
 
@@ -145,23 +124,26 @@ def pre_process(self, torch_image_array):
 
         return input_tensor
 
-    def post_process(self, pil_image_array, boxes, scores):
+    def post_process(self, input_pil_arrays, boxes, scores):
         output_list = []
-        boxes_num = boxes.shape[1]
-        for image_id in range(len(pil_image_array)):
-            image = pil_image_array[image_id]
-            box_data = []
-            for index in range(boxes_num):
-                if (scores[image_id, index, 1] > 0.8):
-                    box = boxes[image_id, index, :]
+        boxes_data = []
+        scores_data = []
+        for image_id in range(len(input_pil_arrays)):
+
+            image = input_pil_arrays[image_id]
+            output_data = []
+            for index in range(len(boxes[image_id])):
+                if (scores[image_id][index][1]) > 0.8:
+                    box = (boxes[image_id][index])
                     x1 = int(box[0] * image.size[0])
                     y1 = int(box[1] * image.size[1])
                     x2 = int(box[2] * image.size[0])
                     y2 = int(box[3] * image.size[1])
-                    detect_result = DetectResult(x1, y1, x2, y2, 1,
-                                                 scores[image_id, index, 1])
-                    box_data.append(detect_result)
-            output_list.append(box_data)
+                    boxes_data.append([x1, y1, x2, y2])
+                    scores_data.append(scores[image_id][index][1])
+
+            nms_boxes, nms_scores = NMS(boxes_data, scores_data)
+            output_list.append(nms_boxes)
         return output_list
 
     def label_frame(self, input_frames, pil_image_array, detect_result_list):
@@ -173,8 +155,8 @@ def label_frame(self, input_frames, pil_image_array, detect_result_list):
             for index_box in range(len(detect_result_list[index_frame])):
                 detect_result = detect_result_list[index_frame][index_box]
                 draw.rectangle([
-                    detect_result.x1_, detect_result.y1_, detect_result.x2_,
-                    detect_result.y2_
+                    detect_result[0], detect_result[1], detect_result[2],
+                    detect_result[3]
                 ])
             del draw
             numpy_image = np.asarray(image)

bmf/hml/cmake/dependencies.cmake

@@ -34,6 +34,7 @@ if(HMP_ENABLE_TORCH)
         OUTPUT_VARIABLE SITE_ROOT
         OUTPUT_STRIP_TRAILING_WHITESPACE)
     set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE)
+    set(CUDNN_LIBRARY_PATH /usr/local/cuda/targets/x86_64-linux/lib/libcudnn.so.8)
     find_package(Torch HINTS ${SITE_ROOT})
     find_library(TORCH_PYTHON_LIB torch_python HINTS ${SITE_ROOT}/*/lib)
     if(Torch_FOUND AND TORCH_PYTHON_LIB)

bmf/hml/py/CMakeLists.txt

@@ -14,6 +14,13 @@ if(HMP_ENABLE_CUDA) # need by ffmpeg.h
     target_include_directories(_hmp PRIVATE ${CUDA_INCLUDE_DIRS})
 endif()
 
+if(BUILD_SHARED_LIBS)
+    target_compile_options(_hmp
+        PRIVATE
+            -DHMP_BUILD_SHARED
+        )
+endif()
+
 
 target_include_directories(_hmp PRIVATE 
     ${CMAKE_CURRENT_SOURCE_DIR}

bmf/hml/py/py_numpy.cpp

@@ -36,8 +36,9 @@ template <> struct npy_format_descriptor<hmp::Half> {
     }
     static constexpr auto name() { return _("float16"); }
 };
-}
-} //
+
+} // namespace detail
+} // namespace pybind11
 
 namespace py = pybind11;
 
@@ -65,7 +66,7 @@ static ScalarType numpyDtypeToScalarType(py::dtype dtype) {
     throw std::runtime_error(std::string("Got unsupported numpy dtype"));
 }
 
-Tensor tensor_from_numpy(const py::array &arr) {
+HMP_API Tensor tensor_from_numpy(const py::array &arr) {
     int ndim = arr.ndim();
     SizeArray shape, strides;
     auto itemsize = arr.itemsize();
@@ -80,19 +81,20 @@ Tensor tensor_from_numpy(const py::array &arr) {
     }
 
     auto buf_info = std::make_shared<pybind11::buffer_info>(arr.request());
-    auto ptr = DataPtr(buf_info->ptr,
-                       [buf_info](void *ptr) mutable {
-                           py::gil_scoped_acquire acquire;
-                           // explict release in gil guard
-                           buf_info.reset();
-                       },
-                       kCPU);
+    auto ptr = DataPtr(
+        buf_info->ptr,
+        [buf_info](void *ptr) mutable {
+            py::gil_scoped_acquire acquire;
+            // explict release in gil guard
+            buf_info.reset();
+        },
+        kCPU);
 
     return from_buffer(std::move(ptr), numpyDtypeToScalarType(arr.dtype()),
                        shape, strides);
 }
 
-py::array tensor_to_numpy(const Tensor &tensor) {
+HMP_API py::array tensor_to_numpy(const Tensor &tensor) {
     HMP_REQUIRE(tensor.is_cpu(),
                 "Only support convert cpu tensor to numpy, got {}",
                 tensor.device_type());

bmf/python/CMakeLists.txt

@@ -2,18 +2,27 @@ file(GLOB PY_SRCS *.cpp *.h)
 
 pybind11_add_module(_bmf SHARED ${PY_SRCS})
 
+
 if(BMF_ENABLE_FFMPEG)
     target_link_libraries(_bmf PRIVATE nlohmann bmf_module_sdk engine ${BMF_FFMPEG_TARGETS})
 else()
     target_link_libraries(_bmf PRIVATE nlohmann bmf_module_sdk engine)
 endif()
 
+target_link_libraries(_bmf
+        PRIVATE _hmp)
+
 if(BMF_ENABLE_CUDA)
     # need by ffmpeg for hwframe support
     target_link_libraries(_bmf
         PRIVATE cuda::cuda)
 endif()
 
+if(BMF_ENABLE_TORCH)
+    target_include_directories(_bmf PUBLIC ${TORCH_INCLUDE_DIRS})
+    target_link_libraries(_bmf PRIVATE ${TORCH_LIBRARIES} ${TORCH_PYTHON_LIB})
+endif()
+
 
 target_include_directories(_bmf PRIVATE 
     ${CMAKE_CURRENT_SOURCE_DIR}