run unit tests automatically for every PR

.github/workflows/build_wheels.yml

@@ -91,7 +91,7 @@ jobs:
           echo "auditwheel show {wheel} done..." &&
           tmp_dir=$(mktemp -d) &&
           unzip -d ${tmp_dir} {wheel} &&
-          auditwheel -v repair $(ldd ${tmp_dir}/bmf/lib/* | awk '{if(NF>1){print $1}}' | grep "avdevice\|avfilter\|avformat\|avcodec\|postproc\|swresample\|swscale\|avutil\|cuda\|bmf_module_sdk\|engine\|hmp" | sort | uniq | awk '{printf(" --exclude %s", $1);}') -w {dest_dir} {wheel} &&
+          auditwheel -v repair $(ldd ${tmp_dir}/bmf/lib/* | awk '{if(NF>1){print $1}}' | grep "avdevice\|avfilter\|avformat\|avcodec\|postproc\|swresample\|swscale\|avutil\|cuda\|libxcb\|libXau\|libbz2\|bmf_module_sdk\|engine\|hmp" | sort | uniq | awk '{printf(" --exclude %s", $1);}') -w {dest_dir} {wheel} &&
           rm -rf ${tmp_dir}
 
     - uses: actions/upload-artifact@v3

.github/workflows/ci.yml

@@ -18,39 +18,41 @@ jobs:
       with:
         submodules: 'recursive'
 
-    - name: prepare envs
+    - name: prepare libraries
       run: |
-        apt update
-        apt install -y make git pkg-config libssl-dev cmake binutils-dev libgoogle-glog-dev gcc g++ golang wget libgl1
-        apt install -y python3.9 python3-dev python3-pip
-        python3 -m pip install timeout_decorator numpy onnxruntime pytest opencv-python
+        sudo apt update
+        sudo apt install -y make git pkg-config libssl-dev cmake binutils-dev libgoogle-glog-dev gcc g++ golang wget libgl1
+        sudo apt install -y python3.9 python3-dev python3-pip
+        sudo python3 -m pip install timeout_decorator numpy onnxruntime pytest opencv-python
         # The version of ffmpeg installed via apt is 4.2, but the current test results are based on version 4.4, so here we need to compile version 4.4 of ffmpeg from source code
         #apt install -y ffmpeg libavcodec-dev libavdevice-dev libavfilter-dev libavformat-dev libavresample-dev libavutil-dev libpostproc-dev libswresample-dev libswscale-dev
-        scripts/build_ffmpeg.sh nasm yasm x264 x265 opus
+        # When running unit tests, nasm/yasm installed via apt will cause the indicators to be misaligned, so it needs to be compiled from source code
+        #apt install -y nasm yasm
+        sudo scripts/build_ffmpeg.sh nasm yasm x264 x265 opus
 
     - name: build
       id: build
-      env:
-        CMAKE_ARGS: "-DCMAKE_VERBOSE_MAKEFILE=ON"
       run: |
-        ./build.sh
+        ./build.sh debug
+
+    - name: set envs
+      run: |
+        echo "C_INCLUDE_PATH=${C_INCLUDE_PATH}:$(pwd)/output/bmf/include" >> $GITHUB_ENV
+        echo "CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:$(pwd)/output/bmf/include" >> $GITHUB_ENV
+        echo "LIBRARY_PATH=${LIBRARY_PATH}:$(pwd)/output/bmf/lib" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd)/output/bmf/lib:/usr/local/lib" >> $GITHUB_ENV
+        echo "PYTHONPATH=$(pwd)/output/bmf/lib:$(pwd)/output" >> $GITHUB_ENV
+        echo "PATH=${PATH}:$(pwd)/output/bmf/bin" >> $GITHUB_ENV
+        echo "HMP_TEST_DATA_ROOT=$(pwd)/bmf/hml/tests/data" >> $GITHUB_ENV
 
     - name: test
       id: test
-      env:
-        C_INCLUDE_PATH: ${C_INCLUDE_PATH}:$(pwd)/output/bmf/include
-        CPLUS_INCLUDE_PATH: ${CPLUS_INCLUDE_PATH}:$(pwd)/output/bmf/include
-        LIBRARY_PATH: ${LIBRARY_PATH}:$(pwd)/output/bmf/lib
-        LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}:$(pwd)/output/bmf/lib
-        PATH: ${PATH}:$(pwd)/output/bmf/bin
-        PYTHONPATH: $(pwd)/output/bmf/lib:$(pwd)/output
-        HMP_TEST_DATA_ROOT: $(pwd)/bmf/hml/tests/tests/data
       run: |
         # install test modules for c++/python/go
-        module_manager install test_python_module python my_module:my_module $(pwd)/output/test/customize_module v0.0.1
-        module_manager install test_cpp_module c++ libcopy_module:CopyModule $(pwd)/output/test/c_module/lib v0.0.1
+        sudo $(pwd)/output/bmf/bin/module_manager install test_python_module python my_module:my_module $(pwd)/output/test/customize_module v0.0.1
+        sudo $(pwd)/output/bmf/bin/module_manager install test_cpp_module c++ libcopy_module:CopyModule $(pwd)/output/test/c_module/lib v0.0.1
         (cd $(pwd)/output/test/go_module && go build -buildmode c-shared -o lib/test_go_module.so test_go_module.go)
-        module_manager install test_go_module go test_go_module:PassThrough $(pwd)/output/test/go_module/lib v0.0.1
+        sudo $(pwd)/output/bmf/bin/module_manager install test_go_module go test_go_module:PassThrough $(pwd)/output/test/go_module/lib v0.0.1
         # download files
         (cd output && wget https://github.com/BabitMF/bmf/releases/download/files/files.tar.gz && tar xvf files.tar.gz && rm -rf files.tar.gz)
         (cd output && wget https://github.com/BabitMF/bmf/releases/download/files/models.tar.gz && tar xvf models.tar.gz && rm -rf models.tar.gz)
@@ -67,49 +69,10 @@ jobs:
         (cd output/test/c_module                && python3 test_video_c_module.py)
         (cd output/test/predict                 && python3 predict_sample.py)
         (cd output/test/dynamical_graph         && python3 dynamical_graph.py)
-        #(cd output/test/av_log_buffer           && python3 test_av_log_buffer.py)
+        (cd output/test/av_log_buffer           && python3 test_av_log_buffer.py)
         #(cd output/test/push_data_into_graph    && python3 test_push_data.py)
         (cd output/test/complex_edit_case       && python3 test_complex_case.py)
         (cd output/test/complex_edit_case       && python3 test_compare_with_edit.py)
         # test hmp
-        (cd bmf/hml/tests/data                  && ./gen.sh)
+        (cd bmf/hml/tests/data                  && ./gen.sh $(pwd)/../../../../output/files)
         (cd bmf/hml/tests                       && pytest)
-
-    - name: Set PR status check for test result
-      if: ${{ always() }} # This ensures that this step is always executed, even if previous steps fail.
-      uses: actions/github-script@v4
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        script: |
-          const prNumber = context.payload.pull_request.number;
-          const context = 'ci/pull-request';
-
-          if (${{ failure() }}) {
-            // Test failed, set the status check to failure and prevent PR merging
-            const state = 'failure';
-            const description = 'The pull request cannot be merged due to a failed test.';
-            github.repos.createCommitStatus({
-              ...context.repo,
-              sha: context.sha,
-              state,
-              description,
-              context,
-              target_url: `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
-            });
-            core.setFailed('Test failed, merging the pull request is not allowed.');
-          } else {
-            // Test succeeded, set the status check to success
-            const state = 'success';
-            const description = 'The pull request passed all checks.';
-            github.repos.createCommitStatus({
-              ...context.repo,
-              sha: context.sha,
-              state,
-              description,
-              context
-            });
-          }
-
-    - name: Setup tmate session
-      if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3

bmf/c_modules/CMakeLists.txt

@@ -45,6 +45,7 @@ if(BMF_ENABLE_FFMPEG)
     set(SRCS src/ffmpeg_decoder.cpp
              src/ffmpeg_encoder.cpp
              src/ffmpeg_filter.cpp
+	     src/ffmpeg_func_registry.cpp
              src/video_sync.cpp
              src/audio_fifo.cpp
              src/audio_resampler.cpp
@@ -96,7 +97,7 @@ mac_update(pass_through)
 module_install(pass_through)
 
 # clock module
-set(CLOCK_MODULE_HDRS include/clock_module.h)
+set(CLOCK_MODULE_HDRS include/clock_module.h include/fraction.hpp)
 set(CLOCK_MODULE_SRCS src/clock_module.cpp)
 add_library(clock SHARED ${CLOCK_MODULE_HDRS} ${CLOCK_MODULE_SRCS})
 target_include_directories(clock PUBLIC include)

bmf/hml/tests/data/colors/gen.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-JPEG_FILE=../../../../../output/bmf/files/Color-wheel-light-color-spectrum.jpeg
+JPEG_FILE=${FILES_DIR}/Color-wheel-light-color-spectrum.jpeg
 
 # pix_fmt colorspace out
 gen_yuv_rgb()

bmf/hml/tests/data/common/gen.sh

@@ -10,4 +10,4 @@ jpeg_to_rgb()
 
 
 #
-jpeg_to_rgb ../../../../../output/bmf/files/Lenna.png Lenna
+jpeg_to_rgb ${FILES_DIR}/Lenna.png Lenna

bmf/hml/tests/data/gen.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+if [ $# -lt 1 ]
+then
+    printf "Usage: %s <files_dir>\n" $0
+    exit 1
+fi
+
+export FILES_DIR=$1
 export FFMPEG_LOG_OPTS="-hide_banner -loglevel warning"
 #export FFMPEG_LOG_OPTS="-hide_banner"
 

bmf/hml/tests/data/videos/gen.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-VIDEO_FILE=../../../../../output/bmf/files/big_bunny_10s_30fps.mp4
+VIDEO_FILE=${FILES_DIR}/big_bunny_10s_30fps.mp4
 #LOG_OPTS="-hide_banner -loglevel warning"
 LOG_OPTS="-hide_banner"
 

bmf/test/complex_edit_case/flip_and_scale.py

@@ -18,7 +18,7 @@ def create_graph(self, option=None):
         a1 = a1.atrim(start=0, end=5).asetpts('PTS-STARTPTS')
 
         video2 = self.graph.decode(
-            {'input_path': "../../files/big_bunny_1min_30fps_only_video.mp4"})
+            {'input_path': "../../files/big_bunny_1min_30fps.mp4"})
         v2 = video2['video']
         v2 = v2.trim(start=30, end=35).setpts('PTS-STARTPTS').scale(500, 400)
         a2 = video2['audio']

scripts/build_ffmpeg.sh

@@ -149,7 +149,6 @@ function build_ffmpeg_unix() {
               --pkg-config-flags="--static" \
               --enable-shared \
               --disable-static \
-              --disable-autodetect \
               --extra-libs=-lpthread \
               --extra-libs=-lm \
               --cc='clang -arch x86_64' \
@@ -160,7 +159,6 @@ function build_ffmpeg_unix() {
               --pkg-config-flags="--static" \
               --enable-shared \
               --disable-static \
-              --disable-autodetect \
               --extra-libs=-lpthread \
               --extra-libs=-lm \
               --cc='clang -arch arm64' \
@@ -171,7 +169,6 @@ function build_ffmpeg_unix() {
               --pkg-config-flags="--static" \
               --enable-shared \
               --disable-static \
-              --disable-autodetect \
               --extra-libs=-lpthread \
               --extra-libs=-lm \
               ${@:4}