Bmf-mobile-dev merge to Bmf-mirror

* Add BMF DockerFile for CPU/GPU * Optimize the face_detect demo * python convert backend * make sure to use clang-format with version 16.0.6 * Add face detect with TRT Colab * Move supper resolution demo * release v0.0.8 (#28) * remove redundant command * Add more information in the Readme.md * optimization for rlease * update transcode demo colab * completion license statement of cpp and oc source code * Fix gpu colab * Fix typos * Mod header files (#39) * unified target documents * rename ff headers * Ci test (#24) * install python wheels from pypi instead of test pypi (#36) * Fix aes colab * add data conversion description in readme (#42) * Add missing headers * Use mutable lambda * Fix npp imgproc * fix bmf.version() * Provide printing macros for ffmpeg header * Revert unnecessary changes * Some lgpl state (#51) * enable ci in codebase * fix: use iterator after erase may cause segmentfault * fix: Memory leaks caused by incorrect buf references. * fix: private data is not copied by default * feat: VideoFrame::copy_props add default parameter copy_private * compile with msvc on windows * input stream and multi generators for c++ graph * use the more general windows api * enable python on windows * only include corecrt.h header file on windows to avoid pybind11's bug * add missed files for windows * Enable and opt windows feature * Enable capalities on Windows * remove the not used patch file * coding style align with CONTRIBUTING.md required * Enable 3rd_party win_rootfs download * Optimize Windows cfg * compile ok android * add video buffer * add gl_texture_video_buffer * fix compile ok * add gl texture ... https://code.byted.org/videoarch/bmf/merge_requests/811
BabitMF · Sep 11, 2024 · 0d94909 · 0d94909
1 parent d3b61cc
commit 0d94909
Show file tree

Hide file tree

Showing 368 changed files with 62,835 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,7 @@
 .idea/*
 .venv/
 *.mov
-*.json
 *.pyc
-*.mp4
-*.png
 *.jpg
 .DS_Store
 *.o
@@ -43,6 +40,7 @@ UserInterface.xcuserstate
 !default.mode2v3
 *.perspectivev3
 !default.perspectivev3
+.vscode
 profile
 *.moved-aside
 DerivedData
@@ -53,3 +51,18 @@ Pods
 !Podfile
 !Podfile.lock
 !Manifest.lock
+
+bmf_lite/android/.idea/
+bmf_lite/android/.gradle
+bmf_lite/android/app/build
+bmf_lite/android/.gradle
+bmf_lite/android/.idea
+bmf_lite/android/lite/.cxx
+bmf_lite/android/lite/build
+bmf_lite/android/local.properties
+bmf_lite/build_android
+bmf_lite/demo/ios_demo/bmf_demo/bmf_demo.xcodeproj/xcshareddata/xcschemes/bmf_demo.xcscheme
+
+*.air
+bmf_lite/build_ios/
+bmf_lite/demo/ios_demo/bmf_demo/bmf_demo.xcodeproj/xcshareddata/
diff --git a/bmf_lite/.gitignore b/bmf_lite/.gitignore
@@ -0,0 +1,5 @@
+build_ohos/
+
+output/
+
+adb_run_cmd.txt
diff --git a/bmf_lite/CMakeLists.txt b/bmf_lite/CMakeLists.txt
@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+
+cmake_policy(SET CMP0077 NEW)
+
+project(HML CXX C)
+
+### general settings
+set(CMAKE_CXX_STANDARD 20)
+
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
+
+### options
+
+option(BMF_LITE_ENABLE_CVPIXELBUFFER "Enable build CVpixelBuffer" OFF)
+option(BMF_LITE_ENABLE_METALBUFFER "Enable build Metal texture buffer" OFF)
+option(BMF_LITE_ENABLE_OPENGLTEXTUREBUFFER "Enable build opengl texture buffer" OFF)
+option(BMF_LITE_ENABLE_CPUMEMORYBUFFER "Enable build cpu memory buffer" OFF)
+option(BMF_LITE_ENABLE_BUFFER_TRANSFORM "Enable build buffer transform" OFF)
+
+option(BMF_LITE_ENABLE_SUPER_RESOLUTION "Enable build super resolution" OFF)
+option(BMF_LITE_ENABLE_DENOISE "Enable build denoise" OFF)
+option(BMF_LITE_ENABLE_CANNY "Enable build canny" OFF)
+option(BMF_LITE_ENABLE_TEX_GEN_PIC "Enable build tex_gen_pic" OFF)
+
+option(BMF_ENABLE_LOG "Enable log" ON)
+
+add_subdirectory(src)
+
+add_subdirectory(demo)
diff --git a/bmf_lite/README.md b/bmf_lite/README.md
@@ -0,0 +1,174 @@
+# 1. Building Android Libraries And Running
+
+### Preliminaries
+
+a. Install cmake 3.22.1 in Android Studio SDK Manager.
+
+> Check cmake version with `cmake -version` and make sure cmake version is not less than 3.20.4.
+
+b. Install build tools with
+
+> Install the NDK and jdk.
+> The NDK version is greater than or equal to r23.
+> The JDK version is greater than or equal to 11.
+
+c. Steps to install QNN SDK
+
+> Register Qualcomm ID: https://myaccount.qualcomm.com/login.
+
+> Download Qualcomm Package Manager: https://qpm.qualcomm.com/#/main/tools/details/QPM3.
+
+> Use Qualcomm Package Manager to extract Quaclomm QNN SDK: https://qpm.qualcomm.com/#/main/tools/details/qualcomm_neural_processing_sdk.
+
+####
+
+    1. qpm-cli --login <username>
+    2. qpm-cli --license-activate qualcomm_neural_processing_sdk
+    3. qpm-cli --extract qualcomm_neural_processing_sdk
+    4. Get header files from include/QNN, copy include/QNN to bmf_lite/src/algorithm/contrib_modules/QnnControlNet/QNN
+    5. Get libQnnSystem.so libQnnHtp.so libQnnHtpV75Stub.so from lib/aarch64-android, copy them to bmf_lite/android/app/SRC/main/assets folder
+    6. Get libQnnHtpV75Skel.so from lib/hexagon-v75/unsigned, copy them to bmf_lite/android/app/SRC/main/assets folder
+
+> Download QNN ControlNet Models: https://aihub.qualcomm.com/models/controlnet_quantized.
+
+> Rename the 4 downloaded model files, i.e. unet.serialized.bin, text_encoder.serialized.bin, vae_decoder.serialized.bin, controlnet.serialized. bin, copy them to the mobile phone's /data/local/TMP/ControlNetData folder.
+
+> Copy the downloaded configuration file(vocab.txt and merges.txt) to the mobile phone /data/local/tmp/ControlNetData folder.
+
+### Install
+
+#### Build modules only
+
+> ./build_android.sh
+> Cpp library output path: bmf_lite/build_android/lib.
+> Cpp-test-program output path: bmf_lite/build_android/bin.
+> Download [bmf_lite_files.tar.gz](https://github.com/BabitMF/bmf/releases/download/files/bmf_lite_files.tar.gz), extract test.jpg and test-canny.png to demo/test_data.
+> Cpp-test-program need android device.
+> If you need to run the vincennes chart sample through the cpp test program, you need the 8gen3 chip. In addition, you need to add the dependent resource path and the model library path in the cpp test program (similar to the method in the app code).
+> Run-cpp-test-program: bash ci/test_android.sh xxx-device.
+
+#### Build bmf_lite release aar only
+
+> cd android
+> ./gradlew :lite:assembleRelease
+> Arr library output path: bmf_lite/android/lite/build/outputs/arr.
+
+### Run Android App
+
+> Open the Android folder using Android studio.
+> Download [bmf_lite_files.tar.gz](https://github.com/BabitMF/bmf/releases/download/files/bmf_lite_files.tar.gz), extract test.mp4 to bmf_lite/android/app/SRC/main/assets.
+> Copy test-canny.png extracted above to the device album.
+> Execute the android studio Run button.
+
+### App introduction
+
+Android has prepared three demo examples to demonstrate our Android Bmf framework, including super-resolution, denoising and Vincenzo graphics. Through these examples, we show how to engineering media algorithms into bmf-lite dynamic library, involving GPU processing, rendering, playback and other modules. After running the app, you can see three buttons corresponding to hyperscore, noise reduction and Venison chart. Click in to see the respective effects.
+
+> #### Super Resolution
+
+<img src="android/app/src/main/res/effect-data/super-resolution.gif" alt="sr" width="250" height="360">
+
+> #### Denoise
+
+<img src="android/app/src/main/res/effect-data/denoise.gif" alt="sr" width="250" height="360">
+
+> #### Vincent diagram
+
+<img src="android/app/src/main/res/effect-data/tex2pic.gif" alt="sr" width="250" height="360">
+
+# 2. Building iOS Libraries And Running
+
+### Preliminaries
+
+1. Install XCode And CMake
+
+```
+brew install cmake
+```
+
+> Check cmake version with `cmake -version` and make sure cmake version is not less than 3.20.4.
+
+### Build bmf_lite static library
+
+```
+bash build_ios.sh
+```
+
+### Run iOS app
+
+Open the bmf_demo.xcodeproj Using XCode, which in folder iOS/bmf_demo.
+
+#### Known Issues
+
+We have prepared four demonstration examples to showcase our framework's capabilities, including super-resolution, denoising, Canny, and Stable diffusion model. Specifically, super-resolution, denoising, and Canny showcase the integration of algorithmic modules into the bmflite dynamic library, while diffusers illustrates how to leverage the framework's capabilities to integrate and manage modules without modifying the bmflite framework.To execute diffusers, it is necessary to have iOS certificates that support 'Extended Virtual Addressing' and 'Increased Memory Limit'. If you run other cases, you can remove these capability.
+
+> #### Super Resolution
+>
+> In this example, we have implemented a super resolution algorithm as a BMF module, showcasing a BMF pipeline that combines decoding, super resolution and rendering.
+> Download [bmf_lite_files.tar.gz](https://github.com/BabitMF/bmf/releases/download/files/bmf_lite_files.tar.gz), extract test.mp4 and put it into XCode to view the effect.
+
+<img src="iOS/data/sr1.gif" alt="sr" width="250" height="360">
+
+> #### Denoise
+>
+> This example implements the denoise algorithm as a BMF module, showcasing a BMF pipeline that combines video capture, noise reduction and rendering.
+
+<img src="iOS/data/denoise1.gif" alt="sr" width="250" height="360">
+
+> #### Real Time Canny
+>
+> In this simple demonstration example, we demonstrate the implementation of a Canny algorithm module using Apple MPS, showcasing a BMF pipeline that combines video capture, canny and rendering.
+
+<img src="iOS/data/canny1.gif" alt="sr" width="250" height="360">
+
+> #### CoreML Diffusers
+>
+> This case illustrates the process of integrating an external algorithm module into the bmf_lite framework and managing its execution. To execute this case, it is necessary to have iOS certificates that support 'Extended Virtual Addressing' and 'Increased Memory Limit'. This model demands high performance, requiring the iOS system version to be 17 or above, running on the Neural Engine and CPU, and consuming significant memory.
+
+- Kill the app when you are not using.
+- Disconnect from XCode during runtime.
+- Reboot your device.
+- Avoid testing multiple times in a short period of time.
+
+<img src="iOS/data/sd1.gif" alt="sr" width="250" height="360">
+
+# 3. Building OHOS Libraries and running the sample
+
+### Preliminaries
+
+Before you start, make sure to get the following requirements ready.
+
+1. Register on Huawei developer platform to become a developer.
+   https://developer.huawei.com/consumer/en/doc/start/registration-and-verification-0000001053628148
+
+2. Download and install DevEco-Studio.
+   https://developer.huawei.com/consumer/cn/download/
+   https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/ide-software-install-0000001558013317-V5
+
+3. Install the OpenHarmony SDK through DevEco-Studio.
+
+4. Install CMake 3.15 or newer.
+
+5. Download [bmf_lite_files.tar.gz](https://github.com/BabitMF/bmf/releases/download/files/bmf_lite_files.tar.gz), extract test.mp4 and put it into `ohos/entry/src/main/resources/rawfile`.
+
+6. A mobile device running HarmonyOS, or an emulator in DevEco-Studio.
+
+### Building OHOS Libraries
+
+1. Open the `bmf_lite/ohos` project in DevEco-Studio.
+
+2. Select the `bmf_lite` item with an `H` icon from the `Run configurations`.
+
+3. Click the green run button.
+
+### Running the sample
+
+1. Open the `bmf_lite/ohos` project in DevEco-Studio.
+
+2. Select the `entry` item with a yellow square from the `Run configurations`.
+
+3. Click the green run button.
+
+4. Click the `Open signing configs` link in the Run terminal, and log in with a Huawei account on the popped up website.
+
+5. Click the green run button again.
diff --git a/bmf_lite/android/app/build.gradle b/bmf_lite/android/app/build.gradle
@@ -0,0 +1,57 @@
+plugins {
+    id 'com.android.application'
+}
+
+
+android {
+    compileSdk 31
+
+    defaultConfig {
+        minSdk 24
+        targetSdk 28
+        versionCode 1
+        versionName "1.0"
+        ndk {
+            ndkVersion "21.1.6352462"
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    packagingOptions {
+        pickFirst 'lib/*/libbmf_lite.so'
+        pickFirst 'lib/*/libbmf_module_sdk.so'
+        pickFirst 'lib/*/libbytenn.so'
+        doNotStrip '*.so'
+    }
+}
+
+dependencies {
+    implementation project(":lite")
+    implementation "com.google.code.gson:gson:2.8.5"
+    implementation 'androidx.appcompat:appcompat:1.4.1'
+    implementation 'com.google.android.material:material:1.5.0'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
+    implementation 'com.google.code.gson:gson:2.8.5'
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.3'
+    androidTestImplementation 'androidx.test:runner:1.4.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
+
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.3.60"
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.0'
+    implementation 'androidx.lifecycle:lifecycle-extensions:2.0.0'
+    implementation 'androidx.appcompat:appcompat:1.1.0'
+    implementation 'androidx.preference:preference:1.1.0'
+    implementation 'androidx.activity:activity-ktx:1.1.0'
+    implementation "androidx.core:core-ktx:1.2.0"
+
+}
diff --git a/bmf_lite/android/app/src/androidTest/AndroidManifest.xml b/bmf_lite/android/app/src/androidTest/AndroidManifest.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="com.bmf.lite.app">
+    <uses-feature
+        android:glEsVersion="0x00020000"
+        android:required="true" />
+    <supports-gl-texture android:name="GL_OES_compressed_ETC1_RGB8_texture" />
+    <supports-gl-texture android:name="GL_OES_compressed_paletted_texture" />
+</manifest>
diff --git a/bmf_lite/android/app/src/main/AndroidManifest.xml b/bmf_lite/android/app/src/main/AndroidManifest.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="com.bmf.lite.app">
+
+    <supports-gl-texture android:name="GL_OES_compressed_ETC1_RGB8_texture" />
+    <supports-gl-texture android:name="GL_OES_compressed_paletted_texture" />
+    <uses-permission android:name="android.permission.CAMERA" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
+
+    <uses-feature android:name="android.hardware.camera" />
+    <uses-feature android:name="android.hardware.camera.autofocus" />
+
+    <uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    <uses-permission android:name="android.permission.CHANGE_WIFI_STATE" />
+    <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.DISABLE_KEYGUARD" />
+
+    <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.VIBRATE" />
+    <uses-permission android:name="android.permission.WRITE_MEDIA_STORAGE" tools:ignore="ProtectedPermissions" />
+
+    <application
+        android:allowBackup="true"
+        android:requestLegacyExternalStorage="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.BmfLite">
+        <meta-data
+            android:name="com.google.android.actions"
+            android:resource="@xml/actions" />
+
+        <activity
+            android:name=".ui.MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+        <activity
+            android:name=".ui.VideoPlayActivity"
+            android:screenOrientation="portrait"
+            android:exported="true" />
+        <activity
+            android:name=".ui.CameraPlayActivity"
+            android:screenOrientation="portrait"
+            android:exported="true" />
+        <activity
+            android:name=".ui.CameraPlayBrightenActivity"
+            android:screenOrientation="portrait"
+            android:exported="true" />
+        <activity
+            android:name=".ui.TexGenPicActivity"
+            android:screenOrientation="portrait"
+            android:exported="true" />
+    </application>
+
+</manifest>