feature: add WebAssembly basic support

CMakeLists.txt

@@ -121,6 +121,12 @@ else()
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${BMF_ASSEMBLE_ROOT}/bmf/bin)
 endif()
 
+if (EMSCRIPTEN)
+    # Add some flags for wasm
+    # -fPIC , USE_PTHREADS, SHARED_MEMORY for dynamic linking
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -s USE_PTHREADS=1 -s SHARED_MEMORY=1")
+    set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
+endif()
 
 ### dependencies
 if(NOT DEFINED BUILD_SHARED_LIBS)

bmf/CMakeLists.txt

@@ -3,9 +3,13 @@ add_subdirectory(sdk)
 #add_subdirectory(go_module_sdk) #FIXME: compile failed
 add_subdirectory(engine)
 add_subdirectory(c_modules)
-add_subdirectory(python_modules)
-# XXX: no go modules now
-#add_subdirectory(go_modules)
+if (NOT EMSCRIPTEN)
+    add_subdirectory(python_modules)
+    # XXX: no go modules now
+    #add_subdirectory(go_modules)
+    add_subdirectory(cmd)
+endif()
+
 if(BMF_ENABLE_TEST)
     add_subdirectory(test/c_module)
     add_subdirectory(demo/gpu_module)
@@ -14,7 +18,6 @@ if(BMF_ENABLE_TEST)
     endif()
 endif(BMF_ENABLE_TEST)
 
-add_subdirectory(cmd)
 if(HMP_ENABLE_JNI)
     add_subdirectory(java)
 endif()
@@ -113,6 +116,10 @@ if(ANDROID)
     endif()
 endif()
 
+if (EMSCRIPTEN)
+    add_subdirectory(demo/wasm)
+endif()
+
 
 
 add_custom_command(TARGET bmf_assem
@@ -154,7 +161,7 @@ if (BMF_LOCAL_DEPENDENCIES)
 endif()
 
 ## copy symbol link of hmp and _jhmp
-if(NOT APPLE AND NOT ANDROID)
+if(NOT APPLE AND NOT ANDROID AND NOT EMSCRIPTEN)
     if(WIN32)
         add_custom_command(TARGET bmf_assem
             POST_BUILD

bmf/c_modules/CMakeLists.txt

@@ -13,14 +13,23 @@ function(mac_update)
 endfunction()
 
 function(set_soname)
-    if(NOT APPLE AND NOT ANDROID)
+    if(NOT APPLE AND NOT ANDROID AND NOT EMSCRIPTEN)
         set_target_properties(${ARGV0} PROPERTIES
             VERSION "${BMF_VERSION_MAJOR}.${BMF_VERSION_MINOR}.${BMF_VERSION_PATCH}"
             SOVERSION "${BMF_VERSION_MAJOR}"
         )
     endif()
 endfunction()
 
+function(set_emscripten_side_module_property)
+    if (EMSCRIPTEN)
+        set_property(TARGET ${ARGV0} PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
+        target_link_options(${ARGV0} PRIVATE "-sSIDE_MODULE=2")
+        target_compile_options(${ARGV0} PRIVATE "-sSIDE_MODULE=2")
+    endif()
+endfunction()
+
+
 function(module_install)
     if(WIN32)
         set_target_properties(${ARGV0} PROPERTIES
@@ -60,6 +69,8 @@ if(BMF_ENABLE_FFMPEG)
     )
 
     add_library(builtin_modules SHARED ${SRCS} ${HDRS})
+    set_emscripten_side_module_property(builtin_modules)
+
     target_include_directories(builtin_modules
         PUBLIC
             $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/bmf/c_modules/include>
@@ -76,13 +87,20 @@ if(BMF_ENABLE_FFMPEG)
         target_link_libraries(builtin_modules
             PRIVATE cuda::cuda)
     endif()
-
-    target_link_libraries(builtin_modules
-        PRIVATE
+    if (NOT EMSCRIPTEN)
+        target_link_libraries(builtin_modules
+                PRIVATE
+                ${BMF_FFMPEG_TARGETS}
+                bmf_module_sdk
+        )
+    else()
+        target_link_libraries(builtin_modules
+            PRIVATE
+            ${X264_LIB}
             ${BMF_FFMPEG_TARGETS}
             bmf_module_sdk
-    )
-
+        )
+    endif()
     set_soname(builtin_modules)
     mac_update(builtin_modules)
 
@@ -104,6 +122,7 @@ endif()
 set(PASS_THROUGH_HDRS include/pass_through_module.h)
 set(PASS_THROUGH_SRCS src/pass_through_module.cpp)
 add_library(pass_through SHARED ${PASS_THROUGH_HDRS} ${PASS_THROUGH_SRCS})
+set_emscripten_side_module_property(pass_through)
 target_include_directories(pass_through PUBLIC include)
 target_link_libraries(pass_through PRIVATE bmf_module_sdk)
 set_soname(pass_through)
@@ -114,6 +133,7 @@ module_install(pass_through)
 set(CLOCK_MODULE_HDRS include/clock_module.h include/fraction.hpp)
 set(CLOCK_MODULE_SRCS src/clock_module.cpp)
 add_library(clock SHARED ${CLOCK_MODULE_HDRS} ${CLOCK_MODULE_SRCS})
+set_emscripten_side_module_property(clock)
 target_include_directories(clock PUBLIC include)
 target_link_libraries(clock PRIVATE bmf_module_sdk)
 set_soname(clock)
@@ -124,21 +144,24 @@ module_install(clock)
 set(MOCK_DECODER_MODULE_HDRS include/mock_decoder.h)
 set(MOCK_DECODER_MODULE_SRCS src/mock_decoder.cpp)
 add_library(MockDecoder SHARED ${MOCK_DECODER_MODULE_HDRS} ${MOCK_DECODER_MODULE_SRCS})
+set_emscripten_side_module_property(MockDecoder)
 target_include_directories(MockDecoder PUBLIC include)
 target_link_libraries(MockDecoder PRIVATE bmf_module_sdk)
 set_soname(MockDecoder)
 mac_update(MockDecoder)
 module_install(MockDecoder)
 
 # GoMockDecoder
-set(GOMOCK_DECODER_MODULE_HDRS include/go_mock_decoder.h)
-set(GOMOCK_DECODER_MODULE_SRCS src/go_mock_decoder.cpp)
-add_library(GoMockDecoder SHARED ${GOMOCK_DECODER_MODULE_HDRS} ${GOMOCK_DECODER_MODULE_SRCS})
-target_include_directories(GoMockDecoder PUBLIC include)
-target_link_libraries(GoMockDecoder PRIVATE bmf_module_sdk)
-set_soname(GoMockDecoder)
-mac_update(GoMockDecoder)
-module_install(GoMockDecoder)
+if (NOT EMSCRIPTEN)
+    set(GOMOCK_DECODER_MODULE_HDRS include/go_mock_decoder.h)
+    set(GOMOCK_DECODER_MODULE_SRCS src/go_mock_decoder.cpp)
+    add_library(GoMockDecoder SHARED ${GOMOCK_DECODER_MODULE_HDRS} ${GOMOCK_DECODER_MODULE_SRCS})
+    target_include_directories(GoMockDecoder PUBLIC include)
+    target_link_libraries(GoMockDecoder PRIVATE bmf_module_sdk)
+    set_soname(GoMockDecoder)
+    mac_update(GoMockDecoder)
+    module_install(GoMockDecoder)
+endif()
 
 # tests, commented
 if(BMF_ENABLE_TEST)

bmf/demo/wasm/CMakeLists.txt

@@ -0,0 +1,43 @@
+# Recipe to build copy module
+add_library(copy_module SHARED 
+    copy_module.cc
+    copy_module.h
+)
+target_link_libraries(copy_module
+    PRIVATE
+    bmf_module_sdk
+)
+set_property(TARGET copy_module PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
+target_link_options(copy_module PRIVATE "-sSIDE_MODULE=2")
+target_compile_options(copy_module PRIVATE "-sSIDE_MODULE=2")
+
+
+# Gernal flag
+set(DEMO_FLAGS "-fexceptions -s ASYNCIFY -s 'ASYNCIFY_IMPORTS=[\"loadLibrary\"]'")
+list(APPEND DEMO_FLAGS "-s ASYNCIFY_STACK_SIZE=5120000")
+list(APPEND DEMO_FLAGS "-s MAIN_MODULE=1")
+list(APPEND DEMO_FLAGS "-s USE_PTHREADS=1")
+list(APPEND DEMO_FLAGS "-s SHARED_MEMORY=1")
+list(APPEND DEMO_FLAGS "-s INITIAL_MEMORY=64MB")
+list(APPEND DEMO_FLAGS "-s ALLOW_MEMORY_GROWTH")
+list(APPEND DEMO_FLAGS "--preload-file ${CMAKE_SOURCE_DIR}/bmf/c_modules/meta/BUILTIN_CONFIG.json@BUILTIN_CONFIG.json")
+list(APPEND DEMO_FLAGS "--preload-file ${CMAKE_SOURCE_DIR}/big_bunny_10s_30fps.mp4@big_bunny_10s_30fps.mp4")
+string(REPLACE ";" " " DEMO_FLAGS "${DEMO_FLAGS}")
+
+# Demo1: builtin modules with video
+add_executable(demo_builtin_video demo_builtin_video.cc)
+set(CMAKE_EXECUTABLE_SUFFIX ".html")
+set_target_properties(demo_builtin_video PROPERTIES LINK_FLAGS "${DEMO_FLAGS}")
+target_link_libraries(demo_builtin_video
+    PRIVATE 
+    engine bmf_module_sdk
+)
+
+# Demo2: copy module
+add_executable(demo_copy_module demo_copy_module.cc)
+set(CMAKE_EXECUTABLE_SUFFIX ".html")
+set_target_properties(demo_copy_module PROPERTIES LINK_FLAGS "${DEMO_FLAGS}")
+target_link_libraries(demo_copy_module
+    PRIVATE 
+    engine bmf_module_sdk
+)

bmf/demo/wasm/README.md

@@ -0,0 +1,73 @@
+# Introduction
+This demo includes two programs:
+
+* The first program decodes, transforms, and encodes video using built-in modules.
+* The second program decodes video, utilizes an external copy module, and encodes the video.
+
+All modules operate in synchronous mode, as multithreading is not yet supported.
+
+
+
+# Prerequisite
+1. Build your FFmpeg and x264 encoder using Emscripten.
+2. Download the assets from [here](https://github.com/BabitMF/bmf/releases/download/files/files.tar.gz) and copy big_bunny_10s_30fps.mp4 to the base path of your BMF project.
+3. Set up an HTTP server to host the WebAssembly resources.
+4. Create a lib folder in the directory where the demo_*.html files are generated, and copy libbuiltin_modules.so into it. (Typically, this file can be found in `$PATH_TO_YOUR_BMF/build/output/bmf/bin`. You can also use the command `find . -name "demo_copy_module.html"` to locate it.)
+## Example of a python webserver
+### Find the directory
+```bash
+PATH_TO_YOUR_PROJECT=...
+SERVER_DIRECTORY=$(realpath "$(dirname "$(find $PATH_TO_YOUR_PROJECT -name "demo_copy_module.html")")")
+LIB_DIRCTORY=$(realpath $(dirname $(find $PATH_TO_YOUR_PROJECT/build/output/bmf/lib -name "libbuiltin_modules.so")))
+```
+
+### Generate HTTPS Public and Private Key
+```bash
+openssl req -newkey rsa:2048 -x509 -keyout server.pem -out server.pem -days 365 -nodes
+```
+
+### Set Up a HTTPS Server
+Create a Python script with the following code:
+(You can use the script at same directory as well)
+```python
+import http.server
+import ssl
+directory=$SERVER_DIRECTORY # EDIT IT ON YOUR OWN
+
+class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory=directory, **kwargs)
+    def end_headers(self):
+        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
+        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
+        super().end_headers()
+
+server_address = ('0.0.0.0', 443)
+httpd = http.server.HTTPServer(server_address, CustomHTTPRequestHandler)
+httpd.socket = ssl.wrap_socket(httpd.socket,
+                               keyfile="server.pem",  
+                               certfile="server.pem",# Create you own
+                               server_side=True)
+
+print(f"Serving on port {server_address[1]}")
+httpd.serve_forever()
+```
+
+### Copy modules
+```bash
+mkdir $SERVER_DIRECTORY/lib
+cp $LIB_DIRCTORY/*.so $SERVER_DIRECTORY/lib
+```
+# Run tests in browser
+Visit `https://demo_builtin_video.html` and `https://demo_copy_module.html`.
+
+Utilize the developer tools to view additional logs and download resources from the filesystem.
+
+## Screenshoot
+![](demo1.png)
+
+
+![](demo2.png)
+
+
+

bmf/demo/wasm/copy_module.cc

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Babit Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "copy_module.h"
+
+int CopyModule::process(Task &task) {
+    PacketQueueMap &input_queue_map = task.get_inputs();
+    PacketQueueMap::iterator it;
+
+    // process all input queues
+    for (it = input_queue_map.begin(); it != input_queue_map.end(); it++) {
+        // input stream label
+        int label = it->first;
+
+        // input packet queue
+        Packet pkt;
+        // process all packets in one input queue
+        while (task.pop_packet_from_input_queue(label, pkt)) {
+            // Get a input packet
+
+            // if packet is eof, set module done
+            if (pkt.timestamp() == BMF_EOF) {
+                task.set_timestamp(DONE);
+                task.fill_output_packet(label, Packet::generate_eof_packet());
+                return 0;
+            }
+
+            // Get packet data
+            // Here we should know the data type in packet
+            auto vframe = pkt.get<VideoFrame>();
+
+            // Deep copy
+            VideoFrame vframe_out = VideoFrame(vframe.frame().clone());
+            vframe_out.copy_props(vframe);
+
+            // Add output frame to output queue
+            auto output_pkt = Packet(vframe_out);
+
+            task.fill_output_packet(label, output_pkt);
+        }
+    }
+    return 0;
+}
+REGISTER_MODULE_CLASS(CopyModule)

bmf/demo/wasm/copy_module.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 Babit Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef BMF_COPY_MODULE_H
+#define BMF_COPY_MODULE_H
+
+#include <bmf/sdk/bmf.h>
+#include <bmf/sdk/packet.h>
+
+USE_BMF_SDK_NS
+
+class CopyModule : public Module {
+  public:
+    CopyModule(int node_id, JsonParam option) : Module(node_id, option) {}
+
+    ~CopyModule() {}
+
+    virtual int process(Task &task);
+};
+
+#endif