Merge branch 'main' into wasm

jerinphilip · Oct 24, 2024 · a1684d2 · a1684d2
2 parents 3f90cc0 + 9f0b1a2
commit a1684d2
Show file tree

Hide file tree

Showing 55 changed files with 1,258 additions and 1,088 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -343,7 +343,6 @@ jobs:
        uses: marvinpinto/action-automatic-releases@latest
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
-         automatic_release_tag: ${{ github.ref_name }}
          prerelease: false
          title: "${{ github.ref_name }}"
          files: |

diff --git a/.gitignore b/.gitignore
@@ -51,3 +51,7 @@ __pycache__
 slimt.egg-info
 env
 dist
+
+# Java compiled stuff
+*.class
+
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,5 +1,39 @@
 {
     "configurations": [
+        {
+            "name": "(gdb) t12n",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceFolder}/env/bin/python3",
+            "args": [
+                "${workspaceFolder}/scripts/t12n.py",
+                "${workspaceFolder}/../slimt-t12n/outputs/mal-eng/model.nano.npz.decoder.yml",
+                "<",
+                "${workspaceFolder}/data/ml-xlit.txt"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${fileDirname}",
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "Set Disassembly Flavor to Intel",
+                    "text": "-gdb-set disassembly-flavor intel",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "Enable following into child for debugging.",
+                    "text": "set follow-fork-mode child",
+                    "ignoreFailures": true
+                }
+            ]
+        },
         {
             "name": "(gdb) test",
             "type": "cppdbg",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,6 +18,7 @@ option(WITH_RUY "Use ruy" OFF)
 option(WITH_GEMMOLOGY "Use gemmology" ON)
 option(WITH_BLAS "Use BLAS. Otherwise moves to ruy" ON)
 
+option(SLIMT_USE_INTERNAL_PCRE2 "Use external PCRE2, not system" OFF)
 option(USE_BUILTIN_SENTENCEPIECE "Use SentencePiece supplied as 3rd-party" ON)
 
 option(USE_AVX512 "Use AVX512" OFF)
@@ -26,9 +27,13 @@ option(USE_SSSE3 "Use SSSE3" OFF)
 option(USE_SSE2 "Use SSE2" OFF)
 option(USE_NEON "Use NEON" OFF)
 
+option(BUILD_SHARED "Build shared libraries" ON)
+option(BUILD_STATIC "Build static libraries" ON)
+
 option(SLIMT_PACKAGE "Package for cmake, pkgconfig" OFF)
 option(SLIMT_PYTHON_LINK_STATIC
        "link-method to produce python package (static/shared)" ON)
+option(SLIMT_GENERATED_UNIT_TESTS "Generate unit tests to run using Python" OFF)
 
 option(BUILD_CLI "Build command-line applications" ON)
 option(BUILD_PYTHON "Build Python bindings library" OFF)
@@ -213,6 +218,7 @@ endif(UNIX)
 
 add_subdirectory(slimt)
 add_subdirectory(app)
+add_subdirectory(tests)
 
 if(BUILD_PYTHON)
   if(USE_PYBIND11_SOURCE)
@@ -227,3 +233,7 @@ endif(BUILD_PYTHON)
 if(BUILD_WASM)
   add_subdirectory(bindings/wasm)
 endif(BUILD_WASM)
+
+if(BUILD_JNI)
+  add_subdirectory(bindings/java)
+endif(BUILD_JNI)
diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt
@@ -3,10 +3,6 @@ if(BUILD_CLI)
   set_target_properties(slimt_cli PROPERTIES OUTPUT_NAME "slimt-cli")
   target_link_libraries(slimt_cli PUBLIC slimt-static)
 
-  add_executable(slimt_test test.cc)
-  set_target_properties(slimt_test PROPERTIES OUTPUT_NAME "slimt-test")
-  target_link_libraries(slimt_test PUBLIC slimt-static)
-
   set(SLIMT_BINARIES slimt_cli)
 
   if(UNIX)