Update from upstream Sep 20, 2023

.circleci/config.yml

@@ -1,52 +1,6 @@
 version: 2.1
 jobs:
-  build-with-wormhole:
-    docker:
-      - image: 'emscripten/emsdk:3.1.8'
-    resource_class: medium
-
-    working_directory: ~/checkout
-
-    steps:
-      - checkout
-
-      - run:
-          name: Build WASM WORMHOLE
-          command: |
-            bash build-wasm.sh WORMHOLE
-
-      - run:
-          name: Check artifacts
-          working_directory: build-wasm
-          command: |
-            ARTIFACT_BASE="bergamot-translator-worker"
-            ARTIFACT_SUFFIX="with-wormhole"
-            ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX
-
-            if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then
-              echo "Artifacts Successfully Generated"
-              mkdir ../artifacts
-              cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm
-              cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js
-              cd ../artifacts
-              shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
-              ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
-              cp ../BERGAMOT_VERSION .
-            else
-              echo "Failure: Artifacts Not Present"
-              exit 1
-            fi
-
-      - persist_to_workspace:
-          root: .
-          paths:
-            - artifacts/*
-
-      - store_artifacts:
-          path: "artifacts"
-          destination: "wasm-wormhole"
-
-  build-without-wormhole:
+  build:
     docker:
       - image: 'emscripten/emsdk:3.1.8'
     resource_class: medium
@@ -66,17 +20,16 @@ jobs:
           working_directory: build-wasm
           command: |
             ARTIFACT_BASE="bergamot-translator-worker"
-            ARTIFACT_SUFFIX="without-wormhole"
-            ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX
+            ARTIFACT_FINAL=$ARTIFACT_BASE
 
             if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then
               echo "Artifacts Successfully Generated"
               mkdir ../artifacts
               cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm
               cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js
               cd ../artifacts
-              shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
-              ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX
+              shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize
+              ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize
             else
               echo "Failure: Artifacts Not Present"
               exit 1
@@ -89,7 +42,7 @@ jobs:
 
       - store_artifacts:
           path: "artifacts"
-          destination: "wasm-without-wormhole"
+          destination: "wasm"
 
   publish_to_github:
     docker:
@@ -106,18 +59,13 @@ jobs:
                 name: "Publish Release on GitHub"
                 command: |
                   export TAG_VERSION=$(cat ./artifacts/BERGAMOT_VERSION)
-                  cat ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole >> ./artifacts/sha256-filesize
-                  rm ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole ./artifacts/BERGAMOT_VERSION
+                  rm ./artifacts/BERGAMOT_VERSION
                   ghr -t ${GHTOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${TAG_VERSION} ./artifacts/
 
 workflows:
   build:
       jobs:
-          - build-with-wormhole:
-              filters:
-                tags:
-                  only: /^v.*/
-          - build-without-wormhole:
+          - build:
               filters:
                 tags:
                   only: /^v.*/
@@ -128,7 +76,6 @@ workflows:
                 branches:
                   ignore: /.*/
               requires:
-                - build-without-wormhole
-                - build-with-wormhole
+                - build
 
 

.github/workflows/arm.yml

@@ -0,0 +1,139 @@
+name: ARM
+'on':
+  push:
+    branches:
+    - main
+    - ci-sandbox
+  pull_request:
+    branches:
+    - '**'
+env:
+  ccache_basedir: ${{ github.workspace }}
+  ccache_dir: "${{ github.workspace }}/.ccache"
+  ccache_compilercheck: content
+  ccache_compress: 'true'
+  ccache_compresslevel: 9
+  ccache_maxsize: 200M
+  ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache
+  ndk: "${{ github.workspace }}/android-ndk-r23b"
+  abi: "arm64-v8a"
+  minsdk_version : 28
+  android_platform: 28
+
+jobs:
+  ubuntu:
+    name: "arm-v8a cross-compile via Android NDK"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install prerequisites
+      run: |
+          wget -c --quiet https://dl.google.com/android/repository/android-ndk-r23b-linux.zip
+          unzip -qq android-ndk-r23b-linux.zip
+          sudo apt-get -y install ccache cmake
+
+    - name: Generate ccache_vars for ccache based on machine
+      shell: bash
+      id: ccache_vars
+      run: |-
+        echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
+        echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
+
+    - name: Cache-op for build-cache through ccache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.ccache_dir }}
+        key: ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
+        restore-keys: |-
+          ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
+          ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}
+          ccache-${{ matrix.identifier }}
+
+    - name: ccache environment setup
+      run: |-
+        echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV
+        echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
+        echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
+        echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
+
+    - name: ccache prolog
+      run: |-
+        ccache -s # Print current cache stats
+        ccache -z # Zero cache entry
+
+    - name: Generate buildfiles for bergamot-translator on android via cmake
+      run: |-
+        mkdir -p build 
+        cd build
+        NDK=${{ env.ndk }}
+        ABI=${{ env.abi }}
+        MINSDK_VERSION=${{ env.minsdk_version }}
+        ANDROID_PLATFORM=android-${{ env.android_platform }}
+        OTHER_ANDROID_ARGS=(
+            -DANDROID_ARM_NEON=TRUE
+        )
+        OTHER_MARIAN_ARGS=(
+            -DCOMPILE_CUDA=off
+            -DCOMPILE_CPU=on
+            -DCMAKE_HAVE_THREADS_LIBRARY=1
+            -DCMAKE_USE_WIN32_THREADS_INIT=0
+            -DCMAKE_USE_PTHREADS_INIT=1
+            -DTHREADS_PREFER_PTHREAD_FLAG=ON
+            -DBUILD_ARCH=armv8-a
+            # -DCOMPILE_WITHOUT_EXCEPTIONS=on # Apparently this can reduce the binary size, let's see.
+            -DSSPLIT_USE_INTERNAL_PCRE2=ON
+        )
+        # Additionally list variables finally configured.
+        cmake -L \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \
+            -DANDROID_TOOLCHAIN=clang \
+            -DANDROID_ABI=$ABI \
+            -DANDROID_PLATFORM=$ANDROID_PLATFORM \
+            -DANDROID_NATIVE_API_LEVEL=$MINSDKVERSION \
+            -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi-4.8 \
+            -DANDROID_STL=c++_static \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            "${OTHER_ANDROID_ARGS[@]}" "${OTHER_MARIAN_ARGS[@]}" \
+            ..
+
+
+    - name : Build bergamot-translator for android
+      working-directory: build
+      run: |-
+          make -j2 
+
+    - name: ccache epilog
+      run: 'ccache -s # Print current cache stats'
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: ${{github.workspace}}/build/app/bergamot
+
+
+  # Disable release for now.
+  # release:
+  #   name: Release Latest Build
+  #   runs-on: ubuntu-latest
+  #   needs: [ubuntu]
+  #   if: github.ref == 'refs/heads/master'
+  #   steps:
+  #    - name: Download artifacts
+  #      uses: actions/download-artifact@v2
+  #     
+  #    - name: Update GitHub prerelease
+  #      uses: marvinpinto/action-automatic-releases@latest
+  #      with:
+  #        repo_token: ${{ secrets.GITHUB_TOKEN }}
+  #        automatic_release_tag: latest
+  #        prerelease: true
+  #        title: "Latest Build"
+  #        files: |
+  #          artifact/marian-decoder