WebAssembly bindings #763
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "main" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| branches: | |
| - '**' | |
| env: | |
| ccache_basedir: ${{ github.workspace }} | |
| ccache_dir: "${{ github.workspace }}/.ccache" | |
| ccache_compilercheck: content | |
| ccache_compress: 'true' | |
| ccache_compresslevel: 9 | |
| ccache_maxsize: 200M | |
| ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache | |
| ccache_version: '4.5' | |
| emsdk_version: 3.1.8 # For use in emscripten build | |
| jobs: | |
| formatting: | |
| name: "format" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential cmake | |
| python3 -m pip install black isort | |
| python3 -m pip install cmake-format | |
| sudo apt-get install -y shfmt | |
| - name: Setup latest clang | |
| run: | | |
| sudo bash scripts/ci/update-alternatives-clang.sh 17 100 | |
| - name: Run format-script | |
| run: | |
| bash scripts/ci/format-check.sh | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: format | |
| path: ${{ github.workspace }}/build/clang-tidy.*.yml | |
| build-test: | |
| name: "build-test" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: "ubuntu" | |
| os: "ubuntu-latest" | |
| - name: "macos" | |
| os: "macos-latest" | |
| - name: "android" | |
| os: "ubuntu-latest" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Setup | |
| run: | | |
| bash scripts/ci/${{ matrix.name }}/01-setup.sh | |
| - name: Build | |
| run: | |
| bash scripts/ci/${{ matrix.name }}/02-build.sh | |
| - name: Test | |
| run: | |
| bash scripts/ci/${{ matrix.name }}/03-test.sh | |
| - name: Package | |
| run: | |
| bash scripts/ci/${{ matrix.name }}/04-package.sh | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ failure() }} | |
| with: | |
| name: slimt | |
| path: | | |
| ${{ github.workspace }}/slimt-coredump | |
| build-wheels: | |
| strategy: | |
| matrix: | |
| os: | |
| - "ubuntu-20.04" | |
| # - "windows-2019" | |
| # include: | |
| # - os: macos-12 | |
| # BUILD_ARCH_MACOS: x86_64 | |
| # BUILD_ARCH_CMAKE: core-avx-i | |
| # - os: macos-12 | |
| # BUILD_ARCH_MACOS: arm64 | |
| # BUILD_ARCH_CMAKE: armv8-a | |
| fail-fast: false | |
| name: "cibuildwheel / ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: "Fetch pybind11" | |
| run: | | |
| git clone https://github.com/pybind/pybind11 3rd-party/pybind11 | |
| - name: Generate ccache_vars for ccache based on machine | |
| shell: bash | |
| id: ccache_vars | |
| run: |- | |
| echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" | |
| echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" | |
| - name: Cache-op for build-cache through ccache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.ccache_dir }} | |
| key: ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} | |
| restore-keys: |- | |
| ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} | |
| ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }} | |
| ccache-cibuildwheel-${{ matrix.os }} | |
| - name: ccache environment setup | |
| shell: bash | |
| run: |- | |
| mkdir -p ${{ env.ccache_dir }} | |
| - name: Inject local version identifier for non tag builds | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| shell: bash | |
| run: |- | |
| echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV | |
| - name: Apply MacOS patch | |
| if: ${{ startsWith(runner.os, 'mac') }} | |
| run: | | |
| patch -p1 < patches/01-marian-fstream-for-macos.patch | |
| - name: "Windows: Install dependencies" | |
| if: ${{ startsWith(runner.os, 'windows') }} | |
| shell: powershell | |
| run: | | |
| C:\msys64\usr\bin\wget.exe -nv ${{ env.NUGET_MKL_URL }} -O mkl.zip | |
| Expand-Archive -Force mkl.zip ${{ env.WIN_NUGET_DIR}} | |
| New-Item -Path ${{ github.workspace }} -Name "mkl" -ItemType "directory" | |
| Move-Item -Path ${{ env.WIN_NUGET_DIR }}\lib\native\win-x64 -Destination ${{ env.WIN_MKL_ROOT }}\lib | |
| Move-Item -Path ${{ env.WIN_NUGET_DIR }}\lib\native\include -Destination ${{ env.WIN_MKL_ROOT }}\include | |
| Get-ChildItem -Recurse -Path ${{ env.WIN_MKL_ROOT }} | |
| $PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' | |
| echo "set(VCPKG_BUILD_TYPE release)" | Tee-Object -FilePath C:\vcpkg\triplets\x64-windows-static.cmake -Append | |
| echo "set(VCPKG_BUILD_TYPE release)" | Tee-Object -FilePath C:\vcpkg\triplets\x64-windows.cmake -Append | |
| # Commenting out, but useful in checks via CI | |
| # cat C:\vcpkg\triplets\x64-windows-static.cmake | |
| # cat C:\vcpkg\triplets\x64-windows.cmake | |
| # cat C:\vcpkg\scripts\buildsystems\vcpkg.cmake | |
| C:\vcpkg\vcpkg install protobuf:x64-windows-static pcre2:x64-windows-static | |
| C:\vcpkg\vcpkg upgrade --no-dry-run | |
| - name: Download ccache | |
| if: ${{ startsWith(runner.os, 'windows') }} | |
| shell: cmake -P {0} | |
| run: | | |
| set(ccache_url "https://github.com/cristianadam/ccache/releases/download/v${{ env.ccache_version }}/${{ runner.os }}.tar.xz") | |
| file(DOWNLOAD "${ccache_url}" ./ccache.tar.xz SHOW_PROGRESS) | |
| execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvf ./ccache.tar.xz) | |
| if(ret AND NOT ret EQUAL 0) | |
| message( FATAL_ERROR "Bad exit status") | |
| endif() | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| # to supply options, put them in 'env', like: | |
| env: | |
| CIBW_BUILD_VERBOSITY: 3 | |
| CIBW_ARCHS_MACOS: ${{ matrix.BUILD_ARCH_MACOS }} | |
| CIBW_ENVIRONMENT_LINUX: | |
| USE_CCACHE=1 | |
| CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} | |
| CCACHE_COMPRESS=${{ env.ccache_compress }} | |
| CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} | |
| CCACHE_MAXSIZE=${{ env.ccache_maxsize }} | |
| PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} | |
| CCACHE_DIR=/host/${{ env.ccache_dir }} | |
| CCACHE_BASEDIR=/host/${{ env.ccache_basedir }} | |
| CMAKE_ARGS="-DUSE_PYBIND11_SOURCE=ON -DBLA_VENDOR=OpenBLAS -DUSE_BUILTIN_SENTENCEPIECE=ON -DWITH_GEMMOLOGY=ON -DUSE_AVX512=ON -DUSE_AVX2=ON -DUSE_SSSE3=ON -DUSE_SSE2=ON" | |
| CIBW_ENVIRONMENT_MACOS: | |
| USE_CCACHE=1 | |
| CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} | |
| CCACHE_COMPRESS=${{ env.ccache_compress }} | |
| CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} | |
| CCACHE_MAXSIZE=${{ env.ccache_maxsize }} | |
| PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} | |
| CCACHE_DIR=${{ env.ccache_dir }} | |
| CCACHE_BASEDIR=${{ env.ccache_basedir }} | |
| MACOSX_DEPLOYMENT_TARGET=10.9 | |
| CIBW_ENVIRONMENT_WINDOWS: | |
| PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} | |
| MKLROOT='${{ env.WIN_MKL_ROOT }}' | |
| VCPKG_BUILD_TYPE=release | |
| VCPKG_DEFAULT_TRIPLET=x64-windows-static | |
| CMAKE_ARGS='-DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DCMAKE_CXX_COMPILER_LAUNCHER=${{github.workspace}}\ccache.exe -DCMAKE_C_COMPILER_LAUNCHER=${{github.workspace}}\ccache.exe -DUSE_STATIC_LIBS=ON' | |
| CIBW_BEFORE_BUILD_LINUX: | | |
| yum install -y ccache wget | |
| yum install -y gdb | |
| # Install Intel MKL. | |
| # yum-config-manager -y --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo | |
| # yum install -y intel-mkl | |
| XSIMD_VERSION="11.1.0" | |
| wget https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz -O xsimd.tar.gz | |
| tar xf xsimd.tar.gz | |
| (cd xsimd-${XSIMD_VERSION} && cmake -DCMAKE_INSTALL_PREFIX=/usr/ -B build -S . && cmake --build build --target all && make -C build install) | |
| # Install OpenBLAS | |
| yum install -y openblas-devel atlas-devel | |
| # Install CBLAS | |
| # yum install -y atlas-devel blas-devel | |
| chmod -R a+rwx /host/${{ env.ccache_dir }} | |
| lscpu | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| CIBW_BEFORE_BUILD_MACOS: | | |
| brew install openblas protobuf ccache boost pybind11 | |
| chmod -R a+rwx ${{ env.ccache_dir }} | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| CIBW_BEFORE_BUILD_WINDOWS: | | |
| ${{github.workspace}}\ccache.exe -sv # Print current cache stats | |
| ${{github.workspace}}\ccache.exe -z # Print current cache stats | |
| CIBW_BUILD: "cp{37,38,39,310,311}-*manylinux_x86_64 cp{37,38,39,310,311}-macosx_x86_64 cp3{8,9,10}-win_amd64 cp*macosx_arm64" | |
| CIBW_BEFORE_TEST_LINUX: | | |
| ccache -s # Print current ccache stats | |
| CIBW_BEFORE_TEST_MACOS: | | |
| ccache -s # Print current ccache stats | |
| CIBW_BEFORE_TEST_WINDOWS: | | |
| ${{github.workspace}}\\ccache.exe -sv # Print current cache stats | |
| CIBW_TEST_EXTRAS: test | |
| CIBW_TEST_COMMAND: | | |
| python -m pytest --pyargs slimt -s | |
| # gdb -return-child-result -batch -ex 'set follow-fork-mode child' -ex 'run' -ex 'bt' -ex 'quit' --args python -m pytest --pyargs slimt -s | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| release-latest: | |
| name: "Release Latest Build" | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| permissions: | |
| contents: "write" | |
| packages: "write" | |
| pull-requests: "read" | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| # Leave the below be, it will be useful. | |
| - name: List downloaded assets | |
| run: | | |
| find ./ | |
| - name: Update GitHub prerelease | |
| uses: marvinpinto/action-automatic-releases@latest | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| automatic_release_tag: latest | |
| prerelease: true | |
| title: "Latest Build" | |
| files: | | |
| wheels/*.whl | |
| release-version: | |
| name: Release version | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| permissions: | |
| contents: "write" | |
| packages: "write" | |
| pull-requests: "read" | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| # Leave the below be, it will be useful. | |
| - name: List downloaded assets | |
| run: | | |
| find ./ | |
| - name: Update GitHub release | |
| uses: marvinpinto/action-automatic-releases@latest | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| prerelease: false | |
| title: "${{ github.ref_name }}" | |
| files: | | |
| wheels/*.whl | |
| upload-wheels: | |
| name: "Upload wheels to PyPI" | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| needs: [build-wheels] | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: wheels | |
| - name: Publish wheels to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| python3 -m pip install twine | |
| twine upload *.whl | |
| build-wasm: | |
| name: "emscripten" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Set ccache environment for emcc | |
| run: | | |
| # We are hardcoding this to mtime instead of env pickup. Rest use content. | |
| echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV | |
| echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV | |
| echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV | |
| echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV | |
| echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV | |
| echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV | |
| # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper | |
| echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV | |
| # This need to be run before setup, so ccache build caching doesn't complain. | |
| - name: Obtain emsdk sources | |
| run: | | |
| git clone --depth 1 https://github.com/emscripten-core/emsdk.git | |
| - name: Cache-op for build-cache through ccache | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ${{ env.ccache_dir }} | |
| ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/ | |
| key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} | |
| restore-keys: |- | |
| ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} | |
| ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }} | |
| ccache-${{ github.job }}-${{ env.emsdk_version }} | |
| - name: Setup Emscripten toolchain | |
| run: | | |
| (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit) | |
| (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit) | |
| # mtime of this file is checked by ccache, we set it to avoid cache misses. | |
| touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten | |
| # These needs to be done in the activated shell. | |
| eval $(./emsdk/emsdk construct_env \ | |
| | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \ | |
| | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' ); | |
| # This looks more permanent than version pinned, so keeping temporarily to avoid failures. | |
| echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH | |
| - name: Generate ccache_vars for ccache based on machine | |
| shell: bash | |
| id: ccache_vars | |
| run: |- | |
| echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" | |
| echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" | |
| - name: Verify Emscripten setup | |
| run: | | |
| emcc --version | |
| emcmake cmake --version | |
| emmake make --version | |
| - name: ccache prolog | |
| run: |- | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| - name: "Configure builds" | |
| run: | | |
| mkdir -p build-wasm | |
| cd build-wasm | |
| emcmake cmake -DWITH_BLAS=OFF -DWITH_RUY=ON -DWITH_GEMMOLOGY=OFF -DSLIMT_USE_INTERNAL_PCRE2=ON -DBUILD_WASM=ON .. | |
| - name: "Compile" | |
| working-directory: build-wasm | |
| run: | | |
| emmake make -j2 | |
| - name: ccache epilog | |
| run: | | |
| ccache -s # Print current cache stats | |
| # The following is useless as it works within Firefox, that too only in a | |
| # privileged context. Might choose to enable should Mozilla be | |
| # interested. | |
| # | |
| # - name: Import GEMM library from a separate wasm module | |
| # working-directory: build-wasm | |
| # run: bash ../wasm/patch-artifacts-import-gemm-module.sh | |
| # Setup nodejs-18, as nodejs-14 provided by emsdk fails when running | |
| # and newer version of node allows us to use fetch(). | |
| - name: Setup nodejs | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| # - name: Test run | |
| # working-directory: wasm | |
| # run: | | |
| # cp ../build-wasm/bergamot-translator-worker.{js,wasm} ./ | |
| # npm install jsdom | |
| # # --unhandled-rejections make the script exit with a non-zero code (at least on node-14). | |
| # # So leaving this here. | |
| # node --unhandled-rejections=strict node-test.js | |
| # # Upload both together. | |
| # - name: Upload wasm artifact | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: wasm-artefacts | |
| # if-no-files-found: error | |
| # path: | | |
| # ${{github.workspace}}/build-wasm/bergamot-translator-worker.js | |
| # ${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm | |
| # ${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak |