diff --git a/.circleci/config.yml b/.circleci/config.yml index 140e3116d..52d58fc09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,52 +1,6 @@ version: 2.1 jobs: - build-with-wormhole: - docker: - - image: 'emscripten/emsdk:3.1.8' - resource_class: medium - - working_directory: ~/checkout - - steps: - - checkout - - - run: - name: Build WASM WORMHOLE - command: | - bash build-wasm.sh WORMHOLE - - - run: - name: Check artifacts - working_directory: build-wasm - command: | - ARTIFACT_BASE="bergamot-translator-worker" - ARTIFACT_SUFFIX="with-wormhole" - ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX - - if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then - echo "Artifacts Successfully Generated" - mkdir ../artifacts - cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm - cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js - cd ../artifacts - shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX - ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX - cp ../BERGAMOT_VERSION . - else - echo "Failure: Artifacts Not Present" - exit 1 - fi - - - persist_to_workspace: - root: . - paths: - - artifacts/* - - - store_artifacts: - path: "artifacts" - destination: "wasm-wormhole" - - build-without-wormhole: + build: docker: - image: 'emscripten/emsdk:3.1.8' resource_class: medium @@ -66,8 +20,7 @@ jobs: working_directory: build-wasm command: | ARTIFACT_BASE="bergamot-translator-worker" - ARTIFACT_SUFFIX="without-wormhole" - ARTIFACT_FINAL=$ARTIFACT_BASE-$ARTIFACT_SUFFIX + ARTIFACT_FINAL=$ARTIFACT_BASE if [[ -f "$ARTIFACT_BASE.js" && -f "$ARTIFACT_BASE.wasm" ]]; then echo "Artifacts Successfully Generated" @@ -75,8 +28,8 @@ jobs: cp $ARTIFACT_BASE.wasm ../artifacts/$ARTIFACT_FINAL.wasm cp $ARTIFACT_BASE.js ../artifacts/$ARTIFACT_FINAL.js cd ../artifacts - shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX - ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize-$ARTIFACT_SUFFIX + shasum -a 256 $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize + ls -lsa $ARTIFACT_FINAL.wasm $ARTIFACT_FINAL.js >> sha256-filesize else echo "Failure: Artifacts Not Present" exit 1 @@ -89,7 +42,7 @@ jobs: - store_artifacts: path: "artifacts" - destination: "wasm-without-wormhole" + destination: "wasm" publish_to_github: docker: @@ -106,18 +59,13 @@ jobs: name: "Publish Release on GitHub" command: | export TAG_VERSION=$(cat ./artifacts/BERGAMOT_VERSION) - cat ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole >> ./artifacts/sha256-filesize - rm ./artifacts/sha256-filesize-without-wormhole ./artifacts/sha256-filesize-with-wormhole ./artifacts/BERGAMOT_VERSION + rm ./artifacts/BERGAMOT_VERSION ghr -t ${GHTOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${TAG_VERSION} ./artifacts/ workflows: build: jobs: - - build-with-wormhole: - filters: - tags: - only: /^v.*/ - - build-without-wormhole: + - build: filters: tags: only: /^v.*/ @@ -128,7 +76,6 @@ workflows: branches: ignore: /.*/ requires: - - build-without-wormhole - - build-with-wormhole + - build diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml new file mode 100644 index 000000000..2ee14548d --- /dev/null +++ b/.github/workflows/arm.yml @@ -0,0 +1,139 @@ +name: ARM +'on': + push: + branches: + - main + - ci-sandbox + pull_request: + branches: + - '**' +env: + ccache_basedir: ${{ github.workspace }} + ccache_dir: "${{ github.workspace }}/.ccache" + ccache_compilercheck: content + ccache_compress: 'true' + ccache_compresslevel: 9 + ccache_maxsize: 200M + ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache + ndk: "${{ github.workspace }}/android-ndk-r23b" + abi: "arm64-v8a" + minsdk_version : 28 + android_platform: 28 + +jobs: + ubuntu: + name: "arm-v8a cross-compile via Android NDK" + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install prerequisites + run: | + wget -c --quiet https://dl.google.com/android/repository/android-ndk-r23b-linux.zip + unzip -qq android-ndk-r23b-linux.zip + sudo apt-get -y install ccache cmake + + - name: Generate ccache_vars for ccache based on machine + shell: bash + id: ccache_vars + run: |- + echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" + echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" + + - name: Cache-op for build-cache through ccache + uses: actions/cache@v2 + with: + path: ${{ env.ccache_dir }} + key: ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} + restore-keys: |- + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} + ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }} + ccache-${{ matrix.identifier }} + + - name: ccache environment setup + run: |- + echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV + echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV + + - name: ccache prolog + run: |- + ccache -s # Print current cache stats + ccache -z # Zero cache entry + + - name: Generate buildfiles for bergamot-translator on android via cmake + run: |- + mkdir -p build + cd build + NDK=${{ env.ndk }} + ABI=${{ env.abi }} + MINSDK_VERSION=${{ env.minsdk_version }} + ANDROID_PLATFORM=android-${{ env.android_platform }} + OTHER_ANDROID_ARGS=( + -DANDROID_ARM_NEON=TRUE + ) + OTHER_MARIAN_ARGS=( + -DCOMPILE_CUDA=off + -DCOMPILE_CPU=on + -DCMAKE_HAVE_THREADS_LIBRARY=1 + -DCMAKE_USE_WIN32_THREADS_INIT=0 + -DCMAKE_USE_PTHREADS_INIT=1 + -DTHREADS_PREFER_PTHREAD_FLAG=ON + -DBUILD_ARCH=armv8-a + # -DCOMPILE_WITHOUT_EXCEPTIONS=on # Apparently this can reduce the binary size, let's see. + -DSSPLIT_USE_INTERNAL_PCRE2=ON + ) + # Additionally list variables finally configured. + cmake -L \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \ + -DANDROID_TOOLCHAIN=clang \ + -DANDROID_ABI=$ABI \ + -DANDROID_PLATFORM=$ANDROID_PLATFORM \ + -DANDROID_NATIVE_API_LEVEL=$MINSDKVERSION \ + -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi-4.8 \ + -DANDROID_STL=c++_static \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + "${OTHER_ANDROID_ARGS[@]}" "${OTHER_MARIAN_ARGS[@]}" \ + .. + + + - name : Build bergamot-translator for android + working-directory: build + run: |- + make -j2 + + - name: ccache epilog + run: 'ccache -s # Print current cache stats' + + - uses: actions/upload-artifact@v2 + with: + path: ${{github.workspace}}/build/app/bergamot + + + # Disable release for now. + # release: + # name: Release Latest Build + # runs-on: ubuntu-latest + # needs: [ubuntu] + # if: github.ref == 'refs/heads/master' + # steps: + # - name: Download artifacts + # uses: actions/download-artifact@v2 + # + # - name: Update GitHub prerelease + # uses: marvinpinto/action-automatic-releases@latest + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # automatic_release_tag: latest + # prerelease: true + # title: "Latest Build" + # files: | + # artifact/marian-decoder diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da3c37018..830924c2c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,215 +21,143 @@ env: ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache jobs: - python-ubuntu: + build-wheels: strategy: - fail-fast: false matrix: - include: - - name: "Ubuntu 18.04 / py3.6" - os: "ubuntu-18.04" - python-version: "3.6" - - name: "Ubuntu 18.04 / py3.7" - os: "ubuntu-18.04" - python-version: "3.7" - - name: "Ubuntu 20.04 / py3.8" - os: "ubuntu-20.04" - python-version: "3.8" - - name: "Ubuntu 20.04 / py3.9" - os: "ubuntu-20.04" - python-version: "3.9" - - name: "Ubuntu 20.04 / py3.10" - os: "ubuntu-20.04" - python-version: "3.10" - - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - - name: Install Dependencies - run: |- - sudo apt-get update - sudo apt-get install -y \ - ccache libprotobuf-dev protobuf-compiler \ - python3-setuptools python3-pybind11 - - - name: Install MKL - run: |- - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" - sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 - - - name: Generate ccache_vars for ccache based on machine - shell: bash - id: ccache_vars - run: |- - echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" - echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" - - - name: Cache-op for build-cache through ccache - uses: actions/cache@v2 - with: - path: ${{ env.ccache_dir }} - key: ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} - restore-keys: |- - ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} - ccache-${{ matrix.name }}-${{ steps.ccache_vars.outputs.hash }} - ccache-${{ matrix.name }} - - name: ccache environment setup - run: |- - echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV - - - name: ccache prolog - run: |- - ccache -s # Print current cache stats - ccache -z # Zero cache entry - - - name: Inject local version identifier for non tag builds - if: ${{ !startsWith(github.ref, 'refs/tags/v') }} - run: |- - echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + os: [ubuntu-latest, macos-latest] + fail-fast: false - - name: setup.py - run: |- - python3 -m pip install wheel - BUILD_ARCH=core-avx-i python3 setup.py bdist_wheel --universal + name: "cibuildwheel / ${{ matrix.os }}" + runs-on: ${{ matrix.os }} - # We're happy with just compile for the moment, so cache gets some seeding. - - name: Install onto root python lib - run: |- - python3 -m pip install --ignore-installed dist/bergamot-*.whl + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive - - name: Fetch models from translateLocally repository. - run: |- - python3 -m bergamot download -m en-de-tiny - python3 -m bergamot download -m de-en-tiny - python3 -m bergamot ls + - name: Generate ccache_vars for ccache based on machine + shell: bash + id: ccache_vars + run: |- + echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" + echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" - - name: Fetch models from opus repository. - run: |- - python3 -m bergamot download -m eng-fin-tiny -r opus - python3 -m bergamot ls -r opus + - name: Cache-op for build-cache through ccache + uses: actions/cache@v2 + with: + path: ${{ env.ccache_dir }} + key: ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} + restore-keys: |- + ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} + ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }} + ccache-cibuildwheel-${{ matrix.os }} - - name: Run the sample python script shipped with module - run: |- - python3 -m bergamot translate --model en-de-tiny <<< "Hello World" - python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World" - python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World" + - name: ccache environment setup + run: |- + mkdir -p ${{ env.ccache_dir }} - - name: ccache epilog - run: 'ccache -s # Print current cache stats' + - name: Inject local version identifier for non tag builds + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + run: |- + echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 - with: - path: ${{github.workspace}}/dist/bergamot-*.whl + - name: Apply MacOS patch + if: ${{ startsWith(runner.os, 'mac') }} + run: | + patch -p1 < patches/01-marian-fstream-for-macos.patch + - name: Build wheels + uses: pypa/cibuildwheel@v2.6.1 + # to supply options, put them in 'env', like: + env: + CIBW_ENVIRONMENT_LINUX: + BUILD_ARCH=core-avx-i + USE_CCACHE=1 + CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} + CCACHE_COMPRESS=${{ env.ccache_compress }} + CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} + CCACHE_MAXSIZE=${{ env.ccache_maxsize }} + PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} + CCACHE_DIR=/host/${{ env.ccache_dir }} + CCACHE_BASEDIR=/host/${{ env.ccache_basedir }} + + CIBW_ENVIRONMENT_MACOS: + BUILD_ARCH=core-avx-i + USE_CCACHE=1 + CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} + CCACHE_COMPRESS=${{ env.ccache_compress }} + CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} + CCACHE_MAXSIZE=${{ env.ccache_maxsize }} + PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} + CCACHE_DIR=${{ env.ccache_dir }} + CCACHE_BASEDIR=${{ env.ccache_basedir }} + MACOSX_DEPLOYMENT_TARGET=10.9 + + CIBW_BEFORE_BUILD_LINUX: | + yum install -y ccache + + # Install Intel MKL. + yum-config-manager -y --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo + yum install -y intel-mkl + + chmod -R a+rwx /host/${{ env.ccache_dir }} + + ccache -s # Print current cache stats + ccache -z # Zero cache entry + + CIBW_BEFORE_BUILD_MACOS: | + brew install openblas protobuf ccache boost pybind11 + chmod -R a+rwx ${{ env.ccache_dir }} + ccache -s # Print current cache stats + ccache -z # Zero cache entry + + CIBW_BUILD: "cp{36,37,38,39,310}-*manylinux_x86_64 cp{36,37,38,39,310}-macosx_x86_64" + + CIBW_BEFORE_TEST: | + ccache -s # Print current ccache stats + + CIBW_TEST_COMMAND: | + # The wheels are installed automatically and available. + + # Fetch models from translateLocally repository. + python3 -m bergamot download -m en-de-tiny + python3 -m bergamot download -m de-en-tiny + python3 -m bergamot ls + + # Fetch models from opus repository. + python3 -m bergamot download -m eng-fin-tiny -r opus + python3 -m bergamot ls -r opus + + # Run the sample python script shipped with module + python3 -m bergamot translate --model en-de-tiny <<< "Hello World" + python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World" + python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World" + + + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/*.whl - python-macos: - name: "MacOS 10.15 / py3.10" - runs-on: "macos-10.15" + upload-wheels: + name: "Upload wheels to PyPI" + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + needs: [build-wheels] steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Download artifacts + uses: actions/download-artifact@v2 with: - submodules: recursive - - name: Install Dependencies - run: |- - brew update - brew install openblas protobuf ccache boost pybind11 - brew install coreutils findutils libarchive + name: wheels - - name: Generate ccache_vars for ccache based on machine - shell: bash - id: ccache_vars - run: |- - echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" - echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" - - name: Cache-op for build-cache through ccache - uses: actions/cache@v2 - with: - path: ${{ env.ccache_dir }} - key: ccache-${{ job.id }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} - restore-keys: |- - ccache-${{ job.id }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} - ccache-${{ job.id }}-${{ steps.ccache_vars.outputs.hash }} - ccache-${{ job.id }} - - - name: ccache environment setup - run: |- - echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV - echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV - - - name: ccache prolog - run: |- - ccache -s # Print current cache stats - ccache -z # Zero cache entry - - - name: Apply required patches - run: |- - patch -p1 < patches/01-marian-fstream-for-macos.patch - - # Appears to be required per GitHub CI; - - name: Set MACOSX DEPLOYMENT TARGET via environment variable - run: |- - echo "MACOSX_DEPLOYMENT_TARGET=10.15" >> $GITHUB_ENV - - - name: Inject local version identifier for non tag builds - if: ${{ !startsWith(github.ref, 'refs/tags/v') }} - run: |- - echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - - name: setup.py - run: |- - python3 -m pip install --upgrade packaging wheel - BUILD_ARCH=core-avx-i python3 setup.py bdist_wheel --universal - - # We're happy with just compile for the moment, so cache gets some seeding. - - name: Install onto root python lib - run: |- - python3 -m pip install dist/bergamot-*.whl - - - name: Fetch models from translateLocally repository. - run: |- - python3 -m bergamot download -m en-de-tiny - python3 -m bergamot download -m de-en-tiny - - - name: Fetch models from opus repository. - run: |- - python3 -m bergamot download -m eng-fin-tiny -r opus - python3 -m bergamot ls -r opus - - - name: Run the sample python script shipped with module - run: |- - python3 -m bergamot translate --model en-de-tiny <<< "Hello World" - python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World" - python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World" - - - name: ccache epilog - run: 'ccache -s # Print current cache stats' + - name: Publish wheels to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 -m pip install twine + twine upload *.whl - - uses: actions/upload-artifact@v2 - with: - path: ${{github.workspace}}/dist/bergamot-*.whl build-wasm: name: "emscripten" @@ -253,7 +181,7 @@ jobs: echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV - + # This need to be run before setup, so ccache build caching doesn't complain. - name: Obtain emsdk sources run: | @@ -304,16 +232,15 @@ jobs: ccache -s # Print current cache stats ccache -z # Zero cache entry - # WORMHOLE=off - - name: "Configure builds for WORMHOLE=off" + - name: "Configure builds" run: | - mkdir -p build-wasm-without-wormhole - cd build-wasm-without-wormhole - emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off .. + mkdir -p build-wasm + cd build-wasm + emcmake cmake -DCOMPILE_WASM=on .. - - name: "Compile with WORMHOLE=off" - working-directory: build-wasm-without-wormhole + - name: "Compile" + working-directory: build-wasm run: | emmake make -j2 @@ -322,43 +249,25 @@ jobs: ccache -s # Print current cache stats - name: Import GEMM library from a separate wasm module - working-directory: build-wasm-without-wormhole + working-directory: build-wasm run: bash ../wasm/patch-artifacts-import-gemm-module.sh + # Setup nodejs-18, as nodejs-14 provided by emsdk fails when running + # and newer version of node allows us to use fetch(). + - name: Setup nodejs + uses: actions/setup-node@v3 + with: + node-version: 18 - # WORMHOLE=on - - name: "Configure builds for WORMHOLE=on" - run: | - mkdir -p build-wasm-with-wormhole - cd build-wasm-with-wormhole - emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=on .. - - - - name: "Compile with WORMHOLE=on" - working-directory: build-wasm-with-wormhole - run: | - emmake make -j2 - - - name: ccache epilog - run: | - ccache -s # Print current cache stats - - - name: Instantiate simd wormhole - working-directory: build-wasm-with-wormhole - run: bash ../wasm/patch-artifacts-enable-wormhole.sh - - - name: Import GEMM library from a separate wasm module - working-directory: build-wasm-with-wormhole - run: bash ../wasm/patch-artifacts-import-gemm-module.sh - - # Rename the wormhole on builds - - name: Rename artefacts with wormhole - working-directory: build-wasm-with-wormhole + - name: Test run + working-directory: wasm run: | - mv bergamot-translator-worker{,-with-wormhole}.js - mv bergamot-translator-worker{,-with-wormhole}.js.bak - mv bergamot-translator-worker{,-with-wormhole}.wasm + cp ../build-wasm/bergamot-translator-worker.{js,wasm} ./ + npm install jsdom + # --unhandled-rejections make the script exit with a non-zero code (at least on node-14). + # So leaving this here. + node --unhandled-rejections=strict node-test.js # Upload both together. - name: Upload wasm artifact @@ -367,25 +276,49 @@ jobs: name: wasm-artefacts if-no-files-found: error path: | - # Without wormhole - ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js - ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.wasm - ${{github.workspace}}/build-wasm-without-wormhole/bergamot-translator-worker.js.bak + ${{github.workspace}}/build-wasm/bergamot-translator-worker.js + ${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm + ${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak + + + upload-wasm: + name: "Upload node package to NPM" + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + needs: [build-wasm] + steps: + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: wasm-artefacts + path: wasm/module/worker + + - uses: actions/setup-node@v3 + with: + node-version: '18.x' + registry-url: 'https://registry.npmjs.org' + - run: npm ci + - run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js - ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm - ${{github.workspace}}/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js.bak # Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model release-latest: name: Release Latest Build runs-on: ubuntu-latest - needs: [python-ubuntu, python-macos, build-wasm] + needs: [build-wheels, build-wasm] if: github.ref == 'refs/heads/main' steps: - name: Download artifacts uses: actions/download-artifact@v2 - + + # Leave the below be, it will be useful. + - name: List downloaded assets + run: | + find ./ + - name: Update GitHub prerelease uses: marvinpinto/action-automatic-releases@latest with: @@ -394,16 +327,14 @@ jobs: prerelease: true title: "Latest Build" files: | - artifact/*.whl - wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js - wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm - wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js - wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm - + wheels/*.whl + wasm-artefacts/bergamot-translator-worker.js + wasm-artefacts/bergamot-translator-worker.wasm + release-version: - name: Release version + name: Release version runs-on: ubuntu-latest - needs: [python-ubuntu, python-macos, build-wasm] + needs: [build-wheels, build-wasm] permissions: contents: "write" packages: "write" @@ -412,7 +343,12 @@ jobs: steps: - name: Download artifacts uses: actions/download-artifact@v2 - + + # Leave the below be, it will be useful. + - name: List downloaded assets + run: | + find ./ + - name: Update GitHub release uses: marvinpinto/action-automatic-releases@latest with: @@ -421,13 +357,11 @@ jobs: prerelease: false title: "${{ github.ref_name }}" files: | - artifact/*.whl - wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.js - wasm-artefacts/build-wasm-without-wormhole/bergamot-translator-worker.wasm - wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.js - wasm-artefacts/build-wasm-with-wormhole/bergamot-translator-worker-with-wormhole.wasm + wheels/*.whl + wasm-artefacts/bergamot-translator-worker.js + wasm-artefacts/bergamot-translator-worker.wasm + - python-checks: name: "formatting and typechecks" runs-on: "ubuntu-latest" @@ -441,15 +375,15 @@ jobs: python3 -m pip install black isort pytype - name: "Formatting checks: black, isort" run: | - python3 -m black --check bindings/python/ setup.py doc/conf.py + python3 -m black --diff --check bindings/python/ setup.py doc/conf.py python3 -m isort --profile black --diff --check bindings/python setup.py doc/conf.py - name: "Static typing checks: pytype" run: |- python3 -m pytype bindings/python docs: - runs-on: ubuntu-18.04 - needs: [python-ubuntu] + runs-on: ubuntu-latest + needs: [build-wheels] steps: - name: Checkout uses: actions/checkout@v2 @@ -478,8 +412,8 @@ jobs: # Patches the BERGAMOT_VERSION file used by sphinx-docs at run time to # obtain names like 'main' or 'ci-sandbox' to not confuse with version # based documentation built separately. - - name: Deploy-time patch version - run: | + - name: Deploy-time patch version + run: | echo ${{steps.tag.outputs.result }} > BERGAMOT_VERSION - name: Set up Doxygen @@ -502,7 +436,7 @@ jobs: working-directory: ./doc run: | python3 -m pip install -r requirements.txt - python3 -m pip install ${{github.workspace}}/artifact/bergamot-*-cp37*.whl + python3 -m pip install --find-links=${{github.workspace}}/wheels bergamot - name: Build documentation working-directory: ./doc @@ -513,16 +447,16 @@ jobs: uses: JamesIves/github-pages-deploy-action@4.1.3 if: ${{ github.event_name == 'push' && github.repository == 'browsermt/bergamot-translator' }} with: - repository-name: 'browsermt/docs' + repository-name: 'browsermt/docs' branch: gh-pages # The branch the action should deploy to. folder: './doc/build/' # The folder the action should deploy. - target-folder: '${{ steps.tag.outputs.result }}' + target-folder: '${{ steps.tag.outputs.result }}' ssh-key: ${{ secrets.BERGAMOT_SSH_PRIVATE_KEY }} # This artifact contains the HTML output of Sphinx only. # With index.html at the root of the produced zip file. # For use for maintainers to download the zip and check render of - # documentation while generated at pull-request. + # documentation while generated at pull-request. - name: Upload documentation uses: actions/upload-artifact@v2 if: ${{ github.event_name == 'pull_request'}} @@ -530,4 +464,3 @@ jobs: name: api-docs path: ./doc/build/ if-no-files-found: error - diff --git a/.github/workflows/coding-styles.yml b/.github/workflows/coding-styles.yml index 81bdf3361..b13345601 100644 --- a/.github/workflows/coding-styles.yml +++ b/.github/workflows/coding-styles.yml @@ -1,8 +1,6 @@ name: "Coding Style" -env: - clang_version: 10 -on: +on: push: branches: [ main, ci-sandbox ] pull_request: @@ -20,9 +18,9 @@ jobs: - name: Install dependencies run: | - sudo apt-get update + sudo apt-get update sudo apt-get install -y build-essential cmake - sudo apt-get install -y clang-format clang-tidy-${{ env.clang_version }} + sudo apt-get install -y clang-format clang-tidy - name: Run clang-format run: @@ -32,13 +30,13 @@ jobs: - name: Prepare build, compilation database etc. run: | mkdir -p build - cd build + cd build cmake \ -DUSE_WASM_COMPATIBLE_SOURCE=off -DCMAKE_EXPORT_COMPILE_COMMANDS=on \ - -DCMAKE_C_COMPILER=clang-${{ env.clang_version }} -DCMAKE_CXX_COMPILER=clang++-${{ env.clang_version }} \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ .. - name: Run clang-tidy run: | - run-clang-tidy-${{ env.clang_version }} -p build "$PWD/src/.*" - run-clang-tidy-${{ env.clang_version }} -p build "$PWD/app/.*" + run-clang-tidy -p build "$PWD/src/.*" + run-clang-tidy -p build "$PWD/app/.*" diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 8ee8c5c5f..505381cbc 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -21,15 +21,15 @@ jobs: fail-fast: false matrix: include: - - name: Ubuntu 18.04 full - os: ubuntu-18.04 - identifier: ubuntu_1804_full + - name: Ubuntu 22.04 full + os: ubuntu-22.04 + identifier: ubuntu_2204_full cmake: -DCOMPILE_TESTS=on brt_tags: "" unittests: 'true' - - name: Ubuntu 18.04 minimal - os: ubuntu-18.04 - identifier: ubuntu_1804_minimal + - name: Ubuntu 22.04 minimal + os: ubuntu-22.04 + identifier: ubuntu_2204_minimal cmake: -DCOMPILE_TESTS=on -DUSE_WASM_COMPATIBLE_SOURCE=on brt_tags: "'#wasm'" unittests: 'false' @@ -55,12 +55,10 @@ jobs: - name: Install Dependencies run: |- sudo apt-get update - sudo apt-get install -y \ - libgoogle-perftools-dev libprotobuf-dev protobuf-compiler \ - libboost-all-dev ccache + sudo apt-get install -y libprotobuf-dev protobuf-compiler libboost-all-dev ccache libunwind-dev libgoogle-perftools-dev - name: Install MKL run: |- - wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - + wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB" | sudo apt-key add - sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 @@ -140,15 +138,15 @@ jobs: fail-fast: false matrix: include: - - name: MacOS 10.15 full - os: macos-10.15 - identifier: mac_1015_full + - name: MacOS 12 full + os: macos-12 + identifier: mac_12_full cmake: -DCOMPILE_TESTS=on -DUSE_APPLE_ACCELERATE=off -DUSE_FBGEMM=off -DUSE_STATIC_LIBS=off brt_tags: "" unittests: 'true' - - name: MacOS 10.15 minimal - os: macos-10.15 - identifier: mac_1015_minimal + - name: MacOS 12 minimal + os: macos-12 + identifier: mac_12_minimal cmake: -DCOMPILE_TESTS=on -DUSE_APPLE_ACCELERATE=off -DUSE_FBGEMM=off -DUSE_STATIC_LIBS=on -DUSE_WASM_COMPATIBLE_SOURCE=on brt_tags: "'#wasm'" unittests: 'false' diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 569020452..a0ff86b84 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -7,7 +7,7 @@ on: branches: [ '**' ] env: - MKL_URL: "https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip" + MKL_URL: "https://data.statmt.org/romang/marian-regression-tests/ci/mkl-2020.1-windows-static.zip" CCACHE_BASEDIR: "${{ github.workspace }}" CCACHE_DIR: "${{ github.workspace }}\\ccache" CCACHE_COMPILERCHECK: content diff --git a/.gitignore b/.gitignore index 64c1aa3a7..94b32949c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,13 @@ _deps wasm/test_page/node_modules -build-wasm +/build +/build-native +/build-wasm +/emsdk models -wasm/test_page/js/bergamot-translator-worker.* +wasm/module/worker/bergamot-translator-worker.* +wasm/module/browsermt-bergamot-translator-*.tgz # VSCode .vscode diff --git a/3rd_party/CMakeLists.txt b/3rd_party/CMakeLists.txt index 1888d6da6..eac898eb9 100644 --- a/3rd_party/CMakeLists.txt +++ b/3rd_party/CMakeLists.txt @@ -19,6 +19,12 @@ target_include_directories(marian PUBLIC ${INCDIRS}) get_property(INCLUDE_DIRECTORIES DIRECTORY ssplit-cpp/src PROPERTY INCLUDE_DIRECTORIES) target_include_directories(ssplit PUBLIC ${INCLUDE_DIRECTORIES}) +get_property(COMPILE_DEFINITIONS DIRECTORY marian-dev PROPERTY COMPILE_DEFINITIONS) +target_compile_definitions(marian PUBLIC ${COMPILE_DEFINITIONS}) + +get_property(COMPILE_OPTIONS DIRECTORY marian-dev PROPERTY COMPILE_OPTIONS) +target_compile_options(marian PUBLIC ${COMPILE_OPTIONS}) + # Compilation flags get_directory_property(CMAKE_C_FLAGS DIRECTORY marian-dev DEFINITION CMAKE_C_FLAGS) get_directory_property(CMAKE_CXX_FLAGS DIRECTORY marian-dev DEFINITION CMAKE_CXX_FLAGS) diff --git a/3rd_party/marian-dev b/3rd_party/marian-dev index 199201eb8..ecda59e61 160000 --- a/3rd_party/marian-dev +++ b/3rd_party/marian-dev @@ -1 +1 @@ -Subproject commit 199201eb89b2941afdadb14164e936d412f897ad +Subproject commit ecda59e6105fb1d7935892c3bacfbc9562b235f1 diff --git a/3rd_party/ssplit-cpp b/3rd_party/ssplit-cpp index 49fde6df7..a311f9865 160000 --- a/3rd_party/ssplit-cpp +++ b/3rd_party/ssplit-cpp @@ -1 +1 @@ -Subproject commit 49fde6df7ee9199aedb9571be800448192e3515c +Subproject commit a311f9865ade34db1e8e080e6cc146f55dafb067 diff --git a/BERGAMOT_VERSION b/BERGAMOT_VERSION index 79b0815e6..a423f7f06 100644 --- a/BERGAMOT_VERSION +++ b/BERGAMOT_VERSION @@ -1 +1 @@ -v0.4.4 +v0.4.5 diff --git a/CMakeLists.txt b/CMakeLists.txt index dc51acf80..d8a2d00cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,10 @@ project(bergamot_translator CXX C) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +# Generate a compile_commands.json in the build directory. The compile commands allow +# code editors to understand the build process and provide static analysis of the code. +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # Note that with CMake MSVC build, the option CMAKE_BUILD_TYPE is automatically derived from the key # 'configurationType' in CMakeSettings.json configurations if(NOT CMAKE_BUILD_TYPE) @@ -113,7 +117,6 @@ message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}") if(COMPILE_WASM) # See https://github.com/emscripten-core/emscripten/blob/main/src/settings.js - set(WORMHOLE ON CACHE BOOL "Use WASM wormhole in intgemm https://bugzilla.mozilla.org/show_bug.cgi?id=1672160") list(APPEND WASM_COMPILE_FLAGS -O3 # Preserve whitespaces in JS even for release builds; this doesn't increase wasm binary size diff --git a/README.md b/README.md index b70c818ec..05c3c3d25 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,7 @@ To build a version that translates with higher speeds on Firefox Nightly browser The wasm artifacts (.js and .wasm files) will be available in the build directory ("build-wasm" in this case). - 2. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts - ```bash - bash ../wasm/patch-artifacts-enable-wormhole.sh - ``` - - 3. Patch generated artifacts to import GEMM library from a separate wasm module + 2. Patch generated artifacts to import GEMM library from a separate wasm module ```bash bash ../wasm/patch-artifacts-import-gemm-module.sh ``` @@ -57,7 +52,7 @@ To build a version that runs on all browsers (including Firefox Nightly) but tra ```bash mkdir build-wasm cd build-wasm - emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ../ + emcmake cmake -DCOMPILE_WASM=on ../ emmake make -j2 ``` @@ -80,7 +75,7 @@ git submodule update --init --recursive ### Using Native version The builds generate library that can be integrated to any project. All the public header files are specified in `src` folder.\ -A short example of how to use the APIs is provided in `app/main.cpp` file. +A short example of how to use the APIs is provided in `app/bergamot.cpp` file. ### Using WASM version diff --git a/bergamot-translator-tests b/bergamot-translator-tests index 7984d140a..a04432d79 160000 --- a/bergamot-translator-tests +++ b/bergamot-translator-tests @@ -1 +1 @@ -Subproject commit 7984d140aef00489699d0b7711fa942816224294 +Subproject commit a04432d7921bfa1dd62bc2e5cdca46b226f256de diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 70b1a2535..16e3e48d3 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,4 +1,4 @@ -find_package(Python COMPONENTS Interpreter Development REQUIRED) +find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) message("Using Python: " ${Python_EXECUTABLE}) diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 000000000..3797b7dea --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,14 @@ +# bergamot-translator + +The [Bergamot project](https://browser.mt/) adds and improves client-side +machine translation in a web browser. + +This package provides Python bindings to bergamot-translator developed as part +of the Bergamot Project and extras assorted in a package to enable further use +of the library developed for local-translation on the consumer machine. + +Bergamot is a consortium coordinated by the University of Edinburgh with +partners Charles University in Prague, the University of Sheffield, University +of Tartu, and Mozilla. + + diff --git a/bindings/python/bergamot.cpp b/bindings/python/bergamot.cpp index 5e9e830f9..2ffb2267e 100644 --- a/bindings/python/bergamot.cpp +++ b/bindings/python/bergamot.cpp @@ -116,7 +116,7 @@ class ServicePyAdapter { return responses; } - private /*functions*/: + private /*functions*/: static Service make_service(const Service::Config &config) { py::scoped_ostream_redirect outstream(std::cout, // std::ostream& py::module_::import("sys").attr("stdout") // Python output @@ -130,7 +130,7 @@ class ServicePyAdapter { return Service(config); } - private /*data*/: + private /*data*/: Service service_; }; diff --git a/bindings/python/repository.py b/bindings/python/repository.py index 7f89035c8..9667c7242 100644 --- a/bindings/python/repository.py +++ b/bindings/python/repository.py @@ -75,22 +75,38 @@ def __init__(self, name, url): os.makedirs(directory, exist_ok=True) self.models_file_path = os.path.join(self.dirs["config"], "models.json") - self.update() + self.data = self._load_data(self.models_file_path) + + # Update inverse lookup. + self.data_by_code = {} + for model in self.data["models"]: + self.data_by_code[model["code"]] = model @property def name(self) -> str: return self._name + def _load_data(self, models_file_path): + """ + Load model data from existing file. If file does not exist, download from the web. + """ + if os.path.exists(models_file_path): + # File already exists, prefer to work with this. + # A user is expected to update manually if model's already + # downloaded and setup. + with open(models_file_path) as model_file: + return json.load(model_file) + else: + # We are running for the first time. + # Try to fetch this file from the internet. + self.update() + with open(models_file_path) as model_file: + return json.load(model_file) + def update(self) -> None: inventory = requests.get(self.url).text with open(self.models_file_path, "w+") as models_file: models_file.write(inventory) - self.data = json.loads(inventory) - - # Update inverse lookup. - self.data_by_code = {} - for model in self.data["models"]: - self.data_by_code[model["code"]] = model def models(self, filter_downloaded: bool = True) -> t.List[str]: codes = [] @@ -121,7 +137,24 @@ def download(self, model_identifier: str): download_resource(model["url"], save_location) with tarfile.open(save_location) as model_archive: - model_archive.extractall(self.dirs["models"]) + + def is_within_directory(directory, target): + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + safe_extract(model_archive, self.dirs["models"]) fprefix = self._archive_name_without_extension(model["url"]) model_dir = os.path.join(self.dirs["models"], fprefix) symlink = os.path.join(self.dirs["models"], model["code"]) diff --git a/build-wasm.sh b/build-wasm.sh index ff12013d1..443907232 100755 --- a/build-wasm.sh +++ b/build-wasm.sh @@ -2,34 +2,6 @@ set -e set -x -# Usage -Usage="Build translator to wasm (with/without wormhole). - -Usage: $(basename "$0") [WORMHOLE] - - where: - WORMHOLE An optional string argument - - when specified on command line, builds wasm artifacts with wormhole - - when not specified (the default behaviour), builds wasm artifacts without wormhole." - -if [ "$#" -gt 1 ]; then - echo "Illegal number of parameters passed" - echo "$Usage" - exit -fi - -WORMHOLE=false - -if [ "$#" -eq 1 ]; then - if [ "$1" = "WORMHOLE" ]; then - WORMHOLE=true - else - echo "Illegal parameter passed" - echo "$Usage" - exit - fi -fi - # Run script from the context of the script-containing directory cd "$(dirname $0)" @@ -66,20 +38,28 @@ if [ ! -d ${BUILD_DIRECTORY} ]; then fi cd ${BUILD_DIRECTORY} -if [ "$WORMHOLE" = true ]; then - emcmake cmake -DCOMPILE_WASM=on ../ -else - emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ../ -fi +emcmake cmake -DCOMPILE_WASM=on ../ emmake make -j2 -# 2. Enable SIMD Wormhole via Wasm instantiation API in generated artifacts -if [ "$WORMHOLE" = true ]; then - bash ../wasm/patch-artifacts-enable-wormhole.sh -fi - -# 3. Import GEMM library from a separate wasm module +# 2. Import GEMM library from a separate wasm module bash ../wasm/patch-artifacts-import-gemm-module.sh +set +x +echo "" +echo "Build complete" +echo "" +echo " ./build-wasm/bergamot-translator-worker.js" +echo " ./build-wasm/bergamot-translator-worker.wasm" + +WASM_SIZE=$(wc -c bergamot-translator-worker.wasm | awk '{print $1}') +GZIP_SIZE=$(gzip -c bergamot-translator-worker.wasm | wc -c | xargs) # xargs trims the whitespace + +# Convert it to human readable. +WASM_SIZE="$(awk 'BEGIN {printf "%.2f",'$WASM_SIZE'/1048576}')M ($WASM_SIZE bytes)" +GZIP_SIZE="$(awk 'BEGIN {printf "%.2f",'$GZIP_SIZE'/1048576}')M ($GZIP_SIZE bytes)" + +echo " Uncompressed wasm size: $WASM_SIZE" +echo " Compressed wasm size: $GZIP_SIZE" + # The artifacts (.js and .wasm files) will be available in the build directory exit 0 diff --git a/setup.py b/setup.py index 85fa685ff..ed4c6dc81 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ "win-arm64": "ARM64", } + # A CMakeExtension needs a sourcedir instead of a file list. # The name must be the _single_ output extension from the CMake build. # If you need multiple extensions, see scikit-build. @@ -48,12 +49,11 @@ def build_extension(self, ext): f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm - f"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", - f"-DCMAKE_C_COMPILER_LAUNCHER=ccache", f"-DCOMPILE_PYTHON=ON", f"-DSSPLIT_USE_INTERNAL_PCRE2=ON", f"-DBUILD_ARCH={build_arch}", ] + build_args = ["-t", "_bergamot"] # Adding CMake arguments set as environment variable # (needed e.g. to build for ARM OSx on conda-forge) @@ -63,6 +63,13 @@ def build_extension(self, ext): # In this example, we pass in the version to C++. You might not need to. cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"] + use_ccache = os.environ.get("USE_CCACHE", "0") == "1" + if use_ccache: + cmake_args += [ + f"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + f"-DCMAKE_C_COMPILER_LAUNCHER=ccache", + ] + if self.compiler.compiler_type != "msvc": # Using Ninja-build since it a) is available as a wheel and b) # multithreads automatically. MSVC would require all variables be @@ -78,7 +85,6 @@ def build_extension(self, ext): pass else: - # Single config generators are handled "normally" single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) @@ -130,14 +136,15 @@ def build_extension(self, ext): # Import the README and use it as the long-description. # Note: this will only work if 'README.md' is present in your MANIFEST.in file! -with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: +long_description = "" +with io.open(os.path.join(here, "bindings/python/README.md"), encoding="utf-8") as f: long_description = "\n" + f.read() version = None with open(os.path.join(here, "BERGAMOT_VERSION")) as f: version = f.read().strip() suffix = os.environ.get("PYTHON_LOCAL_VERSION_IDENTIFIER", None) - if suffix is not None: + if suffix: version = "{}+{}".format(version, suffix) @@ -191,8 +198,9 @@ def run(self): author="Jerin Philip", author_email="jerinphilip@live.in", url="https://github.com/browsermt/bergamot-translator/", - description="Bergamot translator python binding.", - long_description="", + description="Translate text-content locally in your machine across langauges.", + long_description=long_description, + long_description_content_type="text/markdown", ext_modules=[CMakeExtension("bergamot/_bergamot")], cmdclass={"build_py": build_py, "build_ext": CMakeBuild}, zip_safe=False, @@ -201,10 +209,40 @@ def run(self): python_requires=">=3.6", packages=["bergamot"], package_dir={"bergamot": "bindings/python"}, - install_requires=["requests", "pyyaml", "appdirs"], + install_requires=["requests", "pyyaml>=5.1", "appdirs"], entry_points={ "console_scripts": [ "bergamot = bergamot.__main__:main", ], }, + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + classifiers=[ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + "Development Status :: 3 - Alpha", + # Indicate who your project is intended for + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + # Pick your license as you wish + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + # Specify the Python versions you support here. In particular, ensure + # that you indicate you support Python 3. These classifiers are *not* + # checked by 'pip install'. See instead 'python_requires' below. + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3 :: Only", + ], + project_urls={ + "Bug Reports": "https://github.com/browsermt/bergamot-translator/issues", + "Source": "https://github.com/browsermt/bergamot-translator/", + "Documentation": "https://browser.mt/docs/main/python.html", + }, ) diff --git a/src/translator/byte_array_util.cpp b/src/translator/byte_array_util.cpp index 183dea3c0..c7515e797 100644 --- a/src/translator/byte_array_util.cpp +++ b/src/translator/byte_array_util.cpp @@ -91,21 +91,24 @@ AlignedMemory loadFileToMemory(const std::string& path, size_t alignment) { return alignedMemory; } -AlignedMemory getModelMemoryFromConfig(marian::Ptr options) { +std::vector getModelMemoryFromConfig(marian::Ptr options) { auto models = options->get>("models"); - ABORT_IF(models.size() != 1, "Loading multiple binary models is not supported for now as it is not necessary."); - - // If binary model we load into aligned memory. If .npz we leave it be to - // return empty aligned memory, thus allowing traditional file system loads. - if (marian::io::isBin(models[0])) { - AlignedMemory alignedMemory = loadFileToMemory(models[0], 256); - return alignedMemory; - } else if (marian::io::isNpz(models[0])) { - return AlignedMemory(); - } else { - ABORT("Unknown extension for model: {}, should be one of `.bin` or `.npz`", models[0]); + + std::vector modelMemories(models.size()); + for (size_t i = 0; i < models.size(); ++i) { + const auto model = models[i]; + if (marian::io::isBin(model)) { + modelMemories[i] = loadFileToMemory(model, 256); + } else if (marian::io::isNpz(model)) { + // if any of the models are npz format, we revert to loading from file for all models. + LOG(debug, "Encountered an npz file {}; will use file loading for {} models", model, models.size()); + return {}; + } else { + ABORT("Unknown extension for model: {}, should be one of `.bin` or `.npz`", model); + } } - return AlignedMemory(); + + return modelMemories; } AlignedMemory getShortlistMemoryFromConfig(marian::Ptr options) { @@ -153,7 +156,7 @@ AlignedMemory getQualityEstimatorModel(MemoryBundle& memoryBundle, const marian: MemoryBundle getMemoryBundleFromConfig(marian::Ptr options) { MemoryBundle memoryBundle; - memoryBundle.model = getModelMemoryFromConfig(options); + memoryBundle.models = getModelMemoryFromConfig(options); memoryBundle.shortlist = getShortlistMemoryFromConfig(options); getVocabsMemoryFromConfig(options, memoryBundle.vocabs); memoryBundle.ssplitPrefixFile = getSsplitPrefixFileMemoryFromConfig(options); diff --git a/src/translator/byte_array_util.h b/src/translator/byte_array_util.h index b445b3dec..851a175fd 100644 --- a/src/translator/byte_array_util.h +++ b/src/translator/byte_array_util.h @@ -5,7 +5,7 @@ namespace marian { namespace bergamot { AlignedMemory loadFileToMemory(const std::string& path, size_t alignment); -AlignedMemory getModelMemoryFromConfig(marian::Ptr options); +std::vector getModelMemoryFromConfig(marian::Ptr options); AlignedMemory getQualityEstimatorModel(const marian::Ptr& options); AlignedMemory getQualityEstimatorModel(MemoryBundle& memoryBundle, const marian::Ptr& options); AlignedMemory getShortlistMemoryFromConfig(marian::Ptr options); diff --git a/src/translator/definitions.h b/src/translator/definitions.h index b3bc1019b..efba3f9f6 100644 --- a/src/translator/definitions.h +++ b/src/translator/definitions.h @@ -19,8 +19,8 @@ typedef AlignedVector AlignedMemory; /// Memory bundle for all byte-arrays. /// Can be a set/subset of model, shortlist, vocabs and ssplitPrefixFile bytes. struct MemoryBundle { - AlignedMemory model{}; ///< Byte-array of model (aligned to 256) - AlignedMemory shortlist{}; ///< Byte-array of shortlist (aligned to 64) + std::vector models{}; ///< Byte-array of model (each element is aligned to 256) + AlignedMemory shortlist{}; ///< Byte-array of shortlist (aligned to 64) /// Vector of vocabulary memories (aligned to 64). /// If two vocabularies are the same (based on the filenames), two entries (shared diff --git a/src/translator/threadsafe_batching_pool.cpp b/src/translator/threadsafe_batching_pool.cpp index 0a1a28d4e..29ad35a97 100644 --- a/src/translator/threadsafe_batching_pool.cpp +++ b/src/translator/threadsafe_batching_pool.cpp @@ -10,7 +10,7 @@ namespace bergamot { template template -ThreadsafeBatchingPool::ThreadsafeBatchingPool(Args &&... args) +ThreadsafeBatchingPool::ThreadsafeBatchingPool(Args &&...args) : backend_(std::forward(args)...), enqueued_(0), shutdown_(false) {} template @@ -20,7 +20,7 @@ ThreadsafeBatchingPool::~ThreadsafeBatchingPool() { template template -void ThreadsafeBatchingPool::enqueueRequest(Args &&... args) { +void ThreadsafeBatchingPool::enqueueRequest(Args &&...args) { std::unique_lock lock(mutex_); assert(!shutdown_); enqueued_ += backend_.enqueueRequest(std::forward(args)...); @@ -43,7 +43,7 @@ void ThreadsafeBatchingPool::shutdown() { template template -size_t ThreadsafeBatchingPool::generateBatch(Args &&... args) { +size_t ThreadsafeBatchingPool::generateBatch(Args &&...args) { std::unique_lock lock(mutex_); work_.wait(lock, [this]() { return enqueued_ || shutdown_; }); size_t sentencesInBatch = backend_.generateBatch(std::forward(args)...); diff --git a/src/translator/threadsafe_batching_pool.h b/src/translator/threadsafe_batching_pool.h index fdbf36cdc..9f46abb94 100644 --- a/src/translator/threadsafe_batching_pool.h +++ b/src/translator/threadsafe_batching_pool.h @@ -34,14 +34,14 @@ template class ThreadsafeBatchingPool { public: template - ThreadsafeBatchingPool(Args &&... args); + ThreadsafeBatchingPool(Args &&...args); ~ThreadsafeBatchingPool(); template - void enqueueRequest(Args &&... args); + void enqueueRequest(Args &&...args); template - size_t generateBatch(Args &&... args); + size_t generateBatch(Args &&...args); // Removes any pending requests from the batching pool. void clear(); diff --git a/src/translator/translation_model.cpp b/src/translator/translation_model.cpp index 3f91ebb47..6f8dd4dc8 100644 --- a/src/translator/translation_model.cpp +++ b/src/translator/translation_model.cpp @@ -61,24 +61,35 @@ void TranslationModel::loadBackend(size_t idx) { graph->getBackend()->configureDevice(options_); graph->reserveWorkspaceMB(options_->get("workspace")); - // Marian Model: Load from memoryBundle or shortList - if (memory_.model.size() > 0 && - memory_.model.begin() != - nullptr) { // If we have provided a byte array that contains the model memory, we can initialise the - // model from there, as opposed to from reading in the config file - ABORT_IF((uintptr_t)memory_.model.begin() % 256 != 0, - "The provided memory is not aligned to 256 bytes and will crash when vector instructions are used on it."); - if (options_->get("check-bytearray", false)) { - ABORT_IF(!validateBinaryModel(memory_.model, memory_.model.size()), - "The binary file is invalid. Incomplete or corrupted download?"); - } - const std::vector container = { - memory_.model.begin()}; // Marian supports multiple models initialised in this manner hence std::vector. - // However we will only ever use 1 during decoding. + // if memory_.models is populated, then all models were of binary format + if (memory_.models.size() >= 1) { + const std::vector container = std::invoke([&]() { + std::vector model_ptrs(memory_.models.size()); + for (size_t i = 0; i < memory_.models.size(); ++i) { + const AlignedMemory &model = memory_.models[i]; + + ABORT_IF(model.size() == 0 || model.begin() == nullptr, "The provided memory is empty. Cannot load the model."); + ABORT_IF( + (uintptr_t)model.begin() % 256 != 0, + "The provided memory is not aligned to 256 bytes and will crash when vector instructions are used on it."); + if (options_->get("check-bytearray", false)) { + ABORT_IF(!validateBinaryModel(model, model.size()), + "The binary file is invalid. Incomplete or corrupted download?"); + } + + model_ptrs[i] = model.begin(); + LOG(debug, "Loaded model {} of {} from memory", (i + 1), model_ptrs.size()); + } + return model_ptrs; + }); + scorerEnsemble = createScorers(options_, container); } else { + // load npz format models, or a mixture of binary/npz formats scorerEnsemble = createScorers(options_); + LOG(debug, "Loaded {} model(s) from file", scorerEnsemble.size()); } + for (auto scorer : scorerEnsemble) { scorer->init(graph); if (shortlistGenerator_) { diff --git a/wasm/README.md b/wasm/README.md index 883f80dc5..0f3f77426 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -4,7 +4,9 @@ All the instructions below are meant to run from the current directory. ## Using JS APIs -Please refer to the file `test_page/js/worker.js` that demonstrates how to use the bergamot translator in JavaScript via a ` + diff --git a/wasm/test_page/js/index.js b/wasm/test_page/js/index.js index b1c308e8b..56cbfdc72 100644 --- a/wasm/test_page/js/index.js +++ b/wasm/test_page/js/index.js @@ -1,156 +1,215 @@ -let worker; -let modelRegistry; +import {LatencyOptimisedTranslator, TranslatorBacking, CancelledError, SupersededError} from '../node_modules/@browsermt/bergamot-translator/translator.js'; -const $ = selector => document.querySelector(selector); -const status = message => ($("#status").innerText = message); - -const langFrom = $("#lang-from"); -const langTo = $("#lang-to"); - -if (window.Worker) { - worker = new Worker("js/worker.js"); - worker.postMessage(["import"]); +function $(selector) { + return document.querySelector(selector); } -document.querySelector("#input").addEventListener("keyup", function (event) { - translateCall(); -}); - -const _prepareTranslateOptions = (paragraphs) => { - const translateOptions = []; - paragraphs.forEach(paragraph => { - // Each option object can be different for each entry. But to keep the test page simple, - // we just keep all the options same (specifically avoiding parsing the input to determine - // html/non-html text) - translateOptions.push({"isQualityScores": true, "isHtml": true}); - }); - return translateOptions; -}; +function $$(selector) { + return document.querySelectorAll(selector); +} -const textToHTML = (text) => { +function encodeHTML(text) { const div = document.createElement('div'); div.appendChild(document.createTextNode(text)); return div.innerHTML; -}; - -const translateCall = () => { - const text = document.querySelector("#input").value; - if (!text.trim().length) return; - - const paragraphs = text.split(/\n+/).map(textToHTML); // escape HTML - const translateOptions = _prepareTranslateOptions(paragraphs); - const lngFrom = langFrom.value; - const lngTo = langTo.value; - worker.postMessage(["translate", lngFrom, lngTo, paragraphs, translateOptions]); -}; - -const addQualityClasses = (root) => { - // You can do this wit CSS variables, calc() and min/max, but JS is just easier +} - root.querySelectorAll('[x-bergamot-sentence-score]').forEach(el => { +function addQualityIndicators() { + $$('#output [x-bergamot-sentence-score]').forEach(el => { // The threshold is ln(0.5) (https://github.com/browsermt/bergamot-translator/pull/370#issuecomment-1058123399) - el.classList.toggle('bad', parseFloat(el.getAttribute('x-bergamot-sentence-score')) < -0.6931); + el.classList.toggle('bad', parseFloat(el.getAttribute('x-bergamot-sentence-score')) < Math.log(0.5)); }); - root.querySelectorAll('[x-bergamot-word-score]').forEach(el => { + $$('#output [x-bergamot-word-score]').forEach(el => { // The threshold is ln(0.5) (https://github.com/browsermt/bergamot-translator/pull/370#issuecomment-1058123399) - el.classList.toggle('bad', parseFloat(el.getAttribute('x-bergamot-word-score')) < -0.6931); + el.classList.toggle('bad', parseFloat(el.getAttribute('x-bergamot-word-score')) < Math.log(0.5)); }); // Add tooltips to each (sub)word with sentence and word score. - root.querySelectorAll('[x-bergamot-sentence-score] > [x-bergamot-word-score]').forEach(el => { + $$('#output [x-bergamot-sentence-score] > [x-bergamot-word-score]').forEach(el => { const sentenceScore = parseFloat(el.parentNode.getAttribute('x-bergamot-sentence-score')); const wordScore = parseFloat(el.getAttribute('x-bergamot-word-score')); - el.title = `Sentence: ${sentenceScore} Word: ${wordScore}`; + el.title = `Sentence: ${Math.exp(sentenceScore).toFixed(2)} Word: ${Math.exp(wordScore).toFixed(2)}`; }); } -worker.onmessage = function (e) { - if (e.data[0] === "translate_reply" && e.data[1]) { - // Clear output of previous translation - document.querySelector("#output").innerHTML = ''; - - // Add each translation in its own div to have a known root in which the - // sentence ids are unique. Used for highlighting sentences. - e.data[1].forEach(translatedHTML => { - const translation = document.createElement('div'); - translation.classList.add('translation'); - translation.innerHTML = translatedHTML; - addQualityClasses(translation); - document.querySelector("#output").appendChild(translation); - }); - } else if (e.data[0] === "load_model_reply" && e.data[1]) { - status(e.data[1]); - translateCall(); - } else if (e.data[0] === "import_reply" && e.data[1]) { - modelRegistry = e.data[1]; - init(); +function highlightSentence(element) { + const sentence = element.parentNode.hasAttribute('x-bergamot-sentence-index') + ? element.parentNode.getAttribute('x-bergamot-sentence-index') + : null; + $$('#output font[x-bergamot-sentence-index]').forEach(el => { + el.classList.toggle('highlight-sentence', el.getAttribute('x-bergamot-sentence-index') === sentence); + }) +} + +/** + * Very minimal WISYWIG editor. Just keyboard shortcuts for the IYKYK crowd. + */ +class Editor { + constructor(root) { + this.isApple = window.navigator.platform.startsWith('Mac'); + + this.root = root; + this.root.addEventListener('keydown', this.onkeydown.bind(this)); + + this.mapping = { + "b": "bold", + "i": "italic", + "u": "underline", + }; } -}; - -const loadModel = () => { - const lngFrom = langFrom.value; - const lngTo = langTo.value; - if (lngFrom !== lngTo) { - status(`Installing model...`); - console.log(`Loading model '${lngFrom}${lngTo}'`); - worker.postMessage(["load_model", lngFrom, lngTo]); - } else { - const input = textToHTML(document.querySelector("#input").value); - document.querySelector("#output").innerHTML = input; + + onkeydown(event) { + if (!(this.isApple ? event.metaKey : event.ctrlKey)) + return; + + if (!(event.key in this.mapping)) + return; + + document.execCommand(this.mapping[event.key], false, null); + + event.preventDefault(); } -}; - -langFrom.addEventListener("change", e => { - loadModel(); -}); - -langTo.addEventListener("change", e => { - loadModel(); -}); - -$(".swap").addEventListener("click", e => { - [langFrom.value, langTo.value] = [langTo.value, langFrom.value]; - $("#input").value = $("#output").innerText; - loadModel(); -}); - -$('#output').addEventListener('mouseover', e => { - const root = e.target.closest('.translation'); - const sentence = e.target.parentNode.hasAttribute('x-bergamot-sentence-index') ? e.target.parentNode.getAttribute('x-bergamot-sentence-index') : null; - document.querySelectorAll('#output font[x-bergamot-sentence-index]').forEach(el => { - el.classList.toggle('highlight-sentence', el.getAttribute('x-bergamot-sentence-index') === sentence && el.closest('.translation') === root); - }) -}) +} + +async function main() { + const options = { + cacheSize: 2^13, + downloadTimeout: null // Disable timeout + }; + + const backing = new TranslatorBacking(options); + + let pending = 0; // Number of pending requests + + // Patch the fetch() function to track number of pending requests + backing.fetch = async function(...args) { + try { + $('.app').classList.toggle('loading', ++pending > 0); + return await TranslatorBacking.prototype.fetch.call(backing, ...args); + } finally { + $('.app').classList.toggle('loading', --pending > 0); + } + }; -function init() { - // Populate langs - const langs = Array.from(new Set(Object.keys(modelRegistry).reduce((acc, key) => acc.concat([key.substr(0, 2), key.substr(2, 2)]), []))); - const langNames = new Intl.DisplayNames(undefined, {type: "language"}); + // Wait for the language model registry to load. Once it is loaded, use + // it to fill the "from" and "to" language selection dropdowns. + await backing.registry.then(models => { + const names = new Intl.DisplayNames(['en'], {type: 'language'}); - // Sort languages by display name - langs.sort((a, b) => langNames.of(a).localeCompare(langNames.of(b))); + ['from', 'to'].forEach(field => { + const languages = new Set(models.map(model => model[field])); + const select = $(`#lang-${field}`); - // Populate the dropdowns - langs.forEach(code => { - const name = langNames.of(code); - langFrom.innerHTML += ``; - langTo.innerHTML += ``; + const pairs = Array.from(languages, code => ({code, name: names.of(code)})); + + pairs.sort(({name: a}, {name: b}) => a.localeCompare(b)); + + pairs.forEach(({name, code}) => { + select.add(new Option(name, code)); + }) + }); + + $('#lang-from').value = 'en'; + $('#lang-to').value = 'es'; }); - // try to guess input language from user agent - let myLang = navigator.language; - if (myLang) { - myLang = myLang.split("-")[0]; - let langIndex = langs.indexOf(myLang); - if (langIndex > -1) { - console.log("guessing input language is", myLang); - langFrom.value = myLang; + // Intentionally do this after querying backing.registry to make sure that + // that request is fired off first. Now we can start thinking about loading + // the WASM binary etc. + const translator = new LatencyOptimisedTranslator(options, backing); + + let abortController = new AbortController(); + + const translate = async () => { + try { + const from = $('#lang-from').value; + const to = $('#lang-to').value; + + // Querying models to see whether quality estimation is supported by all + // of them. + const models = await backing.getModels({from, to}); + const qualityScores = models.every(model => 'qualityModel' in model.files); + + $('.app').classList.add('translating'); + + const response = await translator.translate({ + from, + to, + text: $('#input').innerHTML, + html: true, + qualityScores + }, {signal: abortController.signal}); + + $('#output').innerHTML = response.target.text; + $('#output').classList.toggle('has-quality-scores', qualityScores); + + if (qualityScores) + addQualityIndicators(); + + } catch (error) { + // Ignore errors caused by changing the language pair (which triggers abort()) + if (error.constructor === CancelledError) { + return; + } + + // Ignore 'errors' caused by typing too fast or by changing the language + // pair while a translation was still in progress (or being loaded) + if (error.constructor === SupersededError || error.constructor === CancelledError) + return; + + // Ignore errors caused by selecting a bad pair (e.g. en -> en) + if (error.message.startsWith('No model available to translate from')) + return; + + alert(`Error during translation: ${error}\n\n${error.stack}`); + } finally { + const worker = await Promise.race([translator.worker, Promise.resolve(null)]); + $('.app').classList.toggle('translating', worker === null || !worker.idle); } } - // find first output lang that *isn't* input language - langTo.value = langs.find(code => code !== langFrom.value); - // load this model - loadModel(); + const reset = async () => { + // Cancel any pending loading/translation + abortController.abort(); + + // Reset abort controller to a fresh un-aborted one + abortController = new AbortController(); + + // Clear output to make it more clear something is happening + $('#output').innerHTML = ''; + + // Immediately start loading the new selection + translate(); + } + + $('button.swap').addEventListener('click', () => { + const tmp = $('#lang-from').value; + $('#lang-from').value = $('#lang-to').value; + $('#lang-to').value = tmp; + translate(); + }) + + // Simple WYSIWYG controls + const editor = new Editor($('#input')); + + // Translate on any change + $('#input').addEventListener('input', translate); + $('#lang-from').addEventListener('input', reset); + $('#lang-to').addEventListener('input', reset); + + // Hook up sentence boundary highlighting if that information is available. + $('#output').addEventListener('mouseover', (e) => highlightSentence(e.target)) + + // Wait for bergamot-translator to load. This could throw a CompileError + // which we want to catch so we can show "oh noes browser not supported!" + translator.worker.catch(error => { + // Catch CompileErrors because for those we know what to do. + if (error.name === 'CompileError') + $('#unsupported-browser').hidden = false; + else + throw error; + }); } + +main(); diff --git a/wasm/test_page/js/worker.js b/wasm/test_page/js/worker.js deleted file mode 100644 index 3327d8a3a..000000000 --- a/wasm/test_page/js/worker.js +++ /dev/null @@ -1,352 +0,0 @@ -// All variables specific to translation service -var translationService = undefined; - -// Model registry -let modelRegistry = undefined; - -// A map of language-pair to TranslationModel object -var languagePairToTranslationModels = new Map(); - -const BERGAMOT_TRANSLATOR_MODULE = "bergamot-translator-worker.js"; -const MODEL_REGISTRY = "../models/registry.json"; -const MODEL_ROOT_URL = "../models/"; -const PIVOT_LANGUAGE = 'en'; - -// Information corresponding to each file type -const fileInfo = [ - {"type": "model", "alignment": 256}, - {"type": "lex", "alignment": 64}, - {"type": "vocab", "alignment": 64}, - {"type": "qualityModel", "alignment": 64} -]; - -const encoder = new TextEncoder(); // string to utf-8 converter -const decoder = new TextDecoder(); // utf-8 to string converter - -const start = Date.now(); -let moduleLoadStart; -var Module = { - preRun: [function() { - log(`Time until Module.preRun: ${(Date.now() - start) / 1000} secs`); - moduleLoadStart = Date.now(); - }], - onRuntimeInitialized: async function() { - log(`Wasm Runtime initialized Successfully (preRun -> onRuntimeInitialized) in ${(Date.now() - moduleLoadStart) / 1000} secs`); - const response = await fetch(MODEL_REGISTRY); - modelRegistry = await response.json(); - postMessage([`import_reply`, modelRegistry]); - } -}; - -const log = (message) => { - console.debug(message); -} - -onmessage = async function(e) { - const command = e.data[0]; - log(`Message '${command}' received from main script`); - let result = ""; - if (command === 'import') { - importScripts(BERGAMOT_TRANSLATOR_MODULE); - } else if (command === 'load_model') { - let start = Date.now(); - let from = e.data[1]; - let to = e.data[2]; - try { - await constructTranslationService(); - await constructTranslationModel(from, to); - log(`Model '${from}${to}' successfully constructed. Time taken: ${(Date.now() - start) / 1000} secs`); - result = "Model successfully loaded"; - } catch (error) { - log(`Model '${from}${to}' construction failed: '${error.message}'`); - result = "Model loading failed"; - } - log(`'${command}' command done, Posting message back to main script`); - postMessage([`${command}_reply`, result]); - } else if (command === 'translate') { - const from = e.data[1]; - const to = e.data[2]; - const input = e.data[3]; - const translateOptions = e.data[4]; - let inputWordCount = 0; - let inputBlockElements = 0; - input.forEach(sentence => { - inputWordCount += sentence.trim().split(" ").filter(word => word.trim() !== "").length; - inputBlockElements++; - }) - let start = Date.now(); - try { - log(`Blocks to translate: ${inputBlockElements}`); - result = translate(from, to, input, translateOptions); - const secs = (Date.now() - start) / 1000; - log(`Translation '${from}${to}' Successful. Speed: ${Math.round(inputWordCount / secs)} WPS (${inputWordCount} words in ${secs} secs)`); - } catch (error) { - log(`Error: ${error.message}`); - } - log(`'${command}' command done, Posting message back to main script`); - postMessage([`${command}_reply`, result]); - } -} - -// Instantiates the Translation Service -const constructTranslationService = async () => { - if (!translationService) { - var translationServiceConfig = {cacheSize: 20000}; - log(`Creating Translation Service with config: ${translationServiceConfig}`); - translationService = new Module.BlockingService(translationServiceConfig); - log(`Translation Service created successfully`); - } -} - -// Constructs translation model(s) for the source and target language pair (using -// pivoting if required). -const constructTranslationModel = async (from, to) => { - // Delete all previously constructed translation models and clear the map - languagePairToTranslationModels.forEach((value, key) => { - log(`Destructing model '${key}'`); - value.delete(); - }); - languagePairToTranslationModels.clear(); - - if (_isPivotingRequired(from, to)) { - // Pivoting requires 2 translation models - const languagePairSrcToPivot = _getLanguagePair(from, PIVOT_LANGUAGE); - const languagePairPivotToTarget = _getLanguagePair(PIVOT_LANGUAGE, to); - await Promise.all([_constructTranslationModelHelper(languagePairSrcToPivot), - _constructTranslationModelHelper(languagePairPivotToTarget)]); - } - else { - // Non-pivoting case requires only 1 translation model - await _constructTranslationModelHelper(_getLanguagePair(from, to)); - } -} - -// Translates text from source language to target language (via pivoting if necessary). -const translate = (from, to, input, translateOptions) => { - let vectorResponseOptions, vectorSourceText, vectorResponse; - try { - // Prepare the arguments (vectorResponseOptions and vectorSourceText (vector)) of Translation API and call it. - // Result is a vector where each of its item corresponds to one item of vectorSourceText in the same order. - vectorResponseOptions = _prepareResponseOptions(translateOptions); - vectorSourceText = _prepareSourceText(input); - - if (_isPivotingRequired(from, to)) { - // Translate via pivoting - const translationModelSrcToPivot = _getLoadedTranslationModel(from, PIVOT_LANGUAGE); - const translationModelPivotToTarget = _getLoadedTranslationModel(PIVOT_LANGUAGE, to); - vectorResponse = translationService.translateViaPivoting(translationModelSrcToPivot, - translationModelPivotToTarget, - vectorSourceText, - vectorResponseOptions); - } - else { - // Translate without pivoting - const translationModel = _getLoadedTranslationModel(from, to); - vectorResponse = translationService.translate(translationModel, vectorSourceText, vectorResponseOptions); - } - - // Parse all relevant information from vectorResponse - const listTranslatedText = _parseTranslatedText(vectorResponse); - const listSourceText = _parseSourceText(vectorResponse); - const listTranslatedTextSentences = _parseTranslatedTextSentences(vectorResponse); - const listSourceTextSentences = _parseSourceTextSentences(vectorResponse); - - log(`Source text: ${listSourceText}`); - log(`Translated text: ${listTranslatedText}`); - log(`Translated sentences: ${JSON.stringify(listTranslatedTextSentences)}`); - log(`Source sentences: ${JSON.stringify(listSourceTextSentences)}`); - - return listTranslatedText; - } finally { - // Necessary clean up - if (vectorSourceText != null) vectorSourceText.delete(); - if (vectorResponseOptions != null) vectorResponseOptions.delete(); - if (vectorResponse != null) vectorResponse.delete(); - } -} - -// Downloads file from a url and returns the array buffer -const _downloadAsArrayBuffer = async(url) => { - const response = await fetch(url); - if (!response.ok) { - throw Error(`Downloading ${url} failed: HTTP ${response.status} - ${response.statusText}`); - } - return response.arrayBuffer(); -} - -// Constructs and initializes the AlignedMemory from the array buffer and alignment size -const _prepareAlignedMemoryFromBuffer = async (buffer, alignmentSize) => { - var byteArray = new Int8Array(buffer); - var alignedMemory = new Module.AlignedMemory(byteArray.byteLength, alignmentSize); - const alignedByteArrayView = alignedMemory.getByteArrayView(); - alignedByteArrayView.set(byteArray); - return alignedMemory; -} - -async function prepareAlignedMemory(file, languagePair) { - const fileName = `${MODEL_ROOT_URL}/${languagePair}/${modelRegistry[languagePair][file.type].name}`; - const buffer = await _downloadAsArrayBuffer(fileName); - const alignedMemory = await _prepareAlignedMemoryFromBuffer(buffer, file.alignment); - log(`"${file.type}" aligned memory prepared. Size:${alignedMemory.size()} bytes, alignment:${file.alignment}`); - return alignedMemory; -} - -const _constructTranslationModelHelper = async (languagePair) => { - log(`Constructing translation model ${languagePair}`); - - /*Set the Model Configuration as YAML formatted string. - For available configuration options, please check: https://marian-nmt.github.io/docs/cmd/marian-decoder/ - Vocab files are re-used in both translation directions. - DO NOT CHANGE THE SPACES BETWEEN EACH ENTRY OF CONFIG - */ - const modelConfig = `beam-size: 1 -normalize: 1.0 -word-penalty: 0 -max-length-break: 128 -mini-batch-words: 1024 -workspace: 128 -max-length-factor: 2.0 -skip-cost: false -cpu-threads: 0 -quiet: true -quiet-translation: true -gemm-precision: int8shiftAlphaAll -alignment: soft -`; - - const promises = []; - fileInfo.filter(file => modelRegistry[languagePair].hasOwnProperty(file.type)) - .map((file) => { - promises.push(prepareAlignedMemory(file, languagePair)); - }); - - const alignedMemories = await Promise.all(promises); - - log(`Translation Model config: ${modelConfig}`); - log(`Aligned memory sizes: Model:${alignedMemories[0].size()} Shortlist:${alignedMemories[1].size()} Vocab:${alignedMemories[2].size()}`); - const alignedVocabMemoryList = new Module.AlignedMemoryList(); - alignedVocabMemoryList.push_back(alignedMemories[2]); - let translationModel; - if (alignedMemories.length === fileInfo.length) { - log(`QE:${alignedMemories[3].size()}`); - translationModel = new Module.TranslationModel(modelConfig, alignedMemories[0], alignedMemories[1], alignedVocabMemoryList, alignedMemories[3]); - } - else { - translationModel = new Module.TranslationModel(modelConfig, alignedMemories[0], alignedMemories[1], alignedVocabMemoryList, null); - } - languagePairToTranslationModels.set(languagePair, translationModel); -} - -const _isPivotingRequired = (from, to) => { - return (from !== PIVOT_LANGUAGE) && (to !== PIVOT_LANGUAGE); -} - -const _getLanguagePair = (srcLang, tgtLang) => { - return `${srcLang}${tgtLang}`; -} - -const _getLoadedTranslationModel = (srcLang, tgtLang) => { - const languagePair = _getLanguagePair(srcLang, tgtLang); - if (!languagePairToTranslationModels.has(languagePair)) { - throw Error(`Translation model '${languagePair}' not loaded`); - } - return languagePairToTranslationModels.get(languagePair); -} - -const _parseTranslatedText = (vectorResponse) => { - const result = []; - for (let i = 0; i < vectorResponse.size(); i++) { - const response = vectorResponse.get(i); - result.push(response.getTranslatedText()); - } - return result; -} - -const _parseTranslatedTextSentences = (vectorResponse) => { - const result = []; - for (let i = 0; i < vectorResponse.size(); i++) { - const response = vectorResponse.get(i); - result.push(_getTranslatedSentences(response)); - } - return result; -} - -const _parseSourceText = (vectorResponse) => { - const result = []; - for (let i = 0; i < vectorResponse.size(); i++) { - const response = vectorResponse.get(i); - result.push(response.getOriginalText()); - } - return result; -} - -const _parseSourceTextSentences = (vectorResponse) => { - const result = []; - for (let i = 0; i < vectorResponse.size(); i++) { - const response = vectorResponse.get(i); - result.push(_getSourceSentences(response)); - } - return result; -} - -const _prepareResponseOptions = (translateOptions) => { - let vectorResponseOptions = new Module.VectorResponseOptions; - translateOptions.forEach(translateOption => { - vectorResponseOptions.push_back({ - qualityScores: translateOption["isQualityScores"], - alignment: true, - html: translateOption["isHtml"] - }); - }); - if (vectorResponseOptions.size() == 0) { - vectorResponseOptions.delete(); - throw Error(`No Translation Options provided`); - } - return vectorResponseOptions; -} - -const _prepareSourceText = (input) => { - let vectorSourceText = new Module.VectorString; - input.forEach(paragraph => { - // prevent empty paragraph - it breaks the translation - if (paragraph.trim() === "") { - return; - } - vectorSourceText.push_back(paragraph.trim()) - }) - if (vectorSourceText.size() == 0) { - vectorSourceText.delete(); - throw Error(`No text provided to translate`); - } - return vectorSourceText; -} - -const _getTranslatedSentences = (response) => { - const sentences = []; - const text = response.getTranslatedText(); - for (let sentenceIndex = 0; sentenceIndex < response.size(); sentenceIndex++) { - const utf8SentenceByteRange = response.getTranslatedSentence(sentenceIndex); - sentences.push(_getSubString(text, utf8SentenceByteRange)); - } - return sentences; -} - -const _getSourceSentences = (response) => { - const sentences = []; - const text = response.getOriginalText(); - for (let sentenceIndex = 0; sentenceIndex < response.size(); sentenceIndex++) { - const utf8SentenceByteRange = response.getSourceSentence(sentenceIndex); - sentences.push(_getSubString(text, utf8SentenceByteRange)); - } - return sentences; -} - -/* - * Returns a substring of text (a string). The substring is represented by - * byteRange (begin and end endices) within the utf-8 encoded version of the text. - */ -const _getSubString = (text, utf8ByteRange) => { - const textUtf8ByteView = encoder.encode(text); - const substringUtf8ByteView = textUtf8ByteView.subarray(utf8ByteRange.begin, utf8ByteRange.end); - return decoder.decode(substringUtf8ByteView); -} diff --git a/wasm/test_page/logos.png b/wasm/test_page/logos.png new file mode 100644 index 000000000..7646f3ca2 Binary files /dev/null and b/wasm/test_page/logos.png differ diff --git a/wasm/test_page/package-lock.json b/wasm/test_page/package-lock.json index 065c92de8..22d229647 100644 --- a/wasm/test_page/package-lock.json +++ b/wasm/test_page/package-lock.json @@ -5,18 +5,28 @@ "packages": { "": { "dependencies": { + "@browsermt/bergamot-translator": "file:../module", "cors": "^2.8.5", - "express": "^4.17.1", + "express": "^4.18.2", "nocache": "^2.1.0" } }, + "../module": { + "name": "@browsermt/bergamot-translator", + "version": "0.4.8", + "license": "MPL-2.0" + }, + "node_modules/@browsermt/bergamot-translator": { + "resolved": "../module", + "link": true + }, "node_modules/accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "dependencies": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" }, "engines": { "node": ">= 0.6" @@ -28,39 +38,54 @@ "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "node_modules/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "dependencies": { - "bytes": "3.1.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" } }, "node_modules/bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "engines": { "node": ">= 0.8" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "dependencies": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" }, "engines": { "node": ">= 0.6" @@ -75,9 +100,9 @@ } }, "node_modules/cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", "engines": { "node": ">= 0.6" } @@ -108,27 +133,31 @@ } }, "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", "engines": { "node": ">= 0.8" } @@ -136,48 +165,49 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "engines": { "node": ">= 0.6" } }, "node_modules/express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "dependencies": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -187,16 +217,16 @@ } }, "node_modules/finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" }, "engines": { @@ -204,9 +234,9 @@ } }, "node_modules/forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "engines": { "node": ">= 0.6" } @@ -214,24 +244,64 @@ "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "engines": { "node": ">= 0.6" } }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" }, "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/iconv-lite": { @@ -246,9 +316,9 @@ } }, "node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ipaddr.js": { "version": "1.9.1", @@ -261,7 +331,7 @@ "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "engines": { "node": ">= 0.6" } @@ -291,19 +361,19 @@ } }, "node_modules/mime-db": { - "version": "1.45.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", - "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.28", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz", - "integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dependencies": { - "mime-db": "1.45.0" + "mime-db": "1.52.0" }, "engines": { "node": ">= 0.6" @@ -312,12 +382,12 @@ "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "node_modules/negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "engines": { "node": ">= 0.6" } @@ -338,10 +408,18 @@ "node": ">=0.10.0" } }, + "node_modules/object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "dependencies": { "ee-first": "1.1.1" }, @@ -363,11 +441,11 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "node_modules/proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dependencies": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { @@ -375,11 +453,17 @@ } }, "node_modules/qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, "engines": { "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/range-parser": { @@ -391,12 +475,12 @@ } }, "node_modules/raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "dependencies": { - "bytes": "3.1.0", - "http-errors": "1.7.2", + "bytes": "3.1.2", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, @@ -405,9 +489,23 @@ } }, "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/safer-buffer": { "version": "2.1.2", @@ -415,64 +513,77 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "dependencies": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "2.0.0", "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", + "ms": "2.1.3", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/send/node_modules/ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.18.0" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "engines": { "node": ">=0.6" } @@ -492,7 +603,7 @@ "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", "engines": { "node": ">= 0.8" } @@ -515,13 +626,16 @@ } }, "dependencies": { + "@browsermt/bergamot-translator": { + "version": "file:../module" + }, "accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, "array-flatten": { @@ -530,33 +644,44 @@ "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "requires": { - "bytes": "3.1.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, "bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } }, "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" } }, "content-type": { @@ -565,9 +690,9 @@ "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" }, "cookie-signature": { "version": "1.0.6", @@ -592,106 +717,135 @@ } }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "requires": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.5.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "~1.1.2", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", + "on-finished": "2.4.1", "parseurl": "~1.3.3", - "statuses": "~1.5.0", + "statuses": "2.0.1", "unpipe": "~1.0.0" } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" }, "http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" } }, "iconv-lite": { @@ -703,9 +857,9 @@ } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ipaddr.js": { "version": "1.9.1", @@ -715,7 +869,7 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", @@ -733,27 +887,27 @@ "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, "mime-db": { - "version": "1.45.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", - "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.28", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz", - "integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "1.45.0" + "mime-db": "1.52.0" } }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, "nocache": { "version": "2.1.0", @@ -765,10 +919,15 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==" + }, "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } @@ -784,18 +943,21 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { "version": "1.2.1", @@ -803,20 +965,20 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", "requires": { - "bytes": "3.1.0", - "http-errors": "1.7.2", + "bytes": "3.1.2", + "http-errors": "2.0.0", "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { "version": "2.1.2", @@ -824,57 +986,67 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "2.0.0", "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", + "ms": "2.1.3", + "on-finished": "2.4.1", "range-parser": "~1.2.1", - "statuses": "~1.5.0" + "statuses": "2.0.1" }, "dependencies": { "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" } } }, "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.18.0" } }, "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } }, "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" }, "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "type-is": { "version": "1.6.18", @@ -888,7 +1060,7 @@ "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "utils-merge": { "version": "1.0.1", diff --git a/wasm/test_page/package.json b/wasm/test_page/package.json index 20af6d2ab..622b48c1a 100644 --- a/wasm/test_page/package.json +++ b/wasm/test_page/package.json @@ -1,7 +1,14 @@ { "dependencies": { + "@browsermt/bergamot-translator": "file:../module", "cors": "^2.8.5", - "express": "^4.17.1", + "express": "^4.18.2", "nocache": "^2.1.0" + }, + "config": { + "port": 80 + }, + "scripts": { + "start": "node ./bergamot-httpserver.js $npm_package_config_port 1 0" } } diff --git a/wasm/test_page/start_server.sh b/wasm/test_page/start_server.sh index 59d455d14..5b6eeb0a3 100644 --- a/wasm/test_page/start_server.sh +++ b/wasm/test_page/start_server.sh @@ -24,7 +24,7 @@ fi # Prepare a list all wasm artifacts to be copied and copy them to the destination folder ARTIFACTS_BASE_NAME="bergamot-translator-worker" ARTIFACTS="$1/$ARTIFACTS_BASE_NAME.js $1/$ARTIFACTS_BASE_NAME.wasm" -ARTIFACTS_DESTINATION_FOLDER=$SCRIPT_ABSOLUTE_PATH/js +ARTIFACTS_DESTINATION_FOLDER=$SCRIPT_ABSOLUTE_PATH/../module/worker for i in $ARTIFACTS; do [ -f "$i" ] || breaks