Merge branch 'main' into fix/wrong-stream-conn-key

.github/workflows/default.yml

@@ -216,6 +216,67 @@ jobs:
       if: always()  # We want the log even on failures.
 
 
+  build-scies:
+    needs: [lint, typecheck, test]
+    if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+    strategy:
+      fail-fast: false
+      matrix:
+        # ubuntu-latest: intel
+        # linux-aarch64: aarch64 (self-hosted)
+        # macos-12: intel
+        # macos-13: apple silicon
+        os: [ubuntu-latest, linux-aarch64, macos-13-xlarge, macos-12-large]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Fetch remote tags
+      run: git fetch origin 'refs/tags/*:refs/tags/*' -f
+    - name: Create LFS file hash list
+      run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id
+    - name: Restore LFS cache
+      uses: actions/cache@v3
+      id: lfs-cache
+      with:
+          path: .git/lfs
+          key: lfs-${{ hashFiles('.lfs-assets-id') }}
+    - name: Git LFS Pull
+      run: git lfs pull
+    - name: Extract Python version from pants.toml
+      run: |
+        PYTHON_VERSION=$(awk -F'["]' '/CPython==/ {print $2; exit}' pants.toml | sed 's/CPython==//')
+        echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV
+        echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
+
+    - if: ${{ !endsWith(matrix.os, 'linux-aarch64') }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PROJECT_PYTHON_VERSION }}
+        cache: "pip"
+      # For linux-aarch64 runner, we assume that we have the correct prebuilt Python version already.
+    - name: Bootstrap Pants
+      uses: pantsbuild/actions/init-pants@v5-scie-pants
+      with:
+        gha-cache-key: pants-cache-main-1-deploy-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+        named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }}
+        cache-lmdb-store: 'true'
+    - name: Build packages
+      run: |
+        pants --tag="-wheel" package '::'
+        rm -rf dist/src.*/
+    - name: Upload scies
+      uses: actions/upload-artifact@v3
+      with:
+        name: scies
+        path: dist/*
+    - name: Upload pants log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants.deploy.log
+        path: .pants.d/pants.log
+      if: always()  # We want the log even on failures.
+
+
   build-wheels:
     needs: [lint, typecheck, test]
     if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
@@ -237,15 +298,14 @@ jobs:
       run: git lfs pull
     - name: Extract Python version from pants.toml
       run: |
-        PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml)
+        PYTHON_VERSION=$(awk -F'["]' '/CPython==/ {print $2; exit}' pants.toml | sed 's/CPython==//')
         echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV
         echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
-    - name: Prepare the pip cache directory
-      run: mkdir -p ~/.cache/pip
     - name: Set up Python as Runtime
       uses: actions/setup-python@v4
       with:
         python-version: ${{ env.PROJECT_PYTHON_VERSION }}
+        cache: "pip"
     - name: Bootstrap Pants
       uses: pantsbuild/actions/init-pants@v5-scie-pants
       with:
@@ -292,24 +352,53 @@ jobs:
       with:
         name: wheels
         path: dist/*.whl
+    - name: Upload pants log
+      uses: actions/upload-artifact@v3
+      with:
+        name: pants.deploy.log
+        path: .pants.d/pants.log
+      if: always()  # We want the log even on failures.
+
+
+  make-github-release:
+    needs: [build-scies, build-wheels]
+    if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    environment: deploy-to-pypi
+    steps:
+    - uses: actions/checkout@v4
+    - name: Fetch remote tags
+      run: git fetch origin 'refs/tags/*:refs/tags/*' -f
+    - name: Extract Python version from pants.toml
+      run: |
+        PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml)
+        echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV
+        echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
+    - name: Set up Python as Runtime
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PROJECT_PYTHON_VERSION }}
     - name: Extract the release changelog
       run: |
         python ./scripts/extract-release-changelog.py
         python ./scripts/determine-release-type.py
+    - name: Download wheels
+      uses: actions/download-artifact@v3
+      with:
+        name: wheels
+        path: dist
+    - name: Download scies
+      uses: actions/download-artifact@v3
+      with:
+        name: scies
+        path: dist
     - name: Release to GitHub
       uses: softprops/action-gh-release@v1
       with:
         body_path: "CHANGELOG_RELEASE.md"
         prerelease: ${{ env.IS_PRERELEASE }}
         files: |
-          dist/*.whl
-          dist/*.tar.gz
-    - name: Upload pants log
-      uses: actions/upload-artifact@v3
-      with:
-        name: pants.deploy.log
-        path: .pants.d/pants.log
-      if: always()  # We want the log even on failures.
+          dist/*
 
 
   build-conda-pack-for-windows:
@@ -370,47 +459,3 @@ jobs:
     - name: Upload conda-pack to GitHub release
       run: |
         gh release upload ${{ github.ref_name }} backend.ai-client-${{ github.ref_name }}-windows-conda.zip
-
-
-  build-client-image:
-    needs: [build-wheels]
-    runs-on: [ubuntu-latest-8-cores, self-hosted]
-    environment: deploy-to-docker-hub
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Extract Python version from pants.toml
-        shell: bash
-        run: |
-          export LANG=C.UTF-8
-          PYTHON_VERSION=$(grep -m 1 -oP '(?<=CPython==)([^"]+)' pants.toml)
-          echo "PANTS_CONFIG_FILES=pants.ci.toml" >> $GITHUB_ENV
-          echo "PROJECT_PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
-      - name: Download wheels
-        uses: actions/download-artifact@v3
-        with:
-          name: wheels
-          path: dist
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./backend.ai-client.Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            lablup/backend.ai-client:${{ github.ref_name }}
-            lablup/backend.ai-client:latest
-          build-args: |
-            PYTHON_VERSION=${{ env.PROJECT_PYTHON_VERSION }}
-            PKGVER=${{ github.ref_name }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

changes/1518.fix.md

@@ -0,0 +1 @@
+Use "m" as the default suffix if not specified in the resource slots when creating sessions via the client CLI

changes/1627.fix.md

@@ -0,0 +1 @@
+Mask sensitive fields when reading the container registry information via the manager GraphQL API

changes/1629.feature.md

@@ -0,0 +1 @@
+expose `max_ipu_devices_per_container` key to `config.toml`

changes/1632.enhance.md

@@ -0,0 +1 @@
+Upgrade Graphene and GraphQL core (v2 -> v3) for better support of Relay, security rules, and other improvements

changes/1642.fix.md

@@ -0,0 +1 @@
+Lower limit of maximum available characters to name of model service to fix model service session refuses to be created when service name is longer than 28 characters 

changes/1649.feature.md

@@ -0,0 +1 @@
+Detailed Docker container creation failure log.

changes/1651.fix.md

@@ -0,0 +1 @@
+Move database accessing code to context manager scope

changes/1652.feature.md

@@ -0,0 +1 @@
+Allow privileged access for other's VFolder to superadmin

changes/1653.fix.md

@@ -0,0 +1 @@
+Replace the manager's shared redis config with the common's redis config, as this update is missed in #1586

changes/1656.fix.md

@@ -0,0 +1 @@
+Revert #1652

changes/1660.fix.md

@@ -0,0 +1 @@
+Fix `backend.ai-agent` package not recognizing `backend.ai-kernel` as dependency when building python package

changes/1664.feature.md

@@ -0,0 +1 @@
+Add a `allow_app_download_panel` config to webserver to show/hide the webui app download panel on the summary page.

changes/1665.fix.md

@@ -0,0 +1 @@
+Fix symbolic link loop error of vfolder

changes/1666.feature.md

@@ -0,0 +1 @@
+Add a `allow_custom_resource_allocation` config to webserver to show/hide the custom allocation on the session launcher.

changes/1667.fix.md

@@ -0,0 +1 @@
+Fix `execute_with_retry()` not retrying when DB commit has failed due to incorrect exception handling

changes/1668.fix.md

@@ -0,0 +1 @@
+Update the parameter of session-template update API to follow-up change of session-template create API.

changes/1670.feature.md

@@ -0,0 +1 @@
+Allow explicit `null` and empty string to ContainerRegistry mutations.

changes/1674.enhance.md

@@ -0,0 +1 @@
+Use the explicit `graphql.Undefined` value to fill the unspecified fields of GraphQL mutation input objects

changes/1676.misc.md

@@ -0,0 +1 @@
+Include `HOME` env-var when running tests via pants

changes/1677.fix.md

@@ -0,0 +1 @@
+Restore removed graphene fields of resource policies and set them deprecated.

changes/1680.feature.md

@@ -0,0 +1 @@
+Add pex + scie based single-file self-contained self-bootstrapping bindary distributions that can be executed on any modern Linux/macOS machines using the standalone Python builds (thanks to @sureshjoshi)

changes/1686.fix.md

@@ -0,0 +1 @@
+Allow running manager CLI commands without having `manager.toml` when they do not need it

configs/webserver/sample.conf

@@ -52,6 +52,8 @@ mask_user_info = false
 # hide_agents = true
 # URL to download the webui electron app. If blank, https://github.com/lablup/backend.ai-webui/releases/download will be used.
 # app_download_url = ""
+# Allow users to see the panel downloading the webui app from the summary page.
+# allow_app_download_panel = true
 # Enable/disable 2-Factor-Authentication (TOTP).
 enable_2FA = false
 # Force enable 2-Factor-Authentication (TOTP).
@@ -61,6 +63,8 @@ force_2FA = false
 # system_SSH_image = ""
 # If true, display the amount of usage per directory such as folder capacity, and number of files and directories.
 directory_based_usage = false
+# If true, display the custom allocation on the session launcher.
+# allow_custom_resource_allocation = true
 
 [resources]
 # Display "Open port to public" checkbox in the app launcher.

docs/dev/development-setup.rst

@@ -241,7 +241,7 @@ Open yet another terminal for client and run:
    $ source ./env-local-admin-api.sh  # Use the generated local endpoint and credential config.
    $ # source ./env-local-user-api.sh  # Yo may choose an alternative credential config.
    $ ./backend.ai config
-   $ ./backend.ai run python -c 'print("hello world")'
+   $ ./backend.ai run python --rm -c 'print("hello world")'
    ∙ Session token prefix: fb05c73953
    ✔ [0] Session fb05c73953 is ready.
    hello world

pants.toml

@@ -11,6 +11,7 @@ backend_packages = [
     "pants.backend.plugin_development",
     "setupgen",
     "platform_resources",
+    "scie",
     "towncrier_tool",
 ]
 pants_ignore = [
@@ -39,7 +40,7 @@ root_patterns = [
 ]
 
 [test]
-extra_env_vars = ["BACKEND_BUILD_ROOT=%(buildroot)s"]
+extra_env_vars = ["BACKEND_BUILD_ROOT=%(buildroot)s", "HOME"]
 
 [python]
 enable_resolves = true