CI(deploy): Add more installation configs to surface failure cause (#…

…1658)
posit-dev · Aug 29, 2024 · 24013f4 · 24013f4
1 parent ed56097
commit 24013f4
Show file tree

Hide file tree

Showing 12 changed files with 170 additions and 48 deletions.
diff --git a/.github/workflows/deploy-tests.yaml b/.github/workflows/deploy-tests.yaml
@@ -9,13 +9,63 @@ jobs:
   playwright-deploys:
     # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR)
     # Only one is allowed to run at a time because it is deploying to the same server location.
-    concurrency: playwright-deploys
+    concurrency: playwright-deploys-${{ matrix.config.name }}
     runs-on: ${{ matrix.os }}
+    name: ${{ matrix.config.name }}
     strategy:
       matrix:
         # Matches deploy server python version
         python-version: ["3.10"]
         os: [ubuntu-latest]
+        config:
+          # Released server, shiny, and rsconnect
+          - name: "pypi-shiny-rsconnect-connect"
+            released_connect_server: true
+            pypi_shiny: true
+            pypi_rsconnect: true
+            base_test_dir: "./tests/playwright/deploys/express-page_sidebar"
+            app_name: "pypi-shiny-rsconnect"
+            test_shinyappsio: false
+
+          # Released shiny and rsconnect
+          # Dev server
+          - name: "pypi-shiny-rsconnect-dogfood"
+            released_connect_server: false
+            pypi_shiny: true
+            pypi_rsconnect: true
+            base_test_dir: "./tests/playwright/deploys/express-page_sidebar"
+            app_name: "pypi-shiny-rsconnect"
+            test_shinyappsio: false
+
+          # Released shiny
+          # Dogfood server and rsconnect
+          - name: "pypi-shiny-dev-rsconnect-dogfood"
+            released_connect_server: false
+            pypi_shiny: true
+            pypi_rsconnect: false
+            base_test_dir: "./tests/playwright/deploys/express-page_sidebar"
+            app_name: "pypi-shiny-dev-rsconnect"
+            test_shinyappsio: false
+
+          # GitHub shiny v1.0.0 - test if github packages can be installed
+          # Dogfood server and rsconnect
+          - name: "github-shiny-dev-rsconnect-dogfood"
+            released_connect_server: false
+            github_shiny: true
+            pypi_shiny: false
+            pypi_rsconnect: false
+            base_test_dir: "./tests/playwright/deploys/express-page_sidebar"
+            app_name: "pypi-shiny-dev-rsconnect"
+            test_shinyappsio: false
+
+          # Dev server, shiny, and rsconnect
+          - name: "dev-shiny-rsconnect-dogfood"
+            released_connect_server: false
+            pypi_shiny: false
+            pypi_rsconnect: false
+            base_test_dir: "./tests/playwright/deploys"
+            test_shinyappsio: true
+
       fail-fast: false
 
     steps:
@@ -27,7 +77,24 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install rsconnect
+      - name: Install pypi shiny and htmltools (uninstall GitHub versions)
+        if: ${{ matrix.config.pypi_shiny }}
+        run: |
+          uv pip uninstall shiny htmltools
+          uv pip install shiny htmltools
+
+      - name: Install GitHub [email protected] and [email protected] (uninstall PyPI versions)
+        if: ${{ matrix.config.github_shiny }}
+        run: |
+          uv pip uninstall shiny htmltools
+          uv pip install "htmltools @ git+https://github.com/posit-dev/[email protected]" "shiny @ git+https://github.com/posit-dev/[email protected]"
+
+      - name: Install rsconnect (PyPI)
+        if: ${{ matrix.config.pypi_rsconnect }}
+        run: |
+          uv pip install rsconnect
+      - name: Install rsconnect (GitHub)
+        if: ${{ ! matrix.config.pypi_rsconnect }}
         run: |
           make ci-install-rsconnect
 
@@ -36,25 +103,27 @@ jobs:
         env:
           DEPLOY_APPS: "false"
         run: |
-          make playwright-deploys SUB_FILE=". -vv"
+          make playwright-deploys TEST_FILE="${{ matrix.config.base_test_dir }} -vv"
 
       - name: Deploy apps and run tests (on `push` or `deploy**` branches)
         env:
           DEPLOY_APPS: "true"
-          DEPLOY_CONNECT_SERVER_URL: "https://rsc.radixu.com/"
-          DEPLOY_CONNECT_SERVER_API_KEY: "${{ secrets.DEPLOY_CONNECT_SERVER_API_KEY }}"
-          DEPLOY_SHINYAPPS_NAME: "${{ secrets.DEPLOY_SHINYAPPS_NAME }}"
-          DEPLOY_SHINYAPPS_TOKEN: "${{ secrets.DEPLOY_SHINYAPPS_TOKEN }}"
-          DEPLOY_SHINYAPPS_SECRET: "${{ secrets.DEPLOY_SHINYAPPS_SECRET }}"
+          DEPLOY_CONNECT_SERVER_URL: "${{ (matrix.config.released_connect_server && 'https://connect.posit.it/') || 'https://rsc.radixu.com/' }}"
+          DEPLOY_CONNECT_SERVER_API_KEY: "${{ (matrix.config.released_connect_server && secrets.DEPLOY_CONNECT_POSIT_SERVER_API_KEY) || secrets.DEPLOY_CONNECT_SERVER_API_KEY }}"
+          DEPLOY_SHINYAPPS_NAME: "${{ matrix.config.test_shinyappsio && matrix.config.shinyapps_name }}"
+          DEPLOY_SHINYAPPS_TOKEN: "${{ matrix.config.test_shinyappsio && matrix.config.shinyapps_token }}"
+          DEPLOY_SHINYAPPS_SECRET: "${{ matrix.config.test_shinyappsio && matrix.config.shinyapps_secret }}"
+          EXPRESS_PAGE_SIDEBAR_NAME: "${{ matrix.config.app_name }}"
+          DEPLOY_GITHUB_REQUIREMENTS_TXT: "${{ !matrix.config.pypi_shiny }}"
         timeout-minutes: 30
         # Given we are waiting for external servers to finish,
         # we can have many local processes waiting for deployment to finish
         run: |
-          make playwright-deploys SUB_FILE=". -vv --numprocesses 12"
+          make playwright-deploys TEST_FILE="${{ matrix.config.base_test_dir }} -vv --numprocesses 12"
 
-      - uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: "playright-deploys-${{ matrix.os }}-${{ matrix.python-version }}-results"
-          path: test-results/
-          retention-days: 5
+      # - uses: actions/upload-artifact@v4
+      #   if: failure()
+      #   with:
+      #     name: "playright-deploys-${{ matrix.os }}-${{ matrix.python-version }}-results"
+      #     path: test-results/
+      #     retention-days: 5
diff --git a/Makefile b/Makefile
@@ -173,6 +173,8 @@ playwright-shiny: FORCE
 
 # end-to-end tests on deployed apps with playwright; (SUB_FILE="" within tests/playwright/deploys/)
 playwright-deploys: FORCE
+	$(MAKE) playwright PYTEST_BROWSERS="$(PYTEST_DEPLOYS_BROWSERS)" TEST_FILE="$(TEST_FILE)"
+playwright-deploys-legacy: FORCE
 	$(MAKE) playwright TEST_FILE="tests/playwright/deploys/$(SUB_FILE)" PYTEST_BROWSERS="$(PYTEST_DEPLOYS_BROWSERS)"
 
 # end-to-end tests on all py-shiny examples with playwright; (SUB_FILE="" within tests/playwright/examples/)

diff --git a/tests/playwright/deploys/express-accordion/test_deploys_express_accordion.py b/tests/playwright/deploys/express-accordion/test_deploys_express_accordion.py
@@ -1,15 +1,15 @@
 import pytest
 from playwright.sync_api import Page
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 from shiny.playwright import controller
 
-app_url = create_deploys_app_url_fixture("shiny_express_accordion")
+app_url = local_deploys_app_url_fixture("shiny_express_accordion")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-dataframe/test_deploys_express_dataframe.py b/tests/playwright/deploys/express-dataframe/test_deploys_express_dataframe.py
@@ -1,15 +1,15 @@
 import pytest
 from playwright.sync_api import Page
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 from shiny.playwright import controller
 
-app_url = create_deploys_app_url_fixture("shiny-express-dataframe")
+app_url = local_deploys_app_url_fixture("shiny-express-dataframe")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-folium/test_deploys_express_folium.py b/tests/playwright/deploys/express-folium/test_deploys_express_folium.py
@@ -1,13 +1,13 @@
 import pytest
 from playwright.sync_api import Page, expect
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
-app_url = create_deploys_app_url_fixture("shiny-express-folium")
+app_url = local_deploys_app_url_fixture("shiny-express-folium")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-page_default/test_deploys_express_page_default.py b/tests/playwright/deploys/express-page_default/test_deploys_express_page_default.py
@@ -1,7 +1,7 @@
 import pytest
 from playwright.sync_api import Page, expect
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
@@ -11,7 +11,7 @@
 
 TIMEOUT = 2 * 60 * 1000
 
-app_url = create_deploys_app_url_fixture("shiny_express_page_default")
+app_url = local_deploys_app_url_fixture("shiny_express_page_default")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-page_fillable/test_deploys_express_page_fillable.py b/tests/playwright/deploys/express-page_fillable/test_deploys_express_page_fillable.py
@@ -1,15 +1,15 @@
 import pytest
 from playwright.sync_api import Page
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 from shiny.playwright import controller
 
-app_url = create_deploys_app_url_fixture("express_page_fillable")
+app_url = local_deploys_app_url_fixture("express_page_fillable")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-page_fluid/test_deploys_express_page_fluid.py b/tests/playwright/deploys/express-page_fluid/test_deploys_express_page_fluid.py
@@ -1,15 +1,15 @@
 import pytest
 from playwright.sync_api import Page
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 from shiny.playwright import controller
 
-app_url = create_deploys_app_url_fixture("express_page_fluid")
+app_url = local_deploys_app_url_fixture("express_page_fluid")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/express-page_sidebar/test_deploys_express_page_sidebar.py b/tests/playwright/deploys/express-page_sidebar/test_deploys_express_page_sidebar.py
@@ -1,15 +1,20 @@
+import os
+
 import pytest
 from playwright.sync_api import Page
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 from shiny.playwright import controller
 
-app_url = create_deploys_app_url_fixture("express_page_sidebar")
+app_url = local_deploys_app_url_fixture(
+    # Possibly use a different app name given by an GHA env var
+    os.getenv("EXPRESS_PAGE_SIDEBAR_NAME", "express_page_sidebar")
+)
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/plotly/test_plotly_app.py b/tests/playwright/deploys/plotly/test_plotly_app.py
@@ -1,14 +1,14 @@
 import pytest
 from playwright.sync_api import Page, expect
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
 TIMEOUT = 2 * 60 * 1000
-app_url = create_deploys_app_url_fixture("example_deploy_app_a1")
+app_url = local_deploys_app_url_fixture("example_deploy_app_a1")
 
 
 @skip_if_not_chrome

diff --git a/tests/playwright/deploys/shiny-client-console-error/test_shiny_client_error.py b/tests/playwright/deploys/shiny-client-console-error/test_shiny_client_error.py
@@ -1,13 +1,13 @@
 import pytest
 from playwright.sync_api import Page, expect
 from utils.deploy_utils import (
-    create_deploys_app_url_fixture,
+    local_deploys_app_url_fixture,
     reruns,
     reruns_delay,
     skip_if_not_chrome,
 )
 
-app_url = create_deploys_app_url_fixture("shiny_client_console_error")
+app_url = local_deploys_app_url_fixture("shiny_client_console_error")
 
 
 @skip_if_not_chrome