Move integration tests to ./tests (#1490)

This consolidates integration tests (currently located in `./examples` and `./provider`) under `./tests`. This is prep work for #1474, and the net result is a clearer distinction between "fast" and "slow" tests meant to be run during the "prerequisites" and "tests" steps in CI. Everything under `./provider` can be considered "fast" and everything under `./tests` can be considered "slow" and worthy of sharding. Specifically: * Everything under `./examples/tests` is moved to `./tests/testdata/programs` as these were assumed to not be user-facing. * Upgrade tests under `./provider` are also moved to `./tests` but tagged in a way that they continue to run as their own job. (These are the new `upgrade_test.go` and `validation_test.go` files.) * `examples/utils/utils.go` moves to `tests/internal/utils`. * `TestEksAuthModeUpgrade` wasn't re-enabled as part of #1387, so I re-enabled and re-recorded it. * `go.mod` is consolidated at the repo root to keep dependencies consistent across the provider and tests. (There are a ton of additional DX advantages to structuring things this way.) This project is not meant to be consumed as a library so this is an acceptable breaking change.
pulumi · Nov 14, 2024 · e666a6f · e666a6f
1 parent 2f95d29
commit e666a6f
Show file tree

Hide file tree

Showing 251 changed files with 2,747 additions and 3,786 deletions.
diff --git a/.github/workflows/eks-cron.yml b/.github/workflows/eks-cron.yml
@@ -263,12 +263,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=nodejs
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 10
@@ -368,12 +368,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=python
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:
@@ -470,7 +470,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
   test-go:
     name: Run Go Tests
     needs:
@@ -559,7 +559,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
 name: cron
 "on":
   schedule:

diff --git a/.github/workflows/eks-record.yml b/.github/workflows/eks-record.yml
@@ -4,7 +4,7 @@ on:
     inputs:
       runTests:
         type: string
-        description: "The tests in examples to record snapshots for, as a regex for `go test -run`. Defaults to `TestExamplesUpgrades`"
+        description: "The tests to record snapshots for, as a regex for `go test -run`. Defaults to `TestExamplesUpgrades`"
         required: true
         default: "TestExamplesUpgrades"
       targetBranch:
@@ -224,7 +224,7 @@ jobs:
 
       - name: Record provider test snapshots
         run: |
-          cd provider && go test -tags nodejs -run '${{ inputs.runTests }}' -provider-snapshot -v -json -count=1 -cover -timeout 4h -parallel 20 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+          cd tests && go test -tags nodejs -run '${{ inputs.runTests }}' -provider-snapshot -v -json -count=1 -cover -timeout 4h -parallel 20 . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
         # TODO,tkappler Not sure why this is happening, but gotestfmt-action seems to download and unpack into the repository's root folder.
         # https://github.com/GoTestTools/gotestfmt-action/blob/v2/index.js#L6

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -279,12 +279,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=nodejs
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 10
@@ -384,12 +384,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=python
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 4
@@ -487,7 +487,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
   test-go:
     name: Run Go Tests
     needs:
@@ -576,7 +576,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
 name: master
 "on":
   push:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -271,12 +271,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=nodejs
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 10
@@ -376,12 +376,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=python
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 4
@@ -479,7 +479,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
   test-go:
     name: Run Go Tests
     needs:
@@ -568,7 +568,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
 name: release
 "on":
   push:

diff --git a/.github/workflows/run-acceptance-tests.yml b/.github/workflows/run-acceptance-tests.yml
@@ -401,12 +401,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=nodejs
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 10
@@ -507,12 +507,12 @@ jobs:
         id: test_split
         uses: hashicorp-forge/[email protected]
         with:
-          working-directory: examples
+          working-directory: tests
           flags: -tags=python
           total: ${{ matrix.total }}
           index: ${{ matrix.index }}
       - name: Run tests
-        run: cd examples && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel 6 . --run="${{ steps.test_split.outputs.run}}" 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       max-parallel: 4
@@ -611,7 +611,7 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
   test-go:
     name: Run Go Tests
     needs:
@@ -701,4 +701,4 @@ jobs:
           cd ${{ github.workspace }}/bin
           yarn install && yarn link @pulumi/eks
       - name: Run tests
-        run: cd examples && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+        run: cd tests && go test -tags=go -v -json -count=1 -cover -timeout 3h -parallel 6 . 2>&1 | tee /tmp/gotest.log | gotestfmt
diff --git a/Makefile b/Makefile
@@ -167,36 +167,33 @@ dist:: dist/${GZIP_PREFIX}-darwin-amd64.tar.gz
 dist:: dist/${GZIP_PREFIX}-darwin-arm64.tar.gz
 dist:: dist/${GZIP_PREFIX}-windows-amd64.tar.gz
 
-test_build::
-	cd examples/utils/testvpc && yarn install && yarn run tsc
-
 test_nodejs:: PATH := $(WORKING_DIR)/bin:$(PATH)
 test_nodejs:: provider install_nodejs_sdk
-	cd examples && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 test_nodejs_upgrade:: PATH := $(WORKING_DIR)/bin:$(PATH)
 test_nodejs_upgrade:: provider install_nodejs_sdk
-	cd provider && go test -tags=nodejs -v -json -count=1 -cover -timeout 3h -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -v -run Upgrade -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 test_python:: install_provider test_build
-	cd examples && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -tags=python -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 test_dotnet:: install_provider
-	cd examples && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -tags=dotnet -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 test_java:: install_provider
-	cd examples && go test -tags=java -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -tags=java -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 test_unit_tests:
 	cd nodejs/eks && \
 		yarn install && \
 		yarn run test
 
 specific_test:: install_nodejs_sdk test_build
-	cd examples && go test -tags=$(LanguageTags) -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . --run=TestAcc$(TestName) 2>&1 | tee /tmp/gotest.log | gotestfmt
+	cd tests && go test -tags=$(LanguageTags) -v -json -count=1 -cover -timeout 3h -parallel ${TESTPARALLELISM} . --run=TestAcc$(TestName) 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 specific_test_local:: install_nodejs_sdk test_build
-	cd examples && go test -tags=$(LanguageTags) -v -count=1 -cover -timeout 3h . --run=TestAcc$(TestName)
+	cd tests && go test -tags=$(LanguageTags) -v -count=1 -cover -timeout 3h . --run=TestAcc$(TestName)
 
 dev:: lint build_nodejs
 test:: test_nodejs