feat: Optimize test runner to work more smoothly

.github/workflows/test.yml

@@ -107,7 +107,7 @@ jobs:
                     curl -sL "https://github.com/goss-org/goss/releases/download/${{ env.GOSSVERSION }}/goss-${GOSS_ARCH}" -o goss
                     chmod +x goss
                     go build cmd/tests/test-features.go
-                    ./test-features -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch}}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug
+                    ./test-features -c -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch }}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug
             -   name: Cleanup
                 run: |
                     rm -rf .testbed

README.md

@@ -18,7 +18,6 @@ Currently supported cloud flavours are:
 Following features and tools are supported:
 * 🐟 Fish Shell
 * 📷 AzCopy
-* 🔐 Bitwarden
 * 🪪 Certificates
 * ⚙️ Direnv
 * ⛵️ Helm
@@ -52,7 +51,6 @@ Following features and tools are supported:
 * [Features](#features)
     * [Fish Shell](#_fish)
     * [AzCopy](#azcopy)
-    * [Bitwarden](#bitwarden)
     * [Certificates](#certificates)
     * [Direnv](#direnv)
     * [Helm](#helm)
@@ -373,15 +371,6 @@ Installs [AzCopy](https://github.com/Azure/azure-storage-azcopy)
 * USE_azcopy: Enable this feature
 * DEBUG_azcopy: Debug this feature
 
-### <a id="bitwarden"></a> Bitwarden
-
-Installs the [Bitwarden CLI](https://bitwarden.com/help/cli/)
-
-#### Configuration
-
-* USE_bitwarden: Enable this feature
-* DEBUG_bitwarden: Debug this feature
-
 ### <a id="certificates"></a> Certificates
 
 Adds specified trusted certificate authorities into the container
@@ -728,20 +717,19 @@ To build all flavours with the same tag, use
 ## Testing
 
 To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data
-(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables.
+(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called
+ a "testbed" directory.
 
 First, you need to compile the test runner:
 
-    cd tests
-    docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features
+    docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features
 
 After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from
 the [Goss site](https://github.com/goss-org/goss) and put it somewhere local.
 
 Once that is done, run the tests like following:
 
-    cd tests
-    ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
+    ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
 
 This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all
 features together for integration testing. Check out `test-features --help` for other options.
@@ -755,12 +743,19 @@ When the testrunner encounters such file it will check if CloudControl fails to
 
 You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it.
 
+### Unstable tests
+
+As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these
+situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text
+into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown.
+
 ### Test debugging
 
-To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues.
+To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter
+to specify the specific feature to test and use the -x parameter to stop testing if one test fails.
 
-Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the
-tests locally.
+When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so
+you can inspect the failing container as well.
 
 ## Building documentation ##
 

README.md.gotmpl

@@ -384,20 +384,19 @@ To build all flavours with the same tag, use
 ## Testing
 
 To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data
-(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables.
+(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called
+ a "testbed" directory.
 
 First, you need to compile the test runner:
 
-    cd tests
-    docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features
+    docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features
 
 After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from
 the [Goss site](https://github.com/goss-org/goss) and put it somewhere local.
 
 Once that is done, run the tests like following:
 
-    cd tests
-    ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
+    ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary]
 
 This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all
 features together for integration testing. Check out `test-features --help` for other options.
@@ -411,12 +410,19 @@ When the testrunner encounters such file it will check if CloudControl fails to
 
 You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it.
 
+### Unstable tests
+
+As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these
+situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text
+into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown.
+
 ### Test debugging
 
-To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues.
+To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter
+to specify the specific feature to test and use the -x parameter to stop testing if one test fails.
 
-Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the
-tests locally.
+When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so
+you can inspect the failing container as well.
 
 ## Building documentation ##
 

cmd/tests/test-features.go

@@ -138,8 +138,7 @@ func main() {
 	includeFeatures := parser.List("n", "include", &argparse.Options{Help: "Only include these features when testing"})
 	excludeFeatures := parser.List("e", "exclude", &argparse.Options{Help: "Exclude these features when testing"})
 	skipIntegrationTest := parser.Flag("s", "skip-integration", &argparse.Options{
-		Help:    "Skip integration test at the end",
-		Default: false,
+		Help: "Skip integration test at the end",
 	})
 	maxWait := parser.Int("w", "wait", &argparse.Options{
 		Help:    "Maximum number of seconds goss_wait should wait",
@@ -149,6 +148,12 @@ func main() {
 		Help:    "Loglevel to use",
 		Default: "error",
 	})
+	failFast := parser.Flag("x", "failfast", &argparse.Options{
+		Help: "Whether to stop testing directly when one test failes",
+	})
+	cleanup := parser.Flag("c", "cleanup", &argparse.Options{
+		Help: "Clean up container in case of error",
+	})
 
 	err := parser.Parse(os.Args)
 	if err != nil {
@@ -272,7 +277,7 @@ func main() {
 
 	for _, feature := range featuresToTest {
 		var featureTimer = time.Now()
-		if err := lib.TestFeature(feature, testBed, containerAdapter); err != nil {
+		if err := lib.TestFeature(feature, testBed, containerAdapter, *cleanup); err != nil {
 			logrus.Errorf(
 				"❌  %s (%s) ⏱  %ds :\n\n%s\n",
 				feature.Name,
@@ -285,6 +290,10 @@ func main() {
 				ErrorMessage:   err.Error(),
 				ElapsedSeconds: int(math.Round(time.Since(featureTimer).Seconds())),
 			})
+			if *failFast {
+				logrus.Error("Failfast activated, so failing directly.")
+				os.Exit(1)
+			}
 		} else {
 			logrus.Infof(
 				"✅  %s (%s) ⏱  %ds",

feature/azcopy/goss/.might-fail

@@ -0,0 +1 @@
+Might fail because of https://github.com/Azure/azure-storage-azcopy/issues/2483

feature/azcopy/goss/goss.yaml

@@ -1,6 +1,6 @@
 command:
     azcopy:
-        exec: "/home/cloudcontrol/bin/azcopy --version"
+        exec: "/home/cloudcontrol/bin/azcopy --help"
         exit-status: 0
         stdout:
-            - azcopy version
+            - AzCopy

go.mod

@@ -6,25 +6,27 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/akamensky/argparse v1.4.0
-	github.com/docker/docker v20.10.22+incompatible
-	github.com/opencontainers/image-spec v1.0.2
-	github.com/otiai10/copy v1.9.0
-	github.com/sirupsen/logrus v1.9.0
+	github.com/docker/docker v24.0.7+incompatible
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
+	github.com/otiai10/copy v1.14.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/thoas/go-funk v0.9.3
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/containerd/containerd v1.7.11 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/uuid v1.1.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
-	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.0 // indirect
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
@@ -33,11 +35,12 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
-	golang.org/x/crypto v0.3.0 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.2.0 // indirect
-	golang.org/x/sys v0.2.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/tools v0.10.0 // indirect
 	gotest.tools/v3 v3.4.0 // indirect
 )