From 0493dc7a372da858a4acde599cd3161469b56859 Mon Sep 17 00:00:00 2001 From: Dennis Ploeger Date: Mon, 11 Dec 2023 13:28:29 +0100 Subject: [PATCH 1/2] feat: Optimized test runner to work more smoothly Added .might-fail feature go dep updates BREAKING CHANGE: Removed bitwarden feature --- .github/workflows/test.yml | 2 +- README.md | 33 ++++++-------- README.md.gotmpl | 22 ++++++---- cmd/tests/test-features.go | 15 +++++-- feature/azcopy/goss/.might-fail | 1 + feature/azcopy/goss/goss.yaml | 4 +- go.mod | 27 +++++++----- go.sum | 32 ++++++++++++++ internal/tests/lib/container/docker.go | 3 +- internal/tests/lib/testRunner.go | 61 ++++++++++++++++++++++---- 10 files changed, 145 insertions(+), 55 deletions(-) create mode 100644 feature/azcopy/goss/.might-fail diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b17dd9c..9f306b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,7 +107,7 @@ jobs: curl -sL "https://github.com/goss-org/goss/releases/download/${{ env.GOSSVERSION }}/goss-${GOSS_ARCH}" -o goss chmod +x goss go build cmd/tests/test-features.go - ./test-features -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch}}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug + ./test-features -c -f ${{ matrix.flavour }} -i test-image-${{ matrix.flavour }}-${{ matrix.arch }}:local -p ${{ matrix.arch }} -t ${PWD}/.testbed -g ${PWD}/goss -l debug - name: Cleanup run: | rm -rf .testbed diff --git a/README.md b/README.md index 3a2bdf6..c9c43b6 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ Currently supported cloud flavours are: Following features and tools are supported: * 🐟 Fish Shell * 📷 AzCopy -* 🔐 Bitwarden * 🪪 Certificates * ⚙️ Direnv * ⛵️ Helm @@ -52,7 +51,6 @@ Following features and tools are supported: * [Features](#features) * [Fish Shell](#_fish) * [AzCopy](#azcopy) - * [Bitwarden](#bitwarden) * [Certificates](#certificates) * [Direnv](#direnv) * [Helm](#helm) @@ -373,15 +371,6 @@ Installs [AzCopy](https://github.com/Azure/azure-storage-azcopy) * USE_azcopy: Enable this feature * DEBUG_azcopy: Debug this feature -### Bitwarden - -Installs the [Bitwarden CLI](https://bitwarden.com/help/cli/) - -#### Configuration - -* USE_bitwarden: Enable this feature -* DEBUG_bitwarden: Debug this feature - ### Certificates Adds specified trusted certificate authorities into the container @@ -728,20 +717,19 @@ To build all flavours with the same tag, use ## Testing To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data -(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. +(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called + a "testbed" directory. First, you need to compile the test runner: - cd tests - docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features + docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from the [Goss site](https://github.com/goss-org/goss) and put it somewhere local. Once that is done, run the tests like following: - cd tests - ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary] + ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary] This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all features together for integration testing. Check out `test-features --help` for other options. @@ -755,12 +743,19 @@ When the testrunner encounters such file it will check if CloudControl fails to You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it. +### Unstable tests + +As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these +situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text +into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown. + ### Test debugging -To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues. +To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter +to specify the specific feature to test and use the -x parameter to stop testing if one test fails. -Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the -tests locally. +When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so +you can inspect the failing container as well. ## Building documentation ## diff --git a/README.md.gotmpl b/README.md.gotmpl index 01b3b70..07fb2ee 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -384,20 +384,19 @@ To build all flavours with the same tag, use ## Testing To run the test suite for a specific flavour, you need to create a local directory that holds flavour-specific data -(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. +(e.g. keys for authentication) and optionally an .env-file with flavour-specific environment variables. This is called + a "testbed" directory. First, you need to compile the test runner: - cd tests - docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features + docker run --rm -e GOOS=[os, e.g. darwin, linux, windows] -e GOARCH=[architecture, e.g. arm64, amd64] -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.19-alpine go build -o test-features cmd/tests/test-features After that, download the latest goss binary for the target architecture you will test (linux/amd64 or linux/arm64) from the [Goss site](https://github.com/goss-org/goss) and put it somewhere local. Once that is done, run the tests like following: - cd tests - ./test-features -f [flavour] -i [image:tag] -t [path to flavour-data] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary] + ./test-features -f [flavour] -i [image:tag] -t [path to testbed directory] -p [test architecture, e.g. linux/amd64] -g [path to the goss binary] This will run the tests of all features that supply a test suite one by one and, if all succeed, will test all features together for integration testing. Check out `test-features --help` for other options. @@ -411,12 +410,19 @@ When the testrunner encounters such file it will check if CloudControl fails to You can add a regular expression pattern into `.will-fail` to test if the container or command output matches it. +### Unstable tests + +As we're dealing with a lot of moving targets in the features, sometimes a test might not be reliable. For these +situations we support a .might-fail file. Just add it as a text file into the test suite subdirectory and put some text +into it describing the problem. Failed test won't fail the test suite then but instead the description will be shown. + ### Test debugging -To check why a test failed, run the test-runner using the -x bash parameter to see the different commands it issues. +To check why a test failed, use the -l parameter to enable debug logging. Additionally, you can use the -n parameter +to specify the specific feature to test and use the -x parameter to stop testing if one test fails. -Then, take the failing command and instead of `dgoss run` execute `docker run` with the same arguments to analyze the -tests locally. +When a test fails, the test container will not be removed automatically (unless you specified the -c parameter), so +you can inspect the failing container as well. ## Building documentation ## diff --git a/cmd/tests/test-features.go b/cmd/tests/test-features.go index 2282ef3..7250e7a 100644 --- a/cmd/tests/test-features.go +++ b/cmd/tests/test-features.go @@ -138,8 +138,7 @@ func main() { includeFeatures := parser.List("n", "include", &argparse.Options{Help: "Only include these features when testing"}) excludeFeatures := parser.List("e", "exclude", &argparse.Options{Help: "Exclude these features when testing"}) skipIntegrationTest := parser.Flag("s", "skip-integration", &argparse.Options{ - Help: "Skip integration test at the end", - Default: false, + Help: "Skip integration test at the end", }) maxWait := parser.Int("w", "wait", &argparse.Options{ Help: "Maximum number of seconds goss_wait should wait", @@ -149,6 +148,12 @@ func main() { Help: "Loglevel to use", Default: "error", }) + failFast := parser.Flag("x", "failfast", &argparse.Options{ + Help: "Whether to stop testing directly when one test failes", + }) + cleanup := parser.Flag("c", "cleanup", &argparse.Options{ + Help: "Clean up container in case of error", + }) err := parser.Parse(os.Args) if err != nil { @@ -272,7 +277,7 @@ func main() { for _, feature := range featuresToTest { var featureTimer = time.Now() - if err := lib.TestFeature(feature, testBed, containerAdapter); err != nil { + if err := lib.TestFeature(feature, testBed, containerAdapter, *cleanup); err != nil { logrus.Errorf( "❌ %s (%s) ⏱ %ds :\n\n%s\n", feature.Name, @@ -285,6 +290,10 @@ func main() { ErrorMessage: err.Error(), ElapsedSeconds: int(math.Round(time.Since(featureTimer).Seconds())), }) + if *failFast { + logrus.Error("Failfast activated, so failing directly.") + os.Exit(1) + } } else { logrus.Infof( "✅ %s (%s) ⏱ %ds", diff --git a/feature/azcopy/goss/.might-fail b/feature/azcopy/goss/.might-fail new file mode 100644 index 0000000..545ab2e --- /dev/null +++ b/feature/azcopy/goss/.might-fail @@ -0,0 +1 @@ +Might fail because of https://github.com/Azure/azure-storage-azcopy/issues/2483 \ No newline at end of file diff --git a/feature/azcopy/goss/goss.yaml b/feature/azcopy/goss/goss.yaml index 1fa0616..cd84c9f 100644 --- a/feature/azcopy/goss/goss.yaml +++ b/feature/azcopy/goss/goss.yaml @@ -1,6 +1,6 @@ command: azcopy: - exec: "/home/cloudcontrol/bin/azcopy --version" + exec: "/home/cloudcontrol/bin/azcopy --help" exit-status: 0 stdout: - - azcopy version + - AzCopy diff --git a/go.mod b/go.mod index a432475..4c47b05 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 github.com/Masterminds/sprig/v3 v3.2.3 github.com/akamensky/argparse v1.4.0 - github.com/docker/docker v20.10.22+incompatible - github.com/opencontainers/image-spec v1.0.2 - github.com/otiai10/copy v1.9.0 - github.com/sirupsen/logrus v1.9.0 + github.com/docker/docker v24.0.7+incompatible + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b + github.com/otiai10/copy v1.14.0 + github.com/sirupsen/logrus v1.9.3 github.com/thoas/go-funk v0.9.3 gopkg.in/yaml.v3 v3.0.1 ) @@ -17,14 +17,16 @@ require ( require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/containerd/containerd v1.7.11 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/uuid v1.1.1 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect - github.com/imdario/mergo v0.3.11 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect @@ -33,11 +35,12 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect - golang.org/x/crypto v0.3.0 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/sys v0.2.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.10.0 // indirect gotest.tools/v3 v3.4.0 // indirect ) diff --git a/go.sum b/go.sum index 9debe30..0c26192 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,14 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= +github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= +github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -18,6 +24,8 @@ github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk= github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -28,10 +36,14 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= @@ -46,13 +58,18 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -61,6 +78,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -80,10 +99,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -92,10 +115,14 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -107,6 +134,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -123,6 +152,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -133,6 +164,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= diff --git a/internal/tests/lib/container/docker.go b/internal/tests/lib/container/docker.go index 46fe9da..0faec44 100644 --- a/internal/tests/lib/container/docker.go +++ b/internal/tests/lib/container/docker.go @@ -72,7 +72,7 @@ func (d DockerAdapter) FindImage(image string, platform string) error { if len(localImages) == 0 { remoteImages, err := dockerCli.ImageSearch(context.Background(), image, types.ImageSearchOptions{Limit: 1}) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "404") { panic(fmt.Sprintf("Can not search remote images: %s", err.Error())) } if len(remoteImages) == 0 { @@ -137,6 +137,7 @@ func (d DockerAdapter) StopContainer(containerID string) error { } func (d DockerAdapter) RunCommand(containerID string, cmd []string) (string, error) { + logrus.Debugf("Running command %s on container %s", strings.Join(cmd, " "), containerID) dockerCli := d.getClient() var executeID string if idResponse, err := dockerCli.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{ diff --git a/internal/tests/lib/testRunner.go b/internal/tests/lib/testRunner.go index e18187b..739e2f6 100644 --- a/internal/tests/lib/testRunner.go +++ b/internal/tests/lib/testRunner.go @@ -26,7 +26,7 @@ file: ` // TestFeature tests a specific feature. -func TestFeature(feature Feature, testBed TestBed, containerAdapter container.AdapterBase) error { +func TestFeature(feature Feature, testBed TestBed, containerAdapter container.AdapterBase, cleanup bool) error { var testSuites []string if testSuitesGlob, err := filepath.Glob(filepath.Join(feature.FullPath, "goss*")); err != nil { return fmt.Errorf("error searching for testsuites in %s: %w", feature.FullPath, err) @@ -47,8 +47,11 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad containerErr := func() error { var testDir string + var hasProblems = false var willFail = false var failPattern = ".*" + var mightFail = false + var mightFailDescription = "" if _, err := os.Stat(filepath.Join(testSuite, ".will-fail")); err == nil { willFail = true @@ -60,6 +63,16 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad } } + if _, err := os.Stat(filepath.Join(testSuite, ".might-fail")); err == nil { + mightFail = true + logrus.Debug("Found .might-fail file, test might fail.") + if tmp, err := os.ReadFile(filepath.Join(testSuite, ".might-fail")); err != nil { + return fmt.Errorf("loading the contents of %s failed: %w", filepath.Join(testSuite, ".might-fail"), err) + } else { + mightFailDescription = strings.TrimSuffix(string(tmp), "\n") + } + } + if tempReturn, err := os.MkdirTemp("", "cctest-*"); err != nil { return fmt.Errorf("error creating temporary directory for testbed: %w", err) } else { @@ -87,7 +100,7 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad if _, err := fileutils.CopyFile( filepath.Join(testSuite, "goss.yaml"), filepath.Join(testDir, "goss.yaml"), - ); err != nil && !willFail { + ); err != nil && !willFail && !mightFail { return fmt.Errorf("can not copy goss.yaml: %w", err) } @@ -144,15 +157,24 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad binds, testBed.Platform, ); err != nil { + hasProblems = true return fmt.Errorf("error starting container: %w", err) } else { + logrus.Debugf("Started container %s", startedContainerId) containerId = startedContainerId } // deferring the removal of the container defer func() { - if err := containerAdapter.StopContainer(containerId); err != nil { - panic(fmt.Errorf("can not stop goss container: %w", err)) + if cleanup || !hasProblems { + logrus.Debugf("Stopping container %s", containerId) + if err := containerAdapter.StopContainer(containerId); err != nil { + panic(fmt.Errorf("can not stop goss container: %w", err)) + } else { + logrus.Debugf("Stopped container with id %s", containerId) + } + } else { + logrus.Debugf("Not stopping container because cleanup was disabled and problems exist.") } }() @@ -168,11 +190,14 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad var runCommandError *container.RunCommandError switch { case errors.As(err, &runCommandError): - if !willFail { + if mightFail { + logrus.Warnf("Waiting for Goss in %s failed with %s, but it might fail because: \n%s", testSuite, err.Error(), mightFailDescription) + } else if !willFail { + hasProblems = true return &GossError{ returnCode: runCommandError.ReturnCode, logOutput: fmt.Sprintf( - "Command output:\n%s\n\nContainer output:\n%s\n&s", + "Command output:\n%s\n\nContainer output:\n%s\n%s", runCommandError.CommandOutput, runCommandError.ContainerOutput, runCommandError.Error(), @@ -182,6 +207,7 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad failPattern, []byte(fmt.Sprintf("%s%s", runCommandError.CommandOutput, runCommandError.ContainerOutput)), ); err != nil || !found { + hasProblems = true return &GossError{ returnCode: 0, logOutput: fmt.Sprintf( @@ -194,9 +220,12 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad } } default: - return err + if !willFail && !mightFail { + return err + } } } else if err == nil && willFail { + hasProblems = true return &GossError{ returnCode: 0, logOutput: fmt.Sprintf("Container was created successfully when it shouldn't have been: %s", waitOutput), @@ -216,7 +245,10 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad "documentation", }, ); err != nil { - if !willFail { + if mightFail { + logrus.Warnf("Testing %s failed with %s, but it might fail because: \n%s", testSuite, err.Error(), mightFailDescription) + } else if !willFail { + hasProblems = true var runCommandError *container.RunCommandError switch { case errors.As(err, &runCommandError): @@ -229,10 +261,13 @@ func TestFeature(feature Feature, testBed TestBed, containerAdapter container.Ad ), } default: - return err + if !willFail && !mightFail { + return err + } } } } else if err == nil && willFail { + hasProblems = true return &GossError{ returnCode: 0, logOutput: "Container was created successfully when it shouldn't have been.", @@ -296,6 +331,14 @@ func IntegrationTests(features []Feature, testBed TestBed, containerAdapter cont ) continue } + if _, err := os.Stat(filepath.Join(testSuite, ".might-fail")); err == nil { + logrus.Infof( + "Test %s for feature %s is ignored for integration testing, because it might fail", + filepath.Base(testSuite), + feature.Name, + ) + continue + } envs = append(envs, fmt.Sprintf("USE_%s=yes", feature.Name)) for _, envFile := range []string{ From ace0bc32e19798afd52edf91c826f34430b7c045 Mon Sep 17 00:00:00 2001 From: Dennis Ploeger Date: Mon, 18 Dec 2023 09:28:57 +0100 Subject: [PATCH 2/2] fix: Removed bitwarden because of very unstable implementation BREAKING CHANGE: Bitwarden feature not available anymore --- feature/bitwarden/feature.yaml | 3 --- feature/bitwarden/goss/goss.yaml | 5 ----- feature/bitwarden/install.sh | 11 ----------- 3 files changed, 19 deletions(-) delete mode 100644 feature/bitwarden/feature.yaml delete mode 100644 feature/bitwarden/goss/goss.yaml delete mode 100644 feature/bitwarden/install.sh diff --git a/feature/bitwarden/feature.yaml b/feature/bitwarden/feature.yaml deleted file mode 100644 index 7b2bf02..0000000 --- a/feature/bitwarden/feature.yaml +++ /dev/null @@ -1,3 +0,0 @@ -icon: "🔐" -title: "Bitwarden" -description: "Installs the [Bitwarden CLI](https://bitwarden.com/help/cli/)" diff --git a/feature/bitwarden/goss/goss.yaml b/feature/bitwarden/goss/goss.yaml deleted file mode 100644 index 378f2fc..0000000 --- a/feature/bitwarden/goss/goss.yaml +++ /dev/null @@ -1,5 +0,0 @@ -command: - bw: - exec: "/home/cloudcontrol/bin/bw --version" - exit-status: 0 - timeout: 30000 diff --git a/feature/bitwarden/install.sh b/feature/bitwarden/install.sh deleted file mode 100644 index 1222b16..0000000 --- a/feature/bitwarden/install.sh +++ /dev/null @@ -1,11 +0,0 @@ -. /feature-installer-utils.sh - -FLAVOUR="X$(cat /home/cloudcontrol/flavour)X" -if [[ "X${FLAVOUR}X" =~ X(azure|simple|tanzu|gcloud)X ]] -then - execHandle "Installing required libraries" sudo apk add nodejs npm -fi - -cd ~ || exit 1 -execHandle "Installing bitwarden client" npm install @bitwarden/cli -execHandle "Linking bitwarden client" ln -s /home/cloudcontrol/node_modules/.bin/bw bin/bw