From b1946b708ddcb306ca01c036659855efd8a1c6aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 23 Oct 2024 14:33:45 +0200 Subject: [PATCH 1/6] Makefile: implement `make db-tests` enables the option to run the DB tests locally that are executed in the github actions --- .github/workflows/tests.yml | 11 ++--- Makefile | 43 ++++++++++++++++++- .../main_test.go | 6 +-- cmd/osbuild-service-maintenance/db_test.go | 7 ++- .../jobqueue/jobqueuetest/jobqueuetest.go | 10 +++++ tools/dbtest-entrypoint.sh | 5 +++ tools/dbtest-prepare-env.sh | 6 +++ tools/dbtest-run-migrations.sh | 4 ++ 8 files changed, 74 insertions(+), 18 deletions(-) create mode 100755 tools/dbtest-entrypoint.sh create mode 100755 tools/dbtest-prepare-env.sh create mode 100755 tools/dbtest-run-migrations.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dfb9cb9d07..455eee6389 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,14 +79,9 @@ jobs: PGHOST: localhost PGPORT: 5432 run: | - WORKDIR=$(readlink -f pkg/jobqueue/dbjobqueue/schemas) - pushd $(mktemp -d) - go mod init temp - go install github.com/jackc/tern@latest - $(go env GOPATH)/bin/tern migrate -m "$WORKDIR" - popd - - run: go test -tags=integration ./cmd/osbuild-composer-dbjobqueue-tests - - run: go test -tags=integration ./cmd/osbuild-service-maintenance + ./tools/dbtest-prepare-env.sh + ./tools/dbtest-run-migrations.sh + - run: ./tools/dbtest-entrypoint.sh python-lint: name: "🐍 Lint python scripts" diff --git a/Makefile b/Makefile index c70549db88..cafa075169 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,10 @@ SRCDIR ?= . RST2MAN ?= rst2man +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + # see https://hub.docker.com/r/docker/golangci-lint/tags # v1.55 to get golang 1.21 (1.21.3) # v1.53 to get golang 1.20 (1.20.5) @@ -164,7 +168,7 @@ install: build systemctl daemon-reload .PHONY: clean -clean: +clean: db-tests-prune rm -rf $(BUILDDIR)/bin/ rm -rf $(CURDIR)/rpmbuild rm -rf container_composer_golangci_built.info @@ -236,7 +240,6 @@ worker-key-pair: ca rm /etc/osbuild-composer/worker-csr.pem .PHONY: unit-tests -.ONESHELL: unit-tests: go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$$(go list ./... | tr "\n" ",") ./... # go modules with go.mod in subdirs are not tested automatically @@ -248,6 +251,42 @@ coverage-report: unit-tests go tool cover -o coverage.html -html coverage.txt go tool cover -o coverage_splunk_logger.html -html coverage_splunk_logger.txt +CONTAINER_EXECUTABLE ?= podman + +.PHONY: db-tests-prune +db-tests-prune: + -$(CONTAINER_EXECUTABLE) stop composer-test-db + -$(CONTAINER_EXECUTABLE) rm composer-test-db + +CHECK_DB_PORT_READY=$(CONTAINER_EXECUTABLE) exec composer-test-db pg_isready -d osbuildcomposer +CHECK_DB_UP=$(CONTAINER_EXECUTABLE) exec composer-test-db psql -U postgres -d osbuildcomposer -c "SELECT 1" + +.PHONY: db-tests +db-tests: + -$(CONTAINER_EXECUTABLE) stop composer-test-db 2>/dev/null || echo "DB already stopped" + -$(CONTAINER_EXECUTABLE) rm composer-test-db 2>/dev/null || echo "DB already removed" + $(CONTAINER_EXECUTABLE) run -d \ + --name composer-test-db \ + --env POSTGRES_PASSWORD=foobar \ + --env POSTGRES_DB=osbuildcomposer \ + --publish 5432:5432 \ + postgres:12 + echo "Waiting for DB" + until $(CHECK_DB_PORT_READY) ; do sleep 1; done + until $(CHECK_DB_UP) ; do sleep 1; done + env PGPASSWORD=foobar \ + PGDATABASE=osbuildcomposer \ + PGUSER=postgres \ + PGHOST=localhost \ + PGPORT=5432 \ + ./tools/dbtest-run-migrations.sh + ./tools/dbtest-entrypoint.sh + # we'll leave the composer-test-db container running + # for easier inspection is something fails + +.PHONY: test +test: unit-tests db-tests # run all tests + # # Building packages # diff --git a/cmd/osbuild-composer-dbjobqueue-tests/main_test.go b/cmd/osbuild-composer-dbjobqueue-tests/main_test.go index 2eabc22479..24e8deffc9 100644 --- a/cmd/osbuild-composer-dbjobqueue-tests/main_test.go +++ b/cmd/osbuild-composer-dbjobqueue-tests/main_test.go @@ -14,12 +14,10 @@ import ( "github.com/osbuild/osbuild-composer/internal/jobqueue/jobqueuetest" ) -const url = "postgres://postgres:foobar@localhost:5432/osbuildcomposer" - func TestJobQueueInterface(t *testing.T) { makeJobQueue := func() (jobqueue.JobQueue, func(), error) { // clear db before each run - conn, err := pgx.Connect(context.Background(), url) + conn, err := pgx.Connect(context.Background(), jobqueuetest.TestDbURL()) if err != nil { return nil, nil, err } @@ -35,7 +33,7 @@ func TestJobQueueInterface(t *testing.T) { return nil, nil, err } - q, err := dbjobqueue.New(url) + q, err := dbjobqueue.New(jobqueuetest.TestDbURL()) if err != nil { return nil, nil, err } diff --git a/cmd/osbuild-service-maintenance/db_test.go b/cmd/osbuild-service-maintenance/db_test.go index 5176453204..e81ebed2bd 100644 --- a/cmd/osbuild-service-maintenance/db_test.go +++ b/cmd/osbuild-service-maintenance/db_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "github.com/osbuild/osbuild-composer/internal/jobqueue/jobqueuetest" "testing" "github.com/google/uuid" @@ -13,13 +14,11 @@ import ( "github.com/osbuild/osbuild-composer/pkg/jobqueue/dbjobqueue" ) -const url = "postgres://postgres:foobar@localhost:5432/osbuildcomposer" - func TestDBJobQueueMaintenance(t *testing.T) { - dbMaintenance, err := newDB(url) + dbMaintenance, err := newDB(jobqueuetest.TestDbURL()) require.NoError(t, err) defer dbMaintenance.Close() - q, err := dbjobqueue.New(url) + q, err := dbjobqueue.New(jobqueuetest.TestDbURL()) require.NoError(t, err) defer q.Close() diff --git a/internal/jobqueue/jobqueuetest/jobqueuetest.go b/internal/jobqueue/jobqueuetest/jobqueuetest.go index 0da661d94e..94ceba2fb0 100644 --- a/internal/jobqueue/jobqueuetest/jobqueuetest.go +++ b/internal/jobqueue/jobqueuetest/jobqueuetest.go @@ -6,6 +6,8 @@ package jobqueuetest import ( "context" "encoding/json" + "fmt" + "os" "sync" "testing" "time" @@ -20,6 +22,14 @@ type MakeJobQueue func() (q jobqueue.JobQueue, stop func(), err error) type testResult struct { } +func TestDbURL() string { + host := os.Getenv("COMPOSER_TEST_DB_HOST") + if host == "" { + host = "localhost" + } + return fmt.Sprintf("postgres://postgres:foobar@%s:5432/osbuildcomposer", host) +} + func TestJobQueue(t *testing.T, makeJobQueue MakeJobQueue) { wrap := func(f func(t *testing.T, q jobqueue.JobQueue)) func(*testing.T) { q, stop, err := makeJobQueue() diff --git a/tools/dbtest-entrypoint.sh b/tools/dbtest-entrypoint.sh new file mode 100755 index 0000000000..0dff9bab7e --- /dev/null +++ b/tools/dbtest-entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +go version +go test -tags=integration ./cmd/osbuild-composer-dbjobqueue-tests +go test -tags=integration ./cmd/osbuild-service-maintenance diff --git a/tools/dbtest-prepare-env.sh b/tools/dbtest-prepare-env.sh new file mode 100755 index 0000000000..6e8c5f9f13 --- /dev/null +++ b/tools/dbtest-prepare-env.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +pushd $(mktemp -d) +go mod init temp +go install github.com/jackc/tern@latest +popd diff --git a/tools/dbtest-run-migrations.sh b/tools/dbtest-run-migrations.sh new file mode 100755 index 0000000000..d3c44ea988 --- /dev/null +++ b/tools/dbtest-run-migrations.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +WORKDIR=$(readlink -f pkg/jobqueue/dbjobqueue/schemas) +$(go env GOPATH)/bin/tern migrate -m "$WORKDIR" From cb60d45f5bfe2d4d134a1d34196fdeaf1e2e14a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 23 Oct 2024 14:34:50 +0200 Subject: [PATCH 2/6] Makefile: add hint that this just compiles the binaries It does not run the tests because of to the "-c" flag --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index cafa075169..a387ee0130 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,7 @@ build: $(BUILDDIR)/bin/ go build -o $ Date: Wed, 23 Oct 2024 14:36:14 +0200 Subject: [PATCH 3/6] jobqueuetest: avoid warning and provide a valid JSON Not needed for the test but just generates a useless warning --- internal/jobqueue/jobqueuetest/jobqueuetest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/jobqueue/jobqueuetest/jobqueuetest.go b/internal/jobqueue/jobqueuetest/jobqueuetest.go index 94ceba2fb0..11cdd24337 100644 --- a/internal/jobqueue/jobqueuetest/jobqueuetest.go +++ b/internal/jobqueue/jobqueuetest/jobqueuetest.go @@ -87,7 +87,7 @@ func testErrors(t *testing.T, q jobqueue.JobQueue) { require.Equal(t, uuid.Nil, id) // invalid dependency - id, err = q.Enqueue("test", "arg0", []uuid.UUID{uuid.New()}, "") + id, err = q.Enqueue("test", "{}", []uuid.UUID{uuid.New()}, "") require.Error(t, err) require.Equal(t, uuid.Nil, id) From 4b4bfd896e7d3884b12504749bd395831535a749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 23 Oct 2024 14:50:50 +0200 Subject: [PATCH 4/6] Makefile: document `make db-tests` --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a387ee0130..280915888e 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,7 @@ help: @echo " clean: Remove all built binaries" @echo " man: Generate all man-pages" @echo " unit-tests: Run unit tests" + @echo " db-tests: Run postgres DB tests" @echo " push-check: Replicates the github workflow checks as close as possible" @echo " (do this before pushing!)" @echo " lint: Runs linters as close as github workflow as possible" From 2eeff672ccb8db5451cb5181edba238377d21023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 23 Oct 2024 15:22:08 +0200 Subject: [PATCH 5/6] Makefile: implement shellcheck as part of `make lint` --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 280915888e..50ad8b22e6 100644 --- a/Makefile +++ b/Makefile @@ -344,9 +344,14 @@ container_composer_golangci_built.info: Makefile Containerfile_golangci_lint too echo "Image last built on" > $@ date >> $@ +# trying to catch our use cases of the github action implementation +# https://github.com/ludeeus/action-shellcheck/blob/master/action.yaml#L164 +SHELLCHECK_FILES=$(shell find . -name "*.sh" -not -regex "./vendor/.*") + .PHONY: lint lint: $(GOLANGCI_LINT_CACHE_DIR) container_composer_golangci_built.info podman run -t --rm -v $(SRCDIR):/app:z -v $(GOLANGCI_LINT_CACHE_DIR):/root/.cache:z -w /app $(GOLANGCI_COMPOSER_IMAGE) golangci-lint run -v + echo "$(SHELLCHECK_FILES)" | xargs shellcheck --shell bash -e SC1091 -e SC2002 -e SC2317 # The OpenShift CLI - maybe get it from https://access.redhat.com/downloads/content/290 OC_EXECUTABLE ?= oc From e4fb7ee567daedc38c3f7a435a3b44688b97a45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 23 Oct 2024 15:22:26 +0200 Subject: [PATCH 6/6] tools/dbtest-*: fix shellcheck complaints --- tools/dbtest-prepare-env.sh | 4 ++-- tools/dbtest-run-migrations.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/dbtest-prepare-env.sh b/tools/dbtest-prepare-env.sh index 6e8c5f9f13..09d369f09e 100755 --- a/tools/dbtest-prepare-env.sh +++ b/tools/dbtest-prepare-env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -pushd $(mktemp -d) +pushd "$(mktemp -d)" || exit 1 go mod init temp go install github.com/jackc/tern@latest -popd +popd || exit diff --git a/tools/dbtest-run-migrations.sh b/tools/dbtest-run-migrations.sh index d3c44ea988..7be4d81ca8 100755 --- a/tools/dbtest-run-migrations.sh +++ b/tools/dbtest-run-migrations.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash WORKDIR=$(readlink -f pkg/jobqueue/dbjobqueue/schemas) -$(go env GOPATH)/bin/tern migrate -m "$WORKDIR" +"$(go env GOPATH)"/bin/tern migrate -m "$WORKDIR"