diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index a3b532477..febca7951 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -8,6 +8,7 @@ on: - "**/*.md" - ".github/dependabot.yaml" - "charts/**" + - "Makefile" env: PR_NUMBER: ${{ github.event.number }} diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 2ba3c5fad..46835e4ca 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -8,6 +8,7 @@ on: - ".gitignore" - "**/*.md" - "charts/**" + - "Makefile" types: [ closed ] jobs: diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..065801038 --- /dev/null +++ b/Makefile @@ -0,0 +1,91 @@ +# linter modules to be included/excluded +LINT_INCLUDE=-E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck +LINT_EXCLUDE=-e G401,G501,G107 + +help: + @echo 'Welcome!' + @echo '' + @echo 'This Makefile is designed to make the development work go smoothly.' + @echo 'In-depth description of how to use this Makefile can be found in the README.md' + +bootstrap: go-version-check + @for dir in sda sda-auth sda-download; do \ + cd $$dir; \ + go get ./...; \ + cd ..; \ + done + if ! command -v curl >/dev/null; then \ + echo "Can't install golangci-lint because curl is missing."; \ + exit 1; \ + fi + @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \ + sh -s -- -b $$(go env GOPATH)/bin + GO111MODULE=off go get golang.org/x/tools/cmd/goimports + +# build containers +build-all: build-postgresql build-rabbitmq build-sda build-sda-auth build-sda-download build-sda-sftp-inbox +build-postgresql: + @cd postgresql && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-postgres . +build-rabbitmq: + @cd rabbitmq && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-rabbitmq . +build-sda: + @cd sda && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F) . +build-sda-auth: + @cd sda-auth && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-auth . +build-sda-download: + @cd sda-download && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-download . +build-sda-sftp-inbox: + @cd sda-sftp-inbox && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sftp-inbox . + + +go-version-check: SHELL:=/bin/bash +go-version-check: + @GO_VERSION_MIN=$$(grep GOLANG_VERSION $(CURDIR)/sda/Dockerfile | cut -d '-' -f2 | tr -d '}'); \ + GO_VERSION=$$(go version | grep -o 'go[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?' | tr -d 'go'); \ + IFS="." read -r -a GO_VERSION_ARR <<< "$${GO_VERSION}"; \ + IFS="." read -r -a GO_VERSION_REQ <<< "$${GO_VERSION_MIN}"; \ + if [[ $${GO_VERSION_ARR[0]} -lt $${GO_VERSION_REQ[0]} ||\ + ( $${GO_VERSION_ARR[0]} -eq $${GO_VERSION_REQ[0]} &&\ + ( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\ + ( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\ + ]]; then\ + echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}.";\ + exit 1;\ + fi; + + +# run intrgration tests, same as being run in Github Actions during a PR +integrationtest-postgres: build-postgresql + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml run tests + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml down -v --remove-orphans +integrationtest-rabbitmq: build-rabbitmq build-sda + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans +integrationtest-sda: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans + +# lint go code +lint-all: lint-sda lint-sda-auth lint-sda-download +lint-sda: + @echo 'Running golangci-lint in the `sda` folder' + @cd sda && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) +lint-sda-auth: + @echo 'Running golangci-lint in the `sda-auth` folder' + @cd sda-auth && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) +lint-sda-download: + @echo 'Running golangci-lint in the `sda-download` folder' + @cd sda-download && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + +# run static code tests +test-all: test-sda test-sda-auth test-sda-download test-sda-sftp-inbox +test-sda: + @cd sda && go test ./... -count=1 +test-sda-auth: + @cd sda-auth && go test ./... -count=1 +test-sda-download: + @cd sda-download && go test ./... -count=1 +test-sda-sftp-inbox: + @docker run --rm -v ./sda-sftp-inbox:/inbox maven:3.9.4-eclipse-temurin-21-alpine sh -c "cd /inbox && mvn test -B" diff --git a/README.md b/README.md index aaea44581..8f1d51937 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,98 @@ -# sda +# Sensitive Data Archive `SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/) It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a isolated Sensitive Data Archive. For more information about the different components see the readme files in the respecive folders. + +## Developing components of the SDA stack + +If you wish to work on the SDA stack itself you'll first need [Go](https://www.golang.org/) installed on your machine. The recommended version can be checked by running: + +```sh +$ make go-version-check +... +``` + +For local dev first make sure Go is properly installed, including setting up a [GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that $GOPATH/bin is in your path as some distributions bundle the old version of build tools. Next, clone this repository. SDA uses [Go Modules](https://github.com/golang/go/wiki/Modules), so it is recommended that you clone the repository outside of the GOPATH. You can then download any required build tools by bootstrapping your environment: + +```sh +$ make bootstrap +... +``` + +### Makefile options + +The Makefile is primarily designed to be an aid during development work. + +#### Building the containers + +To build all containers for the SDA stack: + +```sh +$ make build-all +... +``` + +To build the container for a speciffic component replace `all` with the folder name: + +```sh +$ make build- +... +``` + +#### Running the integration tests + +This will build the container and run the integration test for the PostgreSQL container. The same test will run on every PR in github: + +```sh +$ make integrationtest-postgres +... +``` + +This will build the RabbitMQ and SDA containers and run the integration test for the RabbitMQ container. The same test will run on every PR in github: + +```sh +$ make integrationtest-rabbitmq +... +``` + +This will build all containers and run the integration tests for the SDA stack. The same test will run on every PR in github: + +```sh +$ make integrationtest-sda +... +``` + +#### Linting the GO code + +To run golangci-lint for all go components: + +```sh +$ make lint-all +... +``` + +To run golangci-lint for a speciffic component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`): + +```sh +$ make lint- +... +``` + +#### Running the static code tests + +For the go code this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a maven container. + +To run the static code tests for all components: + +```sh +$ make test-all +... +``` + +To run the static code tests for a speciffic component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`, `sda-sftp-inbox`): + +```sh +$ make test- +... +```