Merge pull request #447 from neicnordic/dev_env

Develpoment helpers

.github/workflows/build_pr_container.yaml

@@ -8,6 +8,7 @@ on:
       - "**/*.md"
       - ".github/dependabot.yaml"
       - "charts/**"
+      - "Makefile"
 
 env:
   PR_NUMBER: ${{ github.event.number }}

.github/workflows/publish_container.yml

@@ -8,6 +8,7 @@ on:
       - ".gitignore"
       - "**/*.md"
       - "charts/**"
+      - "Makefile"
     types: [ closed ]
 
 jobs:

Makefile

@@ -0,0 +1,91 @@
+# linter modules to be included/excluded
+LINT_INCLUDE=-E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck
+LINT_EXCLUDE=-e G401,G501,G107
+
+help:
+	@echo 'Welcome!'
+	@echo ''
+	@echo 'This Makefile is designed to make the development work go smoothly.'
+	@echo 'In-depth description of how to use this Makefile can be found in the README.md'
+
+bootstrap: go-version-check
+		@for dir in sda sda-auth sda-download; do \
+			cd $$dir; \
+			go get ./...; \
+			cd ..; \
+		done
+		if ! command -v curl >/dev/null; then \
+			echo "Can't install golangci-lint because curl is missing."; \
+			exit 1; \
+		fi
+		@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | \
+		sh -s -- -b $$(go env GOPATH)/bin
+		GO111MODULE=off go get golang.org/x/tools/cmd/goimports
+
+# build containers
+build-all: build-postgresql build-rabbitmq build-sda build-sda-auth build-sda-download build-sda-sftp-inbox
+build-postgresql:
+	@cd postgresql && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-postgres .
+build-rabbitmq:
+	@cd rabbitmq && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-rabbitmq .
+build-sda:
+	@cd sda && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F) .
+build-sda-auth:
+	@cd sda-auth && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-auth .
+build-sda-download:
+	@cd sda-download && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-download .
+build-sda-sftp-inbox:
+	@cd sda-sftp-inbox && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sftp-inbox .
+
+
+go-version-check: SHELL:=/bin/bash
+go-version-check:
+	@GO_VERSION_MIN=$$(grep GOLANG_VERSION $(CURDIR)/sda/Dockerfile | cut -d '-' -f2 | tr -d '}'); \
+	GO_VERSION=$$(go version | grep -o 'go[0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?' | tr -d 'go'); \
+	IFS="." read -r -a GO_VERSION_ARR <<< "$${GO_VERSION}"; \
+	IFS="." read -r -a GO_VERSION_REQ <<< "$${GO_VERSION_MIN}"; \
+	if [[ $${GO_VERSION_ARR[0]} -lt $${GO_VERSION_REQ[0]} ||\
+		( $${GO_VERSION_ARR[0]} -eq $${GO_VERSION_REQ[0]} &&\
+		( $${GO_VERSION_ARR[1]} -lt $${GO_VERSION_REQ[1]} ||\
+		( $${GO_VERSION_ARR[1]} -eq $${GO_VERSION_REQ[1]} && $${GO_VERSION_ARR[2]} -lt $${GO_VERSION_REQ[2]} )))\
+	]]; then\
+		echo "SDA requires go $${GO_VERSION_MIN} to build; found $${GO_VERSION}.";\
+		exit 1;\
+	fi;
+
+
+# run intrgration tests, same as being run in Github Actions during a PR
+integrationtest-postgres: build-postgresql
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml run tests
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml down -v --remove-orphans
+integrationtest-rabbitmq: build-rabbitmq build-sda
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans
+integrationtest-sda: build-all
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml run integration_test
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-s3-integration.yml down -v --remove-orphans
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml run integration_test
+	@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-posix-integration.yml down -v --remove-orphans
+
+# lint go code
+lint-all: lint-sda lint-sda-auth lint-sda-download
+lint-sda:
+	@echo 'Running golangci-lint in the `sda` folder'
+	@cd sda && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE)
+lint-sda-auth:
+	@echo 'Running golangci-lint in the `sda-auth` folder'
+	@cd sda-auth && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE)
+lint-sda-download:
+	@echo 'Running golangci-lint in the `sda-download` folder'
+	@cd sda-download && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE)
+
+# run static code tests
+test-all: test-sda test-sda-auth test-sda-download test-sda-sftp-inbox
+test-sda:
+	@cd sda && go test ./... -count=1
+test-sda-auth:
+	@cd sda-auth && go test ./... -count=1
+test-sda-download:
+	@cd sda-download && go test ./... -count=1
+test-sda-sftp-inbox:
+	@docker run --rm -v ./sda-sftp-inbox:/inbox maven:3.9.4-eclipse-temurin-21-alpine sh -c "cd /inbox && mvn test -B"

README.md

@@ -1,5 +1,98 @@
-# sda
+# Sensitive Data Archive
 
 `SDA` contains all components of [NeIC Sensitive Data Archive](https://neic-sda.readthedocs.io/en/latest/) It can be used as part of a [Federated EGA](https://ega-archive.org/federated) or as a isolated Sensitive Data Archive.
 
 For more information about the different components see the readme files in the respecive folders.
+
+## Developing components of the SDA stack
+
+If you wish to work on the SDA stack itself you'll first need [Go](https://www.golang.org/) installed on your machine. The recommended version can be checked by running:
+
+```sh
+$ make go-version-check
+...
+```
+
+For local dev first make sure Go is properly installed, including setting up a [GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that $GOPATH/bin is in your path as some distributions bundle the old version of build tools. Next, clone this repository. SDA uses [Go Modules](https://github.com/golang/go/wiki/Modules), so it is recommended that you clone the repository outside of the GOPATH. You can then download any required build tools by bootstrapping your environment:
+
+```sh
+$ make bootstrap
+...
+```
+
+### Makefile options
+
+The Makefile is primarily designed to be an aid during development work.
+
+#### Building the containers
+
+To build all containers for the SDA stack:
+
+```sh
+$ make build-all
+...
+```
+
+To build the container for a speciffic component replace `all` with the folder name:
+
+```sh
+$ make build-<folder-name>
+...
+```
+
+#### Running the integration tests
+
+This will build the container and run the integration test for the PostgreSQL container. The same test will run on every PR in github:
+
+```sh
+$ make integrationtest-postgres
+...
+```
+
+This will build the RabbitMQ and SDA containers and run the integration test for the RabbitMQ container. The same test will run on every PR in github:
+
+```sh
+$ make integrationtest-rabbitmq
+...
+```
+
+This will build all containers and run the integration tests for the SDA stack. The same test will run on every PR in github:
+
+```sh
+$ make integrationtest-sda
+...
+```
+
+#### Linting the GO code
+
+To run golangci-lint for all go components:
+
+```sh
+$ make lint-all
+...
+```
+
+To run golangci-lint for a speciffic component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`):
+
+```sh
+$ make lint-<folder-name>
+...
+```
+
+#### Running the static code tests
+
+For the go code this means running `go test -count=1 ./...` in the target folder. For the *sftp-inbox* this calls `mvn test -B` inside a maven container.
+
+To run the static code tests for all components:
+
+```sh
+$ make test-all
+...
+```
+
+To run the static code tests for a speciffic component replace `all` with the folder name (`sda`, `sda-auth`, `sda-download`, `sda-sftp-inbox`):
+
+```sh
+$ make test-<folder-name>
+...
+```