Add a new API for listing logs and artifacts from a log area

.github/workflows/build-push.yml

@@ -66,10 +66,39 @@ jobs:
         echo "::set-output name=version::$VERSION"
     outputs:
       sseVersion: ${{ steps.image.outputs.version }}
+  build_logarea:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build app image
+      run: docker build . -f deploy/etos-logarea/Dockerfile --tag image
+
+    - name: Log into registry
+      run: echo "${{ secrets.REGISTRYPASSWORD }}" | docker login registry.nordix.org -u ${{ secrets.REGISTRYUSERNAME }} --password-stdin
+
+    - name: Push app image
+      id: image
+      run: |
+        IMAGE_ID=registry.nordix.org/eiffel/etos-logarea
+        # Strip git ref prefix from version
+        VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+        # Strip "v" prefix from tag name
+        [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+        # Use Docker `latest` tag convention
+        [ "$VERSION" == "main" ] && VERSION=$(echo ${{ github.sha }} | cut -c1-8)
+        echo IMAGE_ID=$IMAGE_ID
+        echo VERSION=$VERSION
+        docker tag image $IMAGE_ID:$VERSION
+        docker push $IMAGE_ID:$VERSION
+        echo $IMAGE_ID:$VERSION
+        echo "::set-output name=version::$VERSION"
+    outputs:
+      logAreaVersion: ${{ steps.image.outputs.version }}
   update_manifests:
     if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }}
     runs-on: ubuntu-latest
-    needs: [build_api, build_sse]
+    needs: [build_api, build_sse, build_logarea]
     steps:
     - uses: actions/checkout@v3
     - name: Update manifests
@@ -83,7 +112,10 @@ jobs:
             "manifests/base/deployment.yaml": {
               "spec.template.spec.containers[0].image": "registry.nordix.org/eiffel/etos-api:${{ needs.build_api.outputs.apiVersion }}"
             }
+            "manifests/base/logarea/deployment.yaml": {
+              "spec.template.spec.containers[0].image": "registry.nordix.org/eiffel/etos-logarea:${{ needs.build_logarea.outputs.logAreaVersion }}"
+            },
           }
         branch: main
         commitChange: true
-        message: Updating SSE image to ${{ needs.build_sse.outputs.sseVersion }} and API image to ${{ needs.build_api.outputs.apiVersion }}
+        message: Updating SSE image to ${{ needs.build_sse.outputs.sseVersion }}, LogArea image to ${{ needs.build_logarea.outputs.logAreaVersion }} and API image to ${{ needs.build_api.outputs.apiVersion }}

.github/workflows/main.yml

@@ -28,7 +28,20 @@ jobs:
       - name: Run Tox
         run: tox
         working-directory: ./python
-
+  run-gotest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Since we have 1.21 in the go.mod file, but toolchain is go1.22.1
+        go-version: [ '1.21', '1.22.1' ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Go test
+        run: make test
   run-hadolint:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
@@ -46,6 +59,10 @@ jobs:
         uses: hadolint/hadolint-action@master
         with:
           dockerfile: deploy/etos-sse/Dockerfile
+      - name: Run hadolint for LogArea
+        uses: hadolint/hadolint-action@master
+        with:
+          dockerfile: deploy/etos-logarea/Dockerfile
 
   build-docker-images:
     # The type of runner that the job will run on
@@ -66,3 +83,8 @@ jobs:
         with:
           context: .
           file: ./deploy/etos-sse/Dockerfile
+      - name: Build LogArea image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./deploy/etos-logarea/Dockerfile

.gitignore

@@ -51,4 +51,5 @@ MANIFEST
 
 # Per-project virtualenvs
 .venv*/
+.vscode
 .python-version

Makefile

@@ -1,29 +1,62 @@
 # Install tools locally instead of in $HOME/go/bin.
 export GOBIN := $(CURDIR)/bin
+export GOBUILD := go build
 export PATH := $(GOBIN):$(PATH)
 
 export RELEASE_VERSION ?= $(shell git describe --always)
 export DOCKER_REGISTRY ?= registry.nordix.org
 export DOCKER_NAMESPACE ?= eiffel
 export DEPLOY ?= etos-sse
 
+PROGRAMS = sse logarea
 COMPILEDAEMON = $(GOBIN)/CompileDaemon
 GIT = git
-SSE = $(GOBIN)/etos-sse
 GOLANGCI_LINT = $(GOBIN)/golangci-lint
 
 GOLANGCI_LINT_VERSION = v1.52.2
 
-.PHONY: all
-all: test build start
+# Generates a rule for building a single binary from cmd/$1.
+#
+# $1: Name of the program
+define build-binary
+.PHONY: $(strip $(1))
+$(strip $(1)):
+	$$(GOBUILD) -ldflags="-w -s -buildid=" -trimpath -o bin/$(strip $(1)) ./cmd/$(strip $(1))
+endef
+
+# Generates a rule named $(program_name)-docker to to build the named program
+# and copy the locally compiled binary into a Docker image. The image will be
+# tagged with the name of the program.
+#
+# $1: Name of the program
+define build-docker-image
+.PHONY: $(strip $(1))-docker
+# Including the parameter name!
+EXTRA_DOCKER_ARGS=
+export EXTRA_DOCKER_ARGS
+$(strip $(1))-docker: $(strip $(1))
+	docker build . \
+    $(EXTRA_DOCKER_ARGS) \
+		-f deploy/etos-$(strip $(1))/Dockerfile \
+		-t $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/etos-$(strip $(1)):$(RELEASE_VERSION)
+endef
+
+# Generates a rule named $(program_name)-docker-push to push
+# a local Docker image to a remote registry.
+#
+# $1: Name of the program
+define push-docker-image
+.PHONY: $(strip $(1))-docker-push
+$(strip $(1))-docker-push:
+	echo docker push $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(strip $(1))_provider:$(RELEASE_VERSION)
+endef
 
-.PHONY: build
-build:
-	go build -ldflags="-w -s -buildid=" -trimpath -o $(SSE) ./cmd/sse
+.PHONY: all
+all: test $(PROGRAMS)
 
 .PHONY: clean
 clean:
-	$(RM) $(SSE)
+	$(RM) $(GOBIN)/*
 	docker compose --project-directory . -f deploy/$(DEPLOY)/docker-compose.yml rm || true
 
 .PHONY: check
@@ -46,18 +79,6 @@ start: $(COMPILEDAEMON)
 stop:
 	docker compose --project-directory . -f deploy/$(DEPLOY)/docker-compose.yml down
 
-# Build a docker using the production Dockerfiler
-.PHONY: docker
-# Including the parameter name!
-EXTRA_DOCKER_ARGS=
-export EXTRA_DOCKER_ARGS
-docker:
-	docker build $(EXTRA_DOCKER_ARGS) -t $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(DEPLOY):$(RELEASE_VERSION) -f ./deploy/$(DEPLOY)/Dockerfile .
-
-.PHONY: push
-push:
-	docker push $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(DEPLOY):$(RELEASE_VERSION)
-
 .PHONY: tidy
 tidy:
 	go mod tidy
@@ -66,6 +87,15 @@ tidy:
 check-dirty:
 	$(GIT) diff --exit-code HEAD
 
+
+# Setup the dynamic commands
+#
+$(foreach prog,$(PROGRAMS), \
+    $(eval $(call build-binary,$(prog))) \
+    $(eval $(call build-docker-image,$(prog))) \
+    $(eval $(call push-docker-image,$(prog))) \
+)
+
 # Build dependencies
 
 $(COMPILEDAEMON):

cmd/logarea/main.go

@@ -0,0 +1,122 @@
+// Copyright Axis Communications AB.
+//
+// For a full list of individual contributors, please see the commit history.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime/debug"
+	"syscall"
+	"time"
+
+	"github.com/eiffel-community/etos-api/internal/config"
+	"github.com/eiffel-community/etos-api/internal/logging"
+	"github.com/eiffel-community/etos-api/internal/server"
+	"github.com/eiffel-community/etos-api/pkg/application"
+	v1alpha "github.com/eiffel-community/etos-api/pkg/logarea/v1alpha"
+	"github.com/sirupsen/logrus"
+	"github.com/snowzach/rotatefilehook"
+	"go.elastic.co/ecslogrus"
+)
+
+// main sets up logging and starts up the logarea webservice.
+func main() {
+	cfg := config.Get()
+	ctx := context.Background()
+
+	var hooks []logrus.Hook
+	if fileHook := fileLogging(cfg); fileHook != nil {
+		hooks = append(hooks, fileHook)
+	}
+	logger, err := logging.Setup(cfg.LogLevel(), hooks)
+	if err != nil {
+		logrus.Fatal(err.Error())
+	}
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		logrus.Fatal(err.Error())
+	}
+	log := logger.WithFields(logrus.Fields{
+		"hostname":    hostname,
+		"application": "ETOS API LogArea Server",
+		"version":     vcsRevision(),
+		"name":        "ETOS API",
+	})
+
+	log.Info("Loading logarea routes")
+	v1AlphaLogArea := v1alpha.New(cfg, log)
+	defer v1AlphaLogArea.Close()
+
+	app := application.New(v1AlphaLogArea)
+	srv := server.NewWebService(cfg, log, app)
+
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		if err := srv.Start(); err != nil && err != http.ErrServerClosed {
+			log.Errorf("Webserver shutdown: %+v", err)
+		}
+	}()
+
+	sig := <-done
+	log.Infof("%s received", sig.String())
+
+	ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	if err := srv.Close(ctx); err != nil {
+		log.Errorf("Webserver shutdown failed: %+v", err)
+	}
+	log.Info("Wait for shutdown to complete")
+}
+
+// fileLogging adds a hook into a slice of hooks, if the filepath configuration is set
+func fileLogging(cfg config.Config) logrus.Hook {
+	if filePath := cfg.LogFilePath(); filePath != "" {
+		// TODO: Make these parameters configurable.
+		// NewRotateFileHook cannot return an error which is why it's set to '_'.
+		rotateFileHook, _ := rotatefilehook.NewRotateFileHook(rotatefilehook.RotateFileConfig{
+			Filename:   filePath,
+			MaxSize:    10, // megabytes
+			MaxBackups: 3,
+			MaxAge:     0, // days
+			Level:      logrus.DebugLevel,
+			Formatter: &ecslogrus.Formatter{
+				DataKey: "labels",
+			},
+		})
+		return rotateFileHook
+	}
+	return nil
+}
+
+// vcsRevision returns vcs revision from build info, if any. Otherwise '(unknown)'.
+func vcsRevision() string {
+	buildInfo, ok := debug.ReadBuildInfo()
+	if !ok {
+		return "(unknown)"
+	}
+	for _, val := range buildInfo.Settings {
+		if val.Key == "vcs.revision" {
+			return val.Value
+		}
+	}
+	return "(unknown)"
+}

deploy/etos-logarea/Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.21-alpine AS build
+WORKDIR /tmp/logarea
+COPY . .
+RUN apk add --no-cache make=4.4.1-r2 git=2.45.2-r0 && make logarea
+
+FROM alpine:3.17.3
+ARG TZ
+ENV TZ=$TZ
+
+LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-api
+LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com
+LABEL org.opencontainers.image.licenses=Apache-2.0
+
+RUN apk add --no-cache tzdata=2024a-r0
+ENTRYPOINT ["/app/logarea"]
+
+COPY --from=build /tmp/logarea/bin/logarea /app/logarea

deploy/etos-logarea/Dockerfile.dev

@@ -0,0 +1,8 @@
+FROM golang:1.21
+WORKDIR /app
+
+COPY ./go.mod ./go.sum ./
+RUN go mod tidy
+COPY . .
+RUN git config --global --add safe.directory /app
+EXPOSE 8080

deploy/etos-logarea/docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3.7"
+services:
+  etos-logarea:
+    build:
+      context: .
+      dockerfile: ./deploy/etos-logarea/Dockerfile.dev
+      args:
+        http_proxy: "${http_proxy}"
+        https_proxy: "${https_proxy}"
+    volumes:
+      - ./:/app
+    ports:
+      - 8080:8080
+    env_file:
+      - ./configs/development.env
+    entrypoint: ["/bin/bash", "./scripts/entrypoint.sh"]

deploy/etos-sse/Dockerfile

@@ -1,7 +1,7 @@
-FROM golang:1.20-alpine AS build
+FROM golang:1.21-alpine AS build
 WORKDIR /tmp/sse
 COPY . .
-RUN apk add --no-cache make=4.4.1-r2 git=2.43.4-r0 && make build
+RUN apk add --no-cache make=4.4.1-r2 git=2.45.2-r0 && make sse
 
 FROM alpine:3.17.3
 ARG TZ
@@ -12,6 +12,6 @@ LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com
 LABEL org.opencontainers.image.licenses=Apache-2.0
 
 RUN apk add --no-cache tzdata=2024a-r0
-ENTRYPOINT ["/app/etos-sse"]
+ENTRYPOINT ["/app/sse"]
 
-COPY --from=build /tmp/sse/bin/etos-sse /app/etos-sse
+COPY --from=build /tmp/sse/bin/sse /app/sse

deploy/etos-sse/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:1.21
 WORKDIR /app
 
 COPY ./go.mod ./go.sum ./