Skip to content

Commit

Permalink
Add a new API for listing logs and artifacts from a log area (#66)
Browse files Browse the repository at this point in the history
* Add a new API for listing logs and artifacts from a log area
  • Loading branch information
t-persson authored Jun 20, 2024
1 parent ce37ae7 commit d362cd1
Show file tree
Hide file tree
Showing 23 changed files with 1,258 additions and 51 deletions.
36 changes: 34 additions & 2 deletions .github/workflows/build-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,39 @@ jobs:
echo "::set-output name=version::$VERSION"
outputs:
sseVersion: ${{ steps.image.outputs.version }}
build_logarea:
if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build app image
run: docker build . -f deploy/etos-logarea/Dockerfile --tag image

- name: Log into registry
run: echo "${{ secrets.REGISTRYPASSWORD }}" | docker login registry.nordix.org -u ${{ secrets.REGISTRYUSERNAME }} --password-stdin

- name: Push app image
id: image
run: |
IMAGE_ID=registry.nordix.org/eiffel/etos-logarea
# Strip git ref prefix from version
VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
# Strip "v" prefix from tag name
[[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
# Use Docker `latest` tag convention
[ "$VERSION" == "main" ] && VERSION=$(echo ${{ github.sha }} | cut -c1-8)
echo IMAGE_ID=$IMAGE_ID
echo VERSION=$VERSION
docker tag image $IMAGE_ID:$VERSION
docker push $IMAGE_ID:$VERSION
echo $IMAGE_ID:$VERSION
echo "::set-output name=version::$VERSION"
outputs:
logAreaVersion: ${{ steps.image.outputs.version }}
update_manifests:
if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }}
runs-on: ubuntu-latest
needs: [build_api, build_sse]
needs: [build_api, build_sse, build_logarea]
steps:
- uses: actions/checkout@v3
- name: Update manifests
Expand All @@ -83,7 +112,10 @@ jobs:
"manifests/base/deployment.yaml": {
"spec.template.spec.containers[0].image": "registry.nordix.org/eiffel/etos-api:${{ needs.build_api.outputs.apiVersion }}"
}
"manifests/base/logarea/deployment.yaml": {
"spec.template.spec.containers[0].image": "registry.nordix.org/eiffel/etos-logarea:${{ needs.build_logarea.outputs.logAreaVersion }}"
},
}
branch: main
commitChange: true
message: Updating SSE image to ${{ needs.build_sse.outputs.sseVersion }} and API image to ${{ needs.build_api.outputs.apiVersion }}
message: Updating SSE image to ${{ needs.build_sse.outputs.sseVersion }}, LogArea image to ${{ needs.build_logarea.outputs.logAreaVersion }} and API image to ${{ needs.build_api.outputs.apiVersion }}
24 changes: 23 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,20 @@ jobs:
- name: Run Tox
run: tox
working-directory: ./python

run-gotest:
runs-on: ubuntu-latest
strategy:
matrix:
# Since we have 1.21 in the go.mod file, but toolchain is go1.22.1
go-version: [ '1.21', '1.22.1' ]
steps:
- uses: actions/checkout@v4
- name: Setup Go ${{ matrix.go-version }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
- name: Go test
run: make test
run-hadolint:
# The type of runner that the job will run on
runs-on: ubuntu-latest
Expand All @@ -46,6 +59,10 @@ jobs:
uses: hadolint/hadolint-action@master
with:
dockerfile: deploy/etos-sse/Dockerfile
- name: Run hadolint for LogArea
uses: hadolint/hadolint-action@master
with:
dockerfile: deploy/etos-logarea/Dockerfile

build-docker-images:
# The type of runner that the job will run on
Expand All @@ -66,3 +83,8 @@ jobs:
with:
context: .
file: ./deploy/etos-sse/Dockerfile
- name: Build LogArea image
uses: docker/build-push-action@v2
with:
context: .
file: ./deploy/etos-logarea/Dockerfile
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,5 @@ MANIFEST

# Per-project virtualenvs
.venv*/
.vscode
.python-version
68 changes: 49 additions & 19 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,29 +1,62 @@
# Install tools locally instead of in $HOME/go/bin.
export GOBIN := $(CURDIR)/bin
export GOBUILD := go build
export PATH := $(GOBIN):$(PATH)

export RELEASE_VERSION ?= $(shell git describe --always)
export DOCKER_REGISTRY ?= registry.nordix.org
export DOCKER_NAMESPACE ?= eiffel
export DEPLOY ?= etos-sse

PROGRAMS = sse logarea
COMPILEDAEMON = $(GOBIN)/CompileDaemon
GIT = git
SSE = $(GOBIN)/etos-sse
GOLANGCI_LINT = $(GOBIN)/golangci-lint

GOLANGCI_LINT_VERSION = v1.52.2

.PHONY: all
all: test build start
# Generates a rule for building a single binary from cmd/$1.
#
# $1: Name of the program
define build-binary
.PHONY: $(strip $(1))
$(strip $(1)):
$$(GOBUILD) -ldflags="-w -s -buildid=" -trimpath -o bin/$(strip $(1)) ./cmd/$(strip $(1))
endef

# Generates a rule named $(program_name)-docker to to build the named program
# and copy the locally compiled binary into a Docker image. The image will be
# tagged with the name of the program.
#
# $1: Name of the program
define build-docker-image
.PHONY: $(strip $(1))-docker
# Including the parameter name!
EXTRA_DOCKER_ARGS=
export EXTRA_DOCKER_ARGS
$(strip $(1))-docker: $(strip $(1))
docker build . \
$(EXTRA_DOCKER_ARGS) \
-f deploy/etos-$(strip $(1))/Dockerfile \
-t $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/etos-$(strip $(1)):$(RELEASE_VERSION)
endef

# Generates a rule named $(program_name)-docker-push to push
# a local Docker image to a remote registry.
#
# $1: Name of the program
define push-docker-image
.PHONY: $(strip $(1))-docker-push
$(strip $(1))-docker-push:
echo docker push $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(strip $(1))_provider:$(RELEASE_VERSION)
endef

.PHONY: build
build:
go build -ldflags="-w -s -buildid=" -trimpath -o $(SSE) ./cmd/sse
.PHONY: all
all: test $(PROGRAMS)

.PHONY: clean
clean:
$(RM) $(SSE)
$(RM) $(GOBIN)/*
docker compose --project-directory . -f deploy/$(DEPLOY)/docker-compose.yml rm || true

.PHONY: check
Expand All @@ -46,18 +79,6 @@ start: $(COMPILEDAEMON)
stop:
docker compose --project-directory . -f deploy/$(DEPLOY)/docker-compose.yml down

# Build a docker using the production Dockerfiler
.PHONY: docker
# Including the parameter name!
EXTRA_DOCKER_ARGS=
export EXTRA_DOCKER_ARGS
docker:
docker build $(EXTRA_DOCKER_ARGS) -t $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(DEPLOY):$(RELEASE_VERSION) -f ./deploy/$(DEPLOY)/Dockerfile .

.PHONY: push
push:
docker push $(DOCKER_REGISTRY)/$(DOCKER_NAMESPACE)/$(DEPLOY):$(RELEASE_VERSION)

.PHONY: tidy
tidy:
go mod tidy
Expand All @@ -66,6 +87,15 @@ tidy:
check-dirty:
$(GIT) diff --exit-code HEAD


# Setup the dynamic commands
#
$(foreach prog,$(PROGRAMS), \
$(eval $(call build-binary,$(prog))) \
$(eval $(call build-docker-image,$(prog))) \
$(eval $(call push-docker-image,$(prog))) \
)

# Build dependencies

$(COMPILEDAEMON):
Expand Down
122 changes: 122 additions & 0 deletions cmd/logarea/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright Axis Communications AB.
//
// For a full list of individual contributors, please see the commit history.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
"context"
"net/http"
"os"
"os/signal"
"runtime/debug"
"syscall"
"time"

"github.com/eiffel-community/etos-api/internal/config"
"github.com/eiffel-community/etos-api/internal/logging"
"github.com/eiffel-community/etos-api/internal/server"
"github.com/eiffel-community/etos-api/pkg/application"
v1alpha "github.com/eiffel-community/etos-api/pkg/logarea/v1alpha"
"github.com/sirupsen/logrus"
"github.com/snowzach/rotatefilehook"
"go.elastic.co/ecslogrus"
)

// main sets up logging and starts up the logarea webservice.
func main() {
cfg := config.Get()
ctx := context.Background()

var hooks []logrus.Hook
if fileHook := fileLogging(cfg); fileHook != nil {
hooks = append(hooks, fileHook)
}
logger, err := logging.Setup(cfg.LogLevel(), hooks)
if err != nil {
logrus.Fatal(err.Error())
}

hostname, err := os.Hostname()
if err != nil {
logrus.Fatal(err.Error())
}
log := logger.WithFields(logrus.Fields{
"hostname": hostname,
"application": "ETOS API LogArea Server",
"version": vcsRevision(),
"name": "ETOS API",
})

log.Info("Loading logarea routes")
v1AlphaLogArea := v1alpha.New(cfg, log)
defer v1AlphaLogArea.Close()

app := application.New(v1AlphaLogArea)
srv := server.NewWebService(cfg, log, app)

done := make(chan os.Signal, 1)
signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)

go func() {
if err := srv.Start(); err != nil && err != http.ErrServerClosed {
log.Errorf("Webserver shutdown: %+v", err)
}
}()

sig := <-done
log.Infof("%s received", sig.String())

ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

if err := srv.Close(ctx); err != nil {
log.Errorf("Webserver shutdown failed: %+v", err)
}
log.Info("Wait for shutdown to complete")
}

// fileLogging adds a hook into a slice of hooks, if the filepath configuration is set
func fileLogging(cfg config.Config) logrus.Hook {
if filePath := cfg.LogFilePath(); filePath != "" {
// TODO: Make these parameters configurable.
// NewRotateFileHook cannot return an error which is why it's set to '_'.
rotateFileHook, _ := rotatefilehook.NewRotateFileHook(rotatefilehook.RotateFileConfig{
Filename: filePath,
MaxSize: 10, // megabytes
MaxBackups: 3,
MaxAge: 0, // days
Level: logrus.DebugLevel,
Formatter: &ecslogrus.Formatter{
DataKey: "labels",
},
})
return rotateFileHook
}
return nil
}

// vcsRevision returns vcs revision from build info, if any. Otherwise '(unknown)'.
func vcsRevision() string {
buildInfo, ok := debug.ReadBuildInfo()
if !ok {
return "(unknown)"
}
for _, val := range buildInfo.Settings {
if val.Key == "vcs.revision" {
return val.Value
}
}
return "(unknown)"
}
17 changes: 17 additions & 0 deletions deploy/etos-logarea/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
FROM golang:1.21-alpine AS build
WORKDIR /tmp/logarea
COPY . .
RUN apk add --no-cache make=4.4.1-r2 git=2.45.2-r0 && make logarea

FROM alpine:3.17.3
ARG TZ
ENV TZ=$TZ

LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-api
LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com
LABEL org.opencontainers.image.licenses=Apache-2.0

RUN apk add --no-cache tzdata=2024a-r0
ENTRYPOINT ["/app/logarea"]

COPY --from=build /tmp/logarea/bin/logarea /app/logarea
8 changes: 8 additions & 0 deletions deploy/etos-logarea/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM golang:1.21
WORKDIR /app

COPY ./go.mod ./go.sum ./
RUN go mod tidy
COPY . .
RUN git config --global --add safe.directory /app
EXPOSE 8080
16 changes: 16 additions & 0 deletions deploy/etos-logarea/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
version: "3.7"
services:
etos-logarea:
build:
context: .
dockerfile: ./deploy/etos-logarea/Dockerfile.dev
args:
http_proxy: "${http_proxy}"
https_proxy: "${https_proxy}"
volumes:
- ./:/app
ports:
- 8080:8080
env_file:
- ./configs/development.env
entrypoint: ["/bin/bash", "./scripts/entrypoint.sh"]
8 changes: 4 additions & 4 deletions deploy/etos-sse/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
FROM golang:1.20-alpine AS build
FROM golang:1.21-alpine AS build
WORKDIR /tmp/sse
COPY . .
RUN apk add --no-cache make=4.4.1-r2 git=2.43.4-r0 && make build
RUN apk add --no-cache make=4.4.1-r2 git=2.45.2-r0 && make sse

FROM alpine:3.17.3
ARG TZ
Expand All @@ -12,6 +12,6 @@ LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com
LABEL org.opencontainers.image.licenses=Apache-2.0

RUN apk add --no-cache tzdata=2024a-r0
ENTRYPOINT ["/app/etos-sse"]
ENTRYPOINT ["/app/sse"]

COPY --from=build /tmp/sse/bin/etos-sse /app/etos-sse
COPY --from=build /tmp/sse/bin/sse /app/sse
2 changes: 1 addition & 1 deletion deploy/etos-sse/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.20
FROM golang:1.21
WORKDIR /app

COPY ./go.mod ./go.sum ./
Expand Down
Loading

0 comments on commit d362cd1

Please sign in to comment.