diff --git a/Makefile b/Makefile index 2b96f2f..32d1d24 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ export DOCKER_REGISTRY ?= registry.nordix.org export DOCKER_NAMESPACE ?= eiffel export DEPLOY ?= etos-sse -PROGRAMS = sse logarea +PROGRAMS = sse logarea iut COMPILEDAEMON = $(GOBIN)/CompileDaemon GIT = git GOLANGCI_LINT = $(GOBIN)/golangci-lint diff --git a/cmd/iut/main.go b/cmd/iut/main.go new file mode 100644 index 0000000..d002931 --- /dev/null +++ b/cmd/iut/main.go @@ -0,0 +1,155 @@ +// Copyright 2022 Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "context" + "errors" + "net/http" + "os" + "os/signal" + "runtime/debug" + "syscall" + "time" + + config "github.com/eiffel-community/etos-api/internal/configs/iut" + "github.com/eiffel-community/etos-api/internal/iut/contextmanager" + server "github.com/eiffel-community/etos-api/internal/iut/server" + "github.com/eiffel-community/etos-api/internal/logging" + "github.com/eiffel-community/etos-api/pkg/iut/application" + "github.com/eiffel-community/etos-api/pkg/iut/v1alpha1" + "github.com/sirupsen/logrus" + "github.com/snowzach/rotatefilehook" + "go.elastic.co/ecslogrus" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// main sets up logging and starts up the webserver. +func main() { + cfg := config.Get() + ctx := context.Background() + + var hooks []logrus.Hook + if fileHook := fileLogging(cfg); fileHook != nil { + hooks = append(hooks, fileHook) + } + + logger, err := logging.Setup(cfg.LogLevel(), hooks) + if err != nil { + logrus.Fatal(err.Error()) + } + + hostname, err := os.Hostname() + if err != nil { + logrus.Fatal(err.Error()) + } + log := logger.WithFields(logrus.Fields{ + "hostname": hostname, + "application": "Dummy IUT Provider Service", + "version": vcsRevision(), + "name": "Dummy IUT Provider", + "user_log": false, + }) + + if err := validateInput(cfg); err != nil { + log.Panic(err) + } + + // Database connection test + cli, err := clientv3.New(clientv3.Config{ + Endpoints: []string{cfg.DatabaseURI()}, + DialTimeout: 5 * time.Second, + }) + if err != nil { + log.WithError(err).Fatal("failed to create etcd connection") + } + + cm := contextmanager.New(cli) + go cm.Start(ctx) + defer cm.CancelAll() + + log.Info("Loading v1alpha1 routes") + v1alpha1App := v1alpha1.New(cfg, log, ctx, cm, cli) + defer v1alpha1App.Close() + router := application.New(v1alpha1App) + + srv := server.NewWebserver(cfg, log, router) + + done := make(chan os.Signal, 1) + signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + go func() { + if err := srv.Start(); err != nil && err != http.ErrServerClosed { + log.Errorf("Webserver shutdown: %+v", err) + } + }() + + <-done + log.Info("SIGTERM received") + + ctx, cancel := context.WithTimeout(ctx, cfg.Timeout()) + defer cancel() + v1alpha1App.Close() + + if err := srv.Close(ctx); err != nil { + log.Errorf("Webserver shutdown failed: %+v", err) + } + log.Info("Wait for checkout, deploy, status and checkin jobs to complete") +} + +// fileLogging adds a hook into a slice of hooks, if the filepath configuration is set +func fileLogging(cfg config.Config) logrus.Hook { + if filePath := cfg.LogFilePath(); filePath != "" { + // TODO: Make these parameters configurable. + // NewRotateFileHook cannot return an error which is why it's set to '_'. + rotateFileHook, _ := rotatefilehook.NewRotateFileHook(rotatefilehook.RotateFileConfig{ + Filename: filePath, + MaxSize: 10, // megabytes + MaxBackups: 3, + MaxAge: 0, // days + Level: logrus.DebugLevel, + Formatter: &ecslogrus.Formatter{ + DataKey: "labels", + }, + }) + return rotateFileHook + } + return nil +} + +func vcsRevision() string { + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + return "(unknown)" + } + for _, val := range buildInfo.Settings { + if val.Key == "vcs.revision" { + return val.Value + } + } + return "(unknown)" +} + +// validateInput checks that all required input parameters that do not have sensible +// defaults are actually set. +func validateInput(cfg config.Config) error { + switch "" { + case cfg.EventRepositoryHost(): + return errors.New("-eventrepository_url input or ETOS_GRAPHQL_SERVER environment variable must be set") + default: + return nil + } +} diff --git a/cmd/logarea/main.go b/cmd/logarea/main.go index b577f70..7ce6c3e 100644 --- a/cmd/logarea/main.go +++ b/cmd/logarea/main.go @@ -24,7 +24,7 @@ import ( "syscall" "time" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/logarea" "github.com/eiffel-community/etos-api/internal/logging" "github.com/eiffel-community/etos-api/internal/server" "github.com/eiffel-community/etos-api/pkg/application" diff --git a/cmd/sse/main.go b/cmd/sse/main.go index 3fdc0cc..c18b87d 100644 --- a/cmd/sse/main.go +++ b/cmd/sse/main.go @@ -24,7 +24,7 @@ import ( "syscall" "time" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/sse" "github.com/eiffel-community/etos-api/internal/logging" "github.com/eiffel-community/etos-api/internal/server" "github.com/eiffel-community/etos-api/pkg/application" diff --git a/deploy/etos-iut/Dockerfile b/deploy/etos-iut/Dockerfile new file mode 100644 index 0000000..f5d47fe --- /dev/null +++ b/deploy/etos-iut/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.21-alpine AS build +WORKDIR /tmp/iut +COPY . . +RUN apk add --no-cache make=4.4.1-r2 git=2.45.2-r0 && make iut + +FROM alpine:3.17.3 +ARG TZ +ENV TZ=$TZ + +LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-api +LABEL org.opencontainers.image.authors=etos-maintainers@googlegroups.com +LABEL org.opencontainers.image.licenses=Apache-2.0 + +RUN apk add --no-cache tzdata=2024a-r0 +ENTRYPOINT ["/app/iut"] + +COPY --from=build /tmp/iut/bin/iut /app/iut diff --git a/deploy/etos-iut/Dockerfile.dev b/deploy/etos-iut/Dockerfile.dev new file mode 100644 index 0000000..99fd779 --- /dev/null +++ b/deploy/etos-iut/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM golang:1.21 +WORKDIR /app + +COPY ./go.mod ./go.sum ./ +RUN go mod tidy +COPY . . +RUN git config --global --add safe.directory /app +EXPOSE 8080 diff --git a/deploy/etos-iut/docker-compose.yml b/deploy/etos-iut/docker-compose.yml new file mode 100644 index 0000000..68c63c2 --- /dev/null +++ b/deploy/etos-iut/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.7" +services: + etos-logarea: + build: + context: . + dockerfile: ./deploy/etos-iut/Dockerfile.dev + args: + http_proxy: "${http_proxy}" + https_proxy: "${https_proxy}" + volumes: + - ./:/app + ports: + - 8080:8080 + env_file: + - ./configs/development.env + entrypoint: ["/bin/bash", "./scripts/entrypoint.sh"] diff --git a/go.mod b/go.mod index e5d97c9..d576c87 100644 --- a/go.mod +++ b/go.mod @@ -5,24 +5,38 @@ go 1.21 toolchain go1.22.1 require ( + github.com/eiffel-community/eiffelevents-sdk-go v0.0.0-20240807115026-5ca5c194b7dc github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 + github.com/google/uuid v1.6.0 github.com/jmespath/go-jmespath v0.4.0 github.com/julienschmidt/httprouter v1.3.0 + github.com/machinebox/graphql v0.2.2 github.com/maxcnunes/httpfake v1.2.4 + github.com/package-url/packageurl-go v0.1.0 + github.com/sethvargo/go-retry v0.3.0 github.com/sirupsen/logrus v1.9.3 github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.elastic.co/ecslogrus v1.0.0 - go.etcd.io/etcd/client/v3 v3.5.14 + go.etcd.io/etcd/api/v3 v3.5.15 + go.etcd.io/etcd/client/v3 v3.5.15 go.etcd.io/etcd/server/v3 v3.5.14 + go.opentelemetry.io/otel v1.20.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 + go.opentelemetry.io/otel/sdk v1.20.0 + go.opentelemetry.io/otel/trace v1.20.0 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 ) require ( + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Showmax/go-fqdn v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/clarketm/json v1.17.1 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -40,7 +54,6 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -51,10 +64,12 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/magefile/mage v1.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/matryer/is v1.4.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -62,21 +77,18 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.10 // indirect - go.etcd.io/etcd/api/v3 v3.5.14 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect go.etcd.io/etcd/client/v2 v2.305.14 // indirect go.etcd.io/etcd/pkg/v3 v3.5.14 // indirect go.etcd.io/etcd/raft/v3 v3.5.14 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect - go.opentelemetry.io/otel v1.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect go.opentelemetry.io/otel/metric v1.20.0 // indirect - go.opentelemetry.io/otel/sdk v1.20.0 // indirect - go.opentelemetry.io/otel/trace v1.20.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -86,7 +98,7 @@ require ( golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/go.sum b/go.sum index 22aa92c..62a35ab 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= +github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -22,6 +26,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= +github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= @@ -38,6 +44,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eiffel-community/eiffelevents-sdk-go v0.0.0-20240807115026-5ca5c194b7dc h1:yRg84ReJfuVCJ/TMzfCqL12Aoy4vUSrUUgcuE02mBJo= +github.com/eiffel-community/eiffelevents-sdk-go v0.0.0-20240807115026-5ca5c194b7dc/go.mod h1:Lt487E8lrDd/5hkCEyKHU/xZrqDjIgRNIDaoK/F3Yk4= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -111,8 +119,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -153,10 +161,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= +github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/maxcnunes/httpfake v1.2.4 h1:l7s/N7zuG6XpzG+5dUolg5SSoR3hANQxqzAkv+lREko= @@ -177,6 +189,8 @@ github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLyw github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/package-url/packageurl-go v0.1.0 h1:efWBc98O/dBZRg1pw2xiDzovnlMjCa9NPnfaiBduh8I= +github.com/package-url/packageurl-go v0.1.0/go.mod h1:C/ApiuWpmbpni4DIOECf6WCjFUZV7O1Fx7VAzrZHgBw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -205,6 +219,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -229,8 +245,14 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -241,14 +263,14 @@ go.elastic.co/ecslogrus v1.0.0 h1:o1qvcCNaq+eyH804AuK6OOiUupLIXVDfYjDtSLPwukM= go.elastic.co/ecslogrus v1.0.0/go.mod h1:vMdpljurPbwu+iFmNc/HSWCkn1Fu/dYde1o/adaEczo= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0= -go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU= -go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ= -go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= go.etcd.io/etcd/client/v2 v2.305.14 h1:v5ASLyFuMlVd/gKU6uf6Cod+vSWKa4Rsv9+eghl0Nwk= go.etcd.io/etcd/client/v2 v2.305.14/go.mod h1:AWYT0lLEkBuqVaGw0UVMtA4rxCb3/oGE8PxZ8cUS4tI= -go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg= -go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= go.etcd.io/etcd/pkg/v3 v3.5.14 h1:keuxhJiDCPjTKpW77GxJnnVVD5n4IsfvkDaqiqUMNEQ= go.etcd.io/etcd/pkg/v3 v3.5.14/go.mod h1:7o+DL6a7DYz9KSjWByX+NGmQPYinoH3D36VAu/B3JqA= go.etcd.io/etcd/raft/v3 v3.5.14 h1:mHnpbljpBBftmK+YUfp+49ivaCc126aBPLAnwDw0DnE= @@ -347,8 +369,8 @@ golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/config/config.go b/internal/configs/base/config.go similarity index 100% rename from internal/config/config.go rename to internal/configs/base/config.go diff --git a/internal/config/config_test.go b/internal/configs/base/config_test.go similarity index 100% rename from internal/config/config_test.go rename to internal/configs/base/config_test.go diff --git a/internal/configs/iut/config.go b/internal/configs/iut/config.go new file mode 100644 index 0000000..5fb99ab --- /dev/null +++ b/internal/configs/iut/config.go @@ -0,0 +1,139 @@ +// Copyright 2022 Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/sirupsen/logrus" +) + +// Config interface for retrieving configuration options. +type Config interface { + ServiceHost() string + ServicePort() string + LogLevel() string + LogFilePath() string + Timeout() time.Duration + EventRepositoryHost() string + IutWaitTimeoutHard() time.Duration + IutWaitTimeoutSoft() time.Duration + DatabaseURI() string +} + +// cfg implements the Config interface. +type cfg struct { + serviceHost string + servicePort string + logLevel string + logFilePath string + timeout time.Duration + databaseHost string + databasePort string + eventRepositoryHost string + iutWaitTimeoutHard time.Duration + iutWaitTimeoutSoft time.Duration +} + +// Get creates a config interface based on input parameters or environment variables. +func Get() Config { + var conf cfg + + defaultTimeout, err := time.ParseDuration(EnvOrDefault("REQUEST_TIMEOUT", "1m")) + if err != nil { + logrus.Panic(err) + } + + iutWaitTimeoutHard, err := time.ParseDuration(EnvOrDefault("IUT_WAIT_TIMEOUT", "1h")) + if err != nil { + logrus.Panic(err) + } + + iutWaitTimeoutSoft, err := time.ParseDuration(EnvOrDefault("IUT_WAIT_TIMEOUT_SOFT", "30m")) + if err != nil { + logrus.Panic(err) + } + + flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on") + flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on") + flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).") + flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.") + flag.DurationVar(&conf.timeout, "timeout", defaultTimeout, "Maximum timeout for requests to Provider Service.") + flag.StringVar(&conf.databaseHost, "database_host", EnvOrDefault("ETOS_ETCD_HOST", "etcd-client"), "Host to ETOS database") + flag.StringVar(&conf.databasePort, "database_port", EnvOrDefault("ETOS_ETCD_PORT", "2379"), "Port to ETOS database") + flag.StringVar(&conf.eventRepositoryHost, "eventrepository_url", os.Getenv("ETOS_GRAPHQL_SERVER"), "URL to the GraphQL server to use for event lookup.") + flag.DurationVar(&conf.iutWaitTimeoutHard, "hard iut wait timeout", iutWaitTimeoutHard, "Hard wait timeout for IUT checkout") + flag.DurationVar(&conf.iutWaitTimeoutSoft, "soft iut wait timeout", iutWaitTimeoutSoft, "Soft wait timeout for IUT checkout") + flag.Parse() + + return &conf +} + +// ServiceHost returns the host of the service. +func (c *cfg) ServiceHost() string { + return c.serviceHost +} + +// ServicePort returns the port of the service. +func (c *cfg) ServicePort() string { + return c.servicePort +} + +// LogLevel returns the log level. +func (c *cfg) LogLevel() string { + return c.logLevel +} + +// LogFilePath returns the path to where log files should be stored, including filename. +func (c *cfg) LogFilePath() string { + return c.logFilePath +} + +// Timeout returns the request timeout for Provider Service API. +func (c *cfg) Timeout() time.Duration { + return c.timeout +} + +// EventRepositoryHost returns the host to use for event lookups. +func (c *cfg) EventRepositoryHost() string { + return c.eventRepositoryHost +} + +// IutWaitTimeoutHard returns the hard timeout for IUT checkout +func (c *cfg) IutWaitTimeoutHard() time.Duration { + return c.iutWaitTimeoutHard +} + +// IutWaitTimeoutSoft returns the soft timeout for IUT checkout +func (c *cfg) IutWaitTimeoutSoft() time.Duration { + return c.iutWaitTimeoutSoft +} + +// DatabaseURI returns the URI to the ETOS database. +func (c *cfg) DatabaseURI() string { + return fmt.Sprintf("%s:%s", c.databaseHost, c.databasePort) +} + +// EnvOrDefault will look up key in environment variables and return if it exists, else return the fallback value. +func EnvOrDefault(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} diff --git a/internal/configs/iut/config_test.go b/internal/configs/iut/config_test.go new file mode 100644 index 0000000..6ad30ca --- /dev/null +++ b/internal/configs/iut/config_test.go @@ -0,0 +1,98 @@ +// Copyright 2022 Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// Test that it is possible to get a Cfg from Get with values taken from environment variables. +func TestGet(t *testing.T) { + port := "8080" + serverHost := "127.0.0.1" + logLevel := "DEBUG" + logFilePath := "path/to/a/file" + timeoutStr := "1m" + databaseHost := "etcd" + databasePort := "12345" + + os.Setenv("SERVICE_HOST", serverHost) + os.Setenv("SERVICE_PORT", port) + os.Setenv("LOGLEVEL", logLevel) + os.Setenv("LOG_FILE_PATH", logFilePath) + os.Setenv("REQUEST_TIMEOUT", timeoutStr) + os.Setenv("ETOS_ETCD_HOST", databaseHost) + os.Setenv("ETOS_ETCD_PORT", databasePort) + + timeout, _ := time.ParseDuration(timeoutStr) + + conf, ok := Get().(*cfg) + assert.Truef(t, ok, "cfg returned from get is not a config interface") + assert.Equal(t, port, conf.servicePort) + assert.Equal(t, serverHost, conf.serviceHost) + assert.Equal(t, logLevel, conf.logLevel) + assert.Equal(t, logFilePath, conf.logFilePath) + assert.Equal(t, databaseHost, conf.databaseHost) + assert.Equal(t, databasePort, conf.databasePort) + assert.Equal(t, timeout, conf.timeout) + assert.Equal(t, timeout, conf.timeout) +} + +type getter func() string + +// Test that the getters in the Cfg struct return the values from the struct. +func TestGetters(t *testing.T) { + conf := &cfg{ + serviceHost: "127.0.0.1", + servicePort: "8080", + logLevel: "TRACE", + logFilePath: "a/file/path.json", + databaseHost: "etcd", + databasePort: "12345", + } + tests := []struct { + name string + cfg *cfg + function getter + value string + }{ + {name: "ServiceHost", cfg: conf, function: conf.ServiceHost, value: conf.serviceHost}, + {name: "ServicePort", cfg: conf, function: conf.ServicePort, value: conf.servicePort}, + {name: "LogLevel", cfg: conf, function: conf.LogLevel, value: conf.logLevel}, + {name: "LogFilePath", cfg: conf, function: conf.LogFilePath, value: conf.logFilePath}, + {name: "DatabaseURI", cfg: conf, function: conf.DatabaseURI, value: fmt.Sprintf("%s:%s", conf.databaseHost, conf.databasePort)}, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.value, testCase.function()) + }) + } +} + +// TestTimeoutGetter tests the getter for Timeout. Similar to TestGetters, but since +// Timeout is not a "func() string" we separate its test. +func TestTimeoutGetter(t *testing.T) { + timeout, _ := time.ParseDuration("1m") + conf := &cfg{ + timeout: timeout, + } + assert.Equal(t, conf.timeout, conf.Timeout()) +} diff --git a/internal/configs/logarea/config.go b/internal/configs/logarea/config.go new file mode 100644 index 0000000..36c5f67 --- /dev/null +++ b/internal/configs/logarea/config.go @@ -0,0 +1,107 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "flag" + "fmt" + "os" +) + +// Config interface for retreiving configuration options. +type Config interface { + ServiceHost() string + ServicePort() string + LogLevel() string + LogFilePath() string + ETOSNamespace() string + DatabaseURI() string +} + +// cfg implements the Config interface. +type cfg struct { + serviceHost string + servicePort string + logLevel string + logFilePath string + etosNamespace string + databaseHost string + databasePort string +} + +// Get creates a config interface based on input parameters or environment variables. +func Get() Config { + var conf cfg + + flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on") + flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on") + flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).") + flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.") + flag.StringVar(&conf.etosNamespace, "etosnamespace", ReadNamespaceOrEnv("ETOS_NAMESPACE"), "Path, including filename, for the log files to create.") + flag.StringVar(&conf.databaseHost, "databasehost", EnvOrDefault("ETOS_ETCD_HOST", "etcd-client"), "Host to the database.") + flag.StringVar(&conf.databasePort, "databaseport", EnvOrDefault("ETOS_ETCD_PORT", "2379"), "Port to the database.") + + flag.Parse() + return &conf +} + +// ServiceHost returns the host of the service. +func (c *cfg) ServiceHost() string { + return c.serviceHost +} + +// ServicePort returns the port of the service. +func (c *cfg) ServicePort() string { + return c.servicePort +} + +// LogLevel returns the log level. +func (c *cfg) LogLevel() string { + return c.logLevel +} + +// LogFilePath returns the path to where log files should be stored, including filename. +func (c *cfg) LogFilePath() string { + return c.logFilePath +} + +// ETOSNamespace returns the ETOS namespace. +func (c *cfg) ETOSNamespace() string { + return c.etosNamespace +} + +// DatabaseURI returns the URI to the ETOS database. +func (c *cfg) DatabaseURI() string { + return fmt.Sprintf("%s:%s", c.databaseHost, c.databasePort) +} + +// EnvOrDefault will look up key in environment variables and return if it exists, else return the fallback value. +func EnvOrDefault(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +// ReadNamespaceOrEnv checks if there's a nemspace file inside the container, else returns +// environment variable with envKey as name. +func ReadNamespaceOrEnv(envKey string) string { + inClusterNamespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return os.Getenv(envKey) + } + return string(inClusterNamespace) +} diff --git a/internal/configs/logarea/config_test.go b/internal/configs/logarea/config_test.go new file mode 100644 index 0000000..abb00ee --- /dev/null +++ b/internal/configs/logarea/config_test.go @@ -0,0 +1,71 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test that it is possible to get a Cfg from Get with values taken from environment variables. +func TestGet(t *testing.T) { + port := "8080" + serverHost := "127.0.0.1" + logLevel := "DEBUG" + logFilePath := "path/to/a/file" + + os.Setenv("SERVICE_HOST", serverHost) + os.Setenv("SERVICE_PORT", port) + os.Setenv("LOGLEVEL", logLevel) + os.Setenv("LOG_FILE_PATH", logFilePath) + + conf, ok := Get().(*cfg) + assert.Truef(t, ok, "cfg returned from get is not a config interface") + assert.Equal(t, port, conf.servicePort) + assert.Equal(t, serverHost, conf.serviceHost) + assert.Equal(t, logLevel, conf.logLevel) + assert.Equal(t, logFilePath, conf.logFilePath) +} + +type getter func() string + +// Test that the getters in the Cfg struct return the values from the struct. +func TestGetters(t *testing.T) { + conf := &cfg{ + serviceHost: "127.0.0.1", + servicePort: "8080", + logLevel: "TRACE", + logFilePath: "a/file/path.json", + } + tests := []struct { + name string + cfg *cfg + function getter + value string + }{ + {name: "ServiceHost", cfg: conf, function: conf.ServiceHost, value: conf.serviceHost}, + {name: "ServicePort", cfg: conf, function: conf.ServicePort, value: conf.servicePort}, + {name: "LogLevel", cfg: conf, function: conf.LogLevel, value: conf.logLevel}, + {name: "LogFilePath", cfg: conf, function: conf.LogFilePath, value: conf.logFilePath}, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.value, testCase.function()) + }) + } +} diff --git a/internal/configs/sse/config.go b/internal/configs/sse/config.go new file mode 100644 index 0000000..36c5f67 --- /dev/null +++ b/internal/configs/sse/config.go @@ -0,0 +1,107 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "flag" + "fmt" + "os" +) + +// Config interface for retreiving configuration options. +type Config interface { + ServiceHost() string + ServicePort() string + LogLevel() string + LogFilePath() string + ETOSNamespace() string + DatabaseURI() string +} + +// cfg implements the Config interface. +type cfg struct { + serviceHost string + servicePort string + logLevel string + logFilePath string + etosNamespace string + databaseHost string + databasePort string +} + +// Get creates a config interface based on input parameters or environment variables. +func Get() Config { + var conf cfg + + flag.StringVar(&conf.serviceHost, "address", EnvOrDefault("SERVICE_HOST", "127.0.0.1"), "Address to serve API on") + flag.StringVar(&conf.servicePort, "port", EnvOrDefault("SERVICE_PORT", "8080"), "Port to serve API on") + flag.StringVar(&conf.logLevel, "loglevel", EnvOrDefault("LOGLEVEL", "INFO"), "Log level (TRACE, DEBUG, INFO, WARNING, ERROR, FATAL, PANIC).") + flag.StringVar(&conf.logFilePath, "logfilepath", os.Getenv("LOG_FILE_PATH"), "Path, including filename, for the log files to create.") + flag.StringVar(&conf.etosNamespace, "etosnamespace", ReadNamespaceOrEnv("ETOS_NAMESPACE"), "Path, including filename, for the log files to create.") + flag.StringVar(&conf.databaseHost, "databasehost", EnvOrDefault("ETOS_ETCD_HOST", "etcd-client"), "Host to the database.") + flag.StringVar(&conf.databasePort, "databaseport", EnvOrDefault("ETOS_ETCD_PORT", "2379"), "Port to the database.") + + flag.Parse() + return &conf +} + +// ServiceHost returns the host of the service. +func (c *cfg) ServiceHost() string { + return c.serviceHost +} + +// ServicePort returns the port of the service. +func (c *cfg) ServicePort() string { + return c.servicePort +} + +// LogLevel returns the log level. +func (c *cfg) LogLevel() string { + return c.logLevel +} + +// LogFilePath returns the path to where log files should be stored, including filename. +func (c *cfg) LogFilePath() string { + return c.logFilePath +} + +// ETOSNamespace returns the ETOS namespace. +func (c *cfg) ETOSNamespace() string { + return c.etosNamespace +} + +// DatabaseURI returns the URI to the ETOS database. +func (c *cfg) DatabaseURI() string { + return fmt.Sprintf("%s:%s", c.databaseHost, c.databasePort) +} + +// EnvOrDefault will look up key in environment variables and return if it exists, else return the fallback value. +func EnvOrDefault(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +// ReadNamespaceOrEnv checks if there's a nemspace file inside the container, else returns +// environment variable with envKey as name. +func ReadNamespaceOrEnv(envKey string) string { + inClusterNamespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return os.Getenv(envKey) + } + return string(inClusterNamespace) +} diff --git a/internal/configs/sse/config_test.go b/internal/configs/sse/config_test.go new file mode 100644 index 0000000..abb00ee --- /dev/null +++ b/internal/configs/sse/config_test.go @@ -0,0 +1,71 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test that it is possible to get a Cfg from Get with values taken from environment variables. +func TestGet(t *testing.T) { + port := "8080" + serverHost := "127.0.0.1" + logLevel := "DEBUG" + logFilePath := "path/to/a/file" + + os.Setenv("SERVICE_HOST", serverHost) + os.Setenv("SERVICE_PORT", port) + os.Setenv("LOGLEVEL", logLevel) + os.Setenv("LOG_FILE_PATH", logFilePath) + + conf, ok := Get().(*cfg) + assert.Truef(t, ok, "cfg returned from get is not a config interface") + assert.Equal(t, port, conf.servicePort) + assert.Equal(t, serverHost, conf.serviceHost) + assert.Equal(t, logLevel, conf.logLevel) + assert.Equal(t, logFilePath, conf.logFilePath) +} + +type getter func() string + +// Test that the getters in the Cfg struct return the values from the struct. +func TestGetters(t *testing.T) { + conf := &cfg{ + serviceHost: "127.0.0.1", + servicePort: "8080", + logLevel: "TRACE", + logFilePath: "a/file/path.json", + } + tests := []struct { + name string + cfg *cfg + function getter + value string + }{ + {name: "ServiceHost", cfg: conf, function: conf.ServiceHost, value: conf.serviceHost}, + {name: "ServicePort", cfg: conf, function: conf.ServicePort, value: conf.servicePort}, + {name: "LogLevel", cfg: conf, function: conf.LogLevel, value: conf.logLevel}, + {name: "LogFilePath", cfg: conf, function: conf.LogFilePath, value: conf.logFilePath}, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.value, testCase.function()) + }) + } +} diff --git a/internal/iut/checkoutable/checkoutable.go b/internal/iut/checkoutable/checkoutable.go new file mode 100644 index 0000000..6fb9397 --- /dev/null +++ b/internal/iut/checkoutable/checkoutable.go @@ -0,0 +1,74 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package checkoutable handles checkout/checkin operations +package checkoutable + +import ( + "context" + "encoding/json" + "time" + + "github.com/google/uuid" + "github.com/sirupsen/logrus" +) + +type Iut struct { + ProviderId string `json:"provider_id,omitempty"` + Identity string `json:"identity"` + ArtifactId string `json:"artifact_id"` + Reference string `json:"reference"` + TestRunner interface{} `json:"test_runner,omitempty"` + logger *logrus.Entry +} + +// Fulfill json marshall interface for StatusResponses +func (s Iut) MarshalBinary() ([]byte, error) { + return json.Marshal(s) +} + +// NewIut creates a new iut struct +func NewIut(identity string, artifactId string, logger *logrus.Entry) *Iut { + return &Iut{ + Identity: identity, + ArtifactId: artifactId, + logger: logger, + } +} + +// AddLogger adds a logger to the iut struct, can be used if iut is created without NewIut +func (i *Iut) AddLogger(logger *logrus.Entry) { + i.logger = logger +} + +// Checkout checks out an IUT. +func (i *Iut) Checkout(ctx context.Context, configurationName string, eiffelContext uuid.UUID) error { + i.logger.Infof("checking out IUT %s, reference: %s", i.Identity, i.Reference) + return ctx.Err() +} + +// CheckIn checks in IUT +func (i *Iut) CheckIn(withForce bool) ([]string, error) { + // We shall ALWAYS try to check in IUTs if requested. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + if i.Reference == "" { + return nil, ctx.Err() + } + var references []string + return references, ctx.Err() +} diff --git a/internal/iut/contextmanager/contextmanager.go b/internal/iut/contextmanager/contextmanager.go new file mode 100644 index 0000000..ce3e09b --- /dev/null +++ b/internal/iut/contextmanager/contextmanager.go @@ -0,0 +1,84 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package contextmanager + +import ( + "context" + "regexp" + "strings" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const REGEX = "/testrun/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/provider/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/status" + +// ContextManager manages contexts for the IUT provider goroutines, enabling the cancelation +// of the context in the case where a testrun should be removed during the checkout phase. +type ContextManager struct { + contexts map[string]context.CancelFunc + db *clientv3.Client +} + +// New creates a new context manager. +func New(db *clientv3.Client) *ContextManager { + return &ContextManager{ + contexts: make(map[string]context.CancelFunc), + db: db, + } +} + +// CancelAll cancels all saved contexts within the context manager. +func (c *ContextManager) CancelAll() { + for _, cancel := range c.contexts { + cancel() + } + c.contexts = make(map[string]context.CancelFunc) +} + +// Start up the context manager testrun deletaion watcher. +func (c *ContextManager) Start(ctx context.Context) { + regex := regexp.MustCompile(REGEX) + ch := c.db.Watch(ctx, "/testrun", clientv3.WithPrefix()) + for response := range ch { + for _, event := range response.Events { + if event.Type == mvccpb.DELETE { + if !regex.Match(event.Kv.Key) { + continue + } + dbPath := strings.Split(string(event.Kv.Key), "/") + c.Cancel(dbPath[2]) + } + } + } +} + +// Cancel the context for one stored ID. +func (c *ContextManager) Cancel(id string) { + cancel, ok := c.contexts[id] + if !ok { + return + } + delete(c.contexts, id) + cancel() +} + +// Add a new context to the context manager. +func (c *ContextManager) Add(ctx context.Context, id string) context.Context { + ctx, cancel := context.WithCancel(ctx) + c.contexts[id] = cancel + return ctx +} diff --git a/internal/iut/eventrepository/eventrepository.go b/internal/iut/eventrepository/eventrepository.go new file mode 100644 index 0000000..275e832 --- /dev/null +++ b/internal/iut/eventrepository/eventrepository.go @@ -0,0 +1,194 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package eventrepository + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + config "github.com/eiffel-community/etos-api/internal/configs/iut" + + "github.com/eiffel-community/eiffelevents-sdk-go" + "github.com/machinebox/graphql" + "github.com/sirupsen/logrus" +) + +// EventRepository is the interface used no matter which event repository is used. +type EventRepository interface { + GetArtifactByID(context.Context, string) (eiffelevents.ArtifactCreatedV3, error) + GetArtifactByIdentity(context.Context, string) (eiffelevents.ArtifactCreatedV3, error) + GetArtifactPublishedByArtifactCreatedID(ctx context.Context, id string) (eiffelevents.ArtifactPublishedV3, error) +} + +type artCResponse struct { + Items []eiffelevents.ArtifactCreatedV3 `json:"items,omitempty"` +} + +type artPResponse struct { + Items []eiffelevents.ArtifactPublishedV3 `json:"items,omitempty"` +} + +// ER implements EventRepository +type ER struct { + cfg config.Config + logger *logrus.Entry +} + +// NewER creates a new ER client implementing the EventRepository interface. +func NewER(cfg config.Config, logger *logrus.Entry) EventRepository { + return &ER{ + cfg: cfg, + logger: logger, + } +} + +// GetArtifactByID uses the 'meta.id' key in artifact created events to find +// the artifact representation in ER. +func (er *ER) GetArtifactByID(ctx context.Context, id string) (eiffelevents.ArtifactCreatedV3, error) { + query := map[string]string{"meta.id": id, "meta.type": "EiffelArtifactCreatedEvent", "pageSize": "1"} + body, err := er.getEvents(ctx, query) + if err != nil { + return eiffelevents.ArtifactCreatedV3{}, err + } + var event artCResponse + if err = json.Unmarshal(body, &event); err != nil { + return eiffelevents.ArtifactCreatedV3{}, err + } + if len(event.Items) == 0 { + return eiffelevents.ArtifactCreatedV3{}, errors.New("no artifact created event found") + } + return event.Items[0], ctx.Err() +} + +// GetArtifactPublishedByArtifactCreatedID uses the 'meta.id' from an artifact created events to find +// the linked artifact published event in the Eventrepository. +func (er *ER) GetArtifactPublishedByArtifactCreatedID(ctx context.Context, id string) (eiffelevents.ArtifactPublishedV3, error) { + query := map[string]string{"links.target": id, "meta.type": "EiffelArtifactPublishedEvent", "links.type": "ARTIFACT", "pageSize": "1"} + body, err := er.getEvents(ctx, query) + if err != nil { + return eiffelevents.ArtifactPublishedV3{}, err + } + var event artPResponse + if err = json.Unmarshal(body, &event); err != nil { + return eiffelevents.ArtifactPublishedV3{}, err + } + if len(event.Items) == 0 { + return eiffelevents.ArtifactPublishedV3{}, errors.New("no artifact published event found") + } + return event.Items[0], ctx.Err() +} + +// getEvents creates an eventrepository event query and sends it to the GoER eventrepository. +func (er *ER) getEvents(ctx context.Context, query map[string]string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/v1/events", er.cfg.EventRepositoryHost()), nil) + if err != nil { + return nil, err + } + q := req.URL.Query() + for key, value := range query { + q.Add(key, value) + } + req.URL.RawQuery = q.Encode() + + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return nil, err + } + if response.StatusCode == 404 { + return nil, errors.New("event not found in event repository") + } + + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + return body, ctx.Err() +} + +// GetArtifactByIdentity uses the 'data.identity' key in artifact created events to find +// the artifact representation in ER. +func (er *ER) GetArtifactByIdentity(ctx context.Context, identity string) (eiffelevents.ArtifactCreatedV3, error) { + return er.artifact(ctx, fmt.Sprintf("{'data.identity': {'$regex': '^%s'}}", identity)) +} + +// ArtifactQuery is a generic query for finding artifact created identity and id. +// It requires a searchString to be added as a parameter. +var ArtifactQuery = ` +query ArtifactQuery($searchString: String) { + artifactCreated(search: $searchString, last: 1) { + edges { + node { + data { + identity + } + meta { + id + } + } + } + } +} +` + +// Edge holds the Artifact node. +type Edge struct { + Node eiffelevents.ArtifactCreatedV3 `json:"node"` +} + +// GraphQLArtifact is a slice of Edges that represent artifacts. +type GraphQLArtifact struct { + Edges []Edge `json:"edges"` +} + +// GraphQLResponse is the response from the GraphQL event repository. +type GraphQLResponse struct { + ArtifactCreated GraphQLArtifact `json:"artifactCreated"` +} + +// NoArtifact indicates that no artifacts were found. +type NoArtifact struct { + Message string +} + +// Error returns the string representation of the NoArtifact error. +func (e *NoArtifact) Error() string { + return e.Message +} + +// artifact makes a request against the event repository with the searchString provided. +func (er *ER) artifact(ctx context.Context, searchString string) (eiffelevents.ArtifactCreatedV3, error) { + request := graphql.NewRequest(ArtifactQuery) + request.Var("searchString", searchString) + + var response GraphQLResponse + gqlClient := graphql.NewClient(fmt.Sprintf("%s/graphql", er.cfg.EventRepositoryHost())) + if err := gqlClient.Run(ctx, request, &response); err != nil { + return eiffelevents.ArtifactCreatedV3{}, err + } + edges := response.ArtifactCreated.Edges + if len(edges) == 0 { + return eiffelevents.ArtifactCreatedV3{}, &NoArtifact{fmt.Sprintf("no artifact returned for %s", searchString)} + } + // The query is limited to, at most, one response and we test that it is not zero just before this + // which makes it safe to get the 0 index from edges. + return edges[0].Node, ctx.Err() +} diff --git a/internal/iut/eventrepository/eventrepository_test.go b/internal/iut/eventrepository/eventrepository_test.go new file mode 100644 index 0000000..4f05c7d --- /dev/null +++ b/internal/iut/eventrepository/eventrepository_test.go @@ -0,0 +1,16 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package eventrepository diff --git a/internal/iut/responses/responses.go b/internal/iut/responses/responses.go new file mode 100644 index 0000000..458ef23 --- /dev/null +++ b/internal/iut/responses/responses.go @@ -0,0 +1,35 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package responses + +import ( + "encoding/json" + "net/http" +) + +// RespondWithJSON writes a JSON response with a status code to the HTTP ResponseWriter. +func RespondWithJSON(w http.ResponseWriter, code int, payload interface{}) { + response, _ := json.Marshal(payload) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _, _ = w.Write(response) +} + +// RespondWithError writes a JSON response with an error message and status code to the HTTP ResponseWriter. +func RespondWithError(w http.ResponseWriter, code int, message string) { + RespondWithJSON(w, code, map[string]string{"error": message}) +} diff --git a/internal/iut/responses/responses_test.go b/internal/iut/responses/responses_test.go new file mode 100644 index 0000000..a819deb --- /dev/null +++ b/internal/iut/responses/responses_test.go @@ -0,0 +1,42 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package responses + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test that RespondWithJSON writes the correct HTTP code, message and adds a content type header. +func TestRespondWithJSON(t *testing.T) { + responseRecorder := httptest.NewRecorder() + RespondWithJSON(responseRecorder, 200, map[string]string{"hello": "world"}) + assert.Equal(t, "application/json", responseRecorder.Header().Get("Content-Type")) + assert.Equal(t, 200, responseRecorder.Result().StatusCode) + assert.JSONEq(t, `{"hello": "world"}`, responseRecorder.Body.String()) +} + +// Test that RespondWithError writes the correct HTTP code, message and adds a content type header. +func TestRespondWithError(t *testing.T) { + responseRecorder := httptest.NewRecorder() + RespondWithError(responseRecorder, 400, "failure") + assert.Equal(t, "application/json", responseRecorder.Header().Get("Content-Type")) + assert.Equal(t, 400, responseRecorder.Result().StatusCode) + assert.JSONEq(t, `{"error": "failure"}`, responseRecorder.Body.String()) +} diff --git a/internal/iut/server/server.go b/internal/iut/server/server.go new file mode 100644 index 0000000..3772638 --- /dev/null +++ b/internal/iut/server/server.go @@ -0,0 +1,63 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package server + +import ( + "context" + "fmt" + "net/http" + + config "github.com/eiffel-community/etos-api/internal/configs/iut" + "github.com/sirupsen/logrus" +) + +// Server interface for serving up the Provider Service. +type Server interface { + Start() error + Close(ctx context.Context) error +} + +// Webserver is a struct for webservers implementing the Server interface. +type WebServer struct { + server *http.Server + cfg config.Config + logger *logrus.Entry +} + +// NewWebserver creates a new Server of the webserver type. +func NewWebserver(cfg config.Config, log *logrus.Entry, handler http.Handler) Server { + webserver := &WebServer{ + server: &http.Server{ + Addr: fmt.Sprintf("%s:%s", cfg.ServiceHost(), cfg.ServicePort()), + Handler: handler, + }, + cfg: cfg, + logger: log, + } + return webserver +} + +// Start a webserver and block until closed or crashed. +func (s *WebServer) Start() error { + s.logger.Infof("Starting webserver listening on %s:%s", s.cfg.ServiceHost(), s.cfg.ServicePort()) + return s.server.ListenAndServe() +} + +// Close calls shutdown on the webserver. Shutdown times out if context is cancelled. +func (s *WebServer) Close(ctx context.Context) error { + s.logger.Info("Shutting down webserver") + return s.server.Shutdown(ctx) +} diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index a43a955..cb8bee7 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -19,7 +19,7 @@ import ( "context" "fmt" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/base" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" diff --git a/internal/server/server.go b/internal/server/server.go index 6529e53..79fd9f3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -20,7 +20,7 @@ import ( "fmt" "net/http" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/base" "github.com/sirupsen/logrus" ) diff --git a/manifests/base/iut/deployment.yaml b/manifests/base/iut/deployment.yaml new file mode 100644 index 0000000..c7a8d0b --- /dev/null +++ b/manifests/base/iut/deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: etos-api + app.kubernetes.io/part-of: etos + app.kubernetes.io/component: iut + name: etos-iut +spec: + selector: + matchLabels: + app.kubernetes.io/name: etos-api + app.kubernetes.io/component: iut + template: + metadata: + labels: + app.kubernetes.io/name: etos-api + app.kubernetes.io/component: iut + spec: + serviceAccountName: etos-iut + containers: + - name: etos-iut + image: registry.nordix.org/eiffel/etos-iut:672f982e + imagePullPolicy: IfNotPresent + env: + - name: SERVICE_HOST + value: 0.0.0.0 + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /v1alpha/selftest/ping + port: http + readinessProbe: + httpGet: + path: /v1alpha/selftest/ping + port: http diff --git a/manifests/base/iut/kustomization.yaml b/manifests/base/iut/kustomization.yaml new file mode 100644 index 0000000..581fbe2 --- /dev/null +++ b/manifests/base/iut/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - service-account.yaml + - service.yaml + - deployment.yaml diff --git a/manifests/base/iut/service-account.yaml b/manifests/base/iut/service-account.yaml new file mode 100644 index 0000000..9208316 --- /dev/null +++ b/manifests/base/iut/service-account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: etos-api + app.kubernetes.io/part-of: etos + app.kubernetes.io/component: iut + name: etos-iut diff --git a/manifests/base/iut/service.yaml b/manifests/base/iut/service.yaml new file mode 100644 index 0000000..04afeb8 --- /dev/null +++ b/manifests/base/iut/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: etos-api + app.kubernetes.io/part-of: etos + app.kubernetes.io/component: iut + name: etos-iut +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/name: etos-api + app.kubernetes.io/component: iut + type: ClusterIP diff --git a/manifests/base/kustomization.yaml b/manifests/base/kustomization.yaml index 7aa5492..dbaa3bd 100644 --- a/manifests/base/kustomization.yaml +++ b/manifests/base/kustomization.yaml @@ -9,6 +9,7 @@ resources: - deployment.yaml - ./sse - ./logarea + - ./iut # By generating the configmap it will get a unique name on each apply diff --git a/pkg/iut/application/application.go b/pkg/iut/application/application.go new file mode 100644 index 0000000..cb4fc0d --- /dev/null +++ b/pkg/iut/application/application.go @@ -0,0 +1,35 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package application + +import "github.com/julienschmidt/httprouter" + +type Application interface { + LoadRoutes(*httprouter.Router) + Close() +} + +// New loads routes for all applications supplied and returns a new router to +// be used in the server. +func New(application Application, applications ...Application) *httprouter.Router { + router := httprouter.New() + application.LoadRoutes(router) + for _, app := range applications { + app.LoadRoutes(router) + } + return router +} diff --git a/pkg/iut/application/application_test.go b/pkg/iut/application/application_test.go new file mode 100644 index 0000000..0e5d682 --- /dev/null +++ b/pkg/iut/application/application_test.go @@ -0,0 +1,83 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package application + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/julienschmidt/httprouter" + "github.com/stretchr/testify/assert" +) + +type testApp struct { + route string + message string +} + +// testRoute is a test route that prints a test message from the app to which it is "attached". +func (t *testApp) testRoute(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprint(w, t.message) +} + +// LoadRoutes from the application to which it is "attached". +func (t *testApp) LoadRoutes(router *httprouter.Router) { + router.GET(t.route, t.testRoute) +} + +// Close is a placeholder to fulfill implementation of the application interface. +func (t *testApp) Close() {} + +// TestNew verifies that it is possible to load a handlers routes. +func TestNew(t *testing.T) { + app := &testApp{ + route: "/testing", + message: "hello", + } + router := New(app) + + responseRecorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", app.route, nil) + router.ServeHTTP(responseRecorder, request) + assert.Equal(t, 200, responseRecorder.Code) + assert.Equal(t, app.message, responseRecorder.Body.String()) +} + +// TestNew verifies that it is possible to load multiple handlers routes. +func TestNewMultiple(t *testing.T) { + route1 := &testApp{"/route1", "hello1"} + route2 := &testApp{"/route2", "hello2"} + tests := []struct { + name string + app *testApp + }{ + {name: "Route1", app: route1}, + {name: "Route1", app: route2}, + } + + router := New(route1, route2) + + for _, testCase := range tests { + responseRecorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", testCase.app.route, nil) + router.ServeHTTP(responseRecorder, request) + assert.Equal(t, 200, responseRecorder.Code) + assert.Equal(t, testCase.app.message, responseRecorder.Body.String()) + } +} diff --git a/pkg/iut/v1alpha1/checkin.go b/pkg/iut/v1alpha1/checkin.go new file mode 100644 index 0000000..8c1b445 --- /dev/null +++ b/pkg/iut/v1alpha1/checkin.go @@ -0,0 +1,78 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/eiffel-community/etos-api/internal/iut/checkoutable" + "github.com/sirupsen/logrus" +) + +// TrimCheckin trims and returns the provided list of Iuts and also checks in the trimmed Iuts. +func (h V1Alpha1Handler) TrimCheckin(ctx context.Context, logger *logrus.Entry, iuts []*checkoutable.Iut, status *Status, size int) ([]*checkoutable.Iut, error) { + logger.Infof("trimming the length of IUT list and checking in the removed iut(s)") + + if len(iuts) <= size { + return nil, fmt.Errorf( + "the trim size [%d] must be smaller than the length of the provided list [%d]", + size, + len(iuts), + ) + } + + var trimmedIuts []*checkoutable.Iut + for i := 1; i <= len(iuts)-size; i++ { + trimmedIuts = append(trimmedIuts, iuts[len(iuts)-i]) + } + + if err := h.checkIn(ctx, logger, trimmedIuts, status, true); err != nil { + return nil, fmt.Errorf("failed to checkin trimmed iuts - Reason: %s", err.Error()) + } + + return iuts[:size], ctx.Err() +} + +// Checkin checks in the provided list of Iuts +func (h V1Alpha1Handler) checkIn(ctx context.Context, logger *logrus.Entry, iuts []*checkoutable.Iut, status *Status, withForce bool) error { + var anyError error + var references []string + for _, iut := range iuts { + var refs []string + refs, err := iut.CheckIn(withForce) + if err != nil { + anyError = errors.Join(anyError, err) + references = append(references, refs...) + continue + } + references = append(references, refs...) + failCtx, c := context.WithTimeout(context.Background(), time.Second*10) + defer c() + if err := status.RemoveIUT(failCtx, iut); err != nil { + anyError = errors.Join(anyError, err) + } + } + if anyError != nil { + return errors.Join(fmt.Errorf( + "failed to checkin Iut. Checkout refs: %+v", references, + ), anyError) + } + return ctx.Err() +} diff --git a/pkg/iut/v1alpha1/checkout.go b/pkg/iut/v1alpha1/checkout.go new file mode 100644 index 0000000..6564242 --- /dev/null +++ b/pkg/iut/v1alpha1/checkout.go @@ -0,0 +1,250 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "context" + "fmt" + "runtime" + "sync" + "time" + + "github.com/eiffel-community/etos-api/internal/iut/checkoutable" + "github.com/google/uuid" + "github.com/sethvargo/go-retry" + "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/trace" +) + +func (h V1Alpha1Handler) Success( + ctx context.Context, + logger *logrus.Entry, + status *Status, + checkedOutIuts []*checkoutable.Iut, +) error { + if err := status.Done(ctx, "IUTs checked out successfully"); err != nil { + logger.Errorf("failed to write success status to database - Reason: %s", err.Error()) + logger.Infof("clean up, checking in any checked out IUTs") + if checkInErr := h.checkIn(ctx, logger, checkedOutIuts, status, false); checkInErr != nil { + logger.Errorf("clean up failure - Reason: %s", checkInErr.Error()) + } + return err + } + return ctx.Err() +} + +// AvailableIuts list all currently available IUTs. +func (h V1Alpha1Handler) AvailableIuts( + ctx context.Context, + req StartRequest, + logger *logrus.Entry, + identity string, + artifactId string, +) []*checkoutable.Iut { + availableIuts := make([]*checkoutable.Iut, 1) + return availableIuts +} + +// checkout checks out an IUT and stores it in database +func (h V1Alpha1Handler) checkout(ctx context.Context, identifier string, iut *checkoutable.Iut, req StartRequest, status *Status) error { + if err := iut.Checkout( + ctx, + identifier, + req.Context, + ); err != nil { + return err + } + return status.AddIUT(ctx, iut) +} + +// DoCheckout handles the checkout (and flash optionally) process for the provided iuts +func (h V1Alpha1Handler) DoCheckout( + ctx context.Context, + logger *logrus.Entry, + req StartRequest, + checkoutID uuid.UUID, + identifier string, +) { + h.wg.Add(1) + defer h.wg.Done() + + // Panic recovery + defer func() { + if err := recover(); err != nil { + buf := make([]byte, 2048) + n := runtime.Stack(buf, false) + buf = buf[:n] + logger.Errorf("recovering from err %v\n %s", err, buf) + } + }() + + var err error + _, span := h.getOtelTracer().Start(ctx, "do_checkout", trace.WithSpanKind(trace.SpanKindInternal)) + defer span.End() + + ctx = h.cm.Add(ctx, identifier) + defer h.cm.Cancel(identifier) + response, err := h.database.Get(ctx, fmt.Sprintf("/testrun/%s/provider/iut", identifier)) + if err != nil { + msg := fmt.Errorf("error getting testrun from database - %s", err.Error()) + logger.Error(msg) + h.recordOtelException(span, msg) + return + } + if len(response.Kvs) == 0 { + msg := fmt.Errorf("no testrun available for checkout with id: %s", identifier) + logger.Error(msg) + h.recordOtelException(span, msg) + return + } + + key := fmt.Sprintf("/testrun/%s", identifier) + logger.Debugf("storing IUT information at %s", key) + status := NewStatus(key, checkoutID.String(), h.database) + + greed := 0.8 + + ctx, cancel := context.WithTimeout(ctx, h.cfg.IutWaitTimeoutHard()) + defer cancel() + + if err = status.Pending(ctx, "checking out IUTs"); err != nil { + msg := fmt.Errorf("failed to write checkout pending status to database - %s", err.Error()) + logger.Error(msg) + h.recordOtelException(span, msg) + return + } + + var successIuts []*checkoutable.Iut + var iuts []*checkoutable.Iut + // Due to a how retry-go implements WithCappedDuration, we need to have a + // base duration that is larger than the jitter. This is because if a + // negative jitter value causes the duration to become negative the + // WithCappedDuration function handles this by returning the cap duration. + backOff := retry.WithCappedDuration( + 60*time.Second, + retry.WithJitter( + 2*time.Second, + retry.NewFibonacci( + 5*time.Second, + ), + ), + ) + // Try to checkout all the "available" Iuts and assess if we got enough to return successfully, + // if not we retry until we do or the context timeout (default 1h) kicks in. + if err := retry.Do(ctx, backOff, func(ctx context.Context) error { + logger.Debugf("length before update %d", len(iuts)) + iuts = h.AvailableIuts(ctx, req, logger, req.ArtifactIdentity, req.ArtifactID) + logger.Debugf("length after update %d", len(iuts)) + if len(iuts) < req.MinimumAmount { + return fmt.Errorf( + "Not enough Iuts available. Number of available iuts are: %d", + len(iuts), + ) + } + + fairAmount := fair(len(iuts), greed) + if fairAmount >= req.MaximumAmount { + fairAmount = req.MaximumAmount + } + success := make(chan *checkoutable.Iut, len(iuts)) + wg := sync.WaitGroup{} + + startTimestamp := time.Now().Unix() + + for _, iut := range iuts { + wg.Add(1) + // Goroutine to checkout an Iut, and to notify which were successful or not. + go func(ctx context.Context, iut *checkoutable.Iut, req StartRequest) { + defer wg.Done() + select { + case <-ctx.Done(): + return + default: + if err := h.checkout(ctx, identifier, iut, req, status); err != nil { + iut.CheckIn(true) + logger.Error(err) + return + } + success <- iut + } + }(ctx, iut, req) + time.Sleep(1 * time.Second) + } + wg.Wait() + duration := time.Now().Unix() - startTimestamp + softTimeout := int64(h.cfg.IutWaitTimeoutSoft().Seconds()) + + if duration > softTimeout { + logger.Infof("IUT checkout time exceeded soft timeout %d seconds", softTimeout) + } + + close(success) + for iut := range success { + successIuts = append(successIuts, iut) + } + logger.Infof("all checkout attempts done, %d IUTs successfully checked out", len(successIuts)) + + if len(successIuts) < req.MinimumAmount { + logger.Infof("Not enough Iuts were checked out, length is: %d", len(successIuts)) + logger.WithField("user_log", true).Info("Not enough IUTs available yet") + if err := h.checkIn(ctx, logger, successIuts, status, true); err != nil { + return err + } + successIuts = nil + return retry.RetryableError(fmt.Errorf("retrying... ")) + } + + if len(successIuts) > fairAmount { + successIuts, err = h.TrimCheckin(ctx, logger, successIuts, status, fairAmount) + if err != nil { + msg := fmt.Errorf("Error occured during trim check-in (fair amount: %d): %s", fairAmount, err.Error()) + h.recordOtelException(span, msg) + logger.Error(msg) + } + } + return nil + }); err != nil { + if ctx.Err() != nil { + err = fmt.Errorf("Timed out checking out IUT") + logger.WithField("user_log", true).Error(err.Error()) + h.recordOtelException(span, err) + } else { + logger.WithField("user_log", true).Errorf("Failed checking out IUT(s) - %s", err.Error()) + } + failCtx, c := context.WithTimeout(context.Background(), time.Second*10) + defer c() + if statusErr := status.Failed(failCtx, err.Error()); statusErr != nil { + logger.WithError(statusErr).Error("failed to write failure status to database") + } + return + } + successIuts = h.AddTestRunnerInfo(ctx, logger, status, successIuts) + for _, iut := range successIuts { + if err := status.AddIUT(ctx, iut); err != nil { + msg := fmt.Errorf("Failed to add IUT to database: %s - Reason: %s", iut, err.Error()) + h.recordOtelException(span, msg) + if statusErr := status.Failed(ctx, err.Error()); statusErr != nil { + logger.WithError(statusErr).Error("failed to write failure status to database") + } + } + } + + logger.WithField("user_log", true).Infof("Successfully checked out %d IUTs", len(successIuts)) + for _, iut := range successIuts { + logger.WithField("user_log", true).Infof("Reference: %s", iut.Reference) + } +} diff --git a/pkg/iut/v1alpha1/deprecated.go b/pkg/iut/v1alpha1/deprecated.go new file mode 100644 index 0000000..96f9cc5 --- /dev/null +++ b/pkg/iut/v1alpha1/deprecated.go @@ -0,0 +1,88 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/eiffel-community/eiffelevents-sdk-go" + "github.com/google/uuid" + "github.com/sirupsen/logrus" +) + +type DeprecatedStartRequest struct { + MinimumAmount int `json:"minimum_amount"` + MaximumAmount int `json:"maximum_amount"` + ArtifactIdentity string `json:"identity"` + ArtifactID string `json:"artifact_id"` + ArtifactCreated eiffelevents.ArtifactCreatedV3 `json:"artifact_created,omitempty"` + ArtifactPublished eiffelevents.ArtifactPublishedV3 `json:"artifact_published"` + TERCC eiffelevents.TestExecutionRecipeCollectionCreatedV4 `json:"tercc,omitempty"` + Context uuid.UUID `json:"context,omitempty"` + Dataset DeprecatedDataset `json:"dataset,omitempty"` +} + +type DeprecatedDataset struct { + Greed interface{} `json:"greed"` +} + +// startRequestFromDeprecatedStartRequest will take a DeprecatedStartRequest struct and with the power +// of ER will create a StartRequest +func (h V1Alpha1Handler) startRequestFromDeprecatedStartRequest(ctx context.Context, logger *logrus.Entry, deprecatedStartRequest DeprecatedStartRequest) (StartRequest, error) { + request := StartRequest{ + MinimumAmount: deprecatedStartRequest.MinimumAmount, + MaximumAmount: deprecatedStartRequest.MaximumAmount, + ArtifactIdentity: deprecatedStartRequest.ArtifactIdentity, + ArtifactID: deprecatedStartRequest.ArtifactID, + ArtifactCreated: deprecatedStartRequest.ArtifactCreated, + ArtifactPublished: deprecatedStartRequest.ArtifactPublished, + TERCC: deprecatedStartRequest.TERCC, + Context: deprecatedStartRequest.Context, + Dataset: Dataset{ + Greed: deprecatedStartRequest.Dataset.Greed, + }, + } + return request, nil +} + +// tryLoadNewStartRequest will attempt to load a []byte into the StartRequest struct +func (h V1Alpha1Handler) tryLoadNewStartRequest(ctx context.Context, logger *logrus.Entry, data []byte) (StartRequest, error) { + request := StartRequest{} + //er := eventrepository.NewER(h.cfg, logger) + err := json.Unmarshal(data, &request) + if err != nil { + return request, err + } + return request, nil +} + +// tryLoadStartRequest tries to first load the StartRequest, if that fails +// it will try to load the old, deprecated, version. +func (h V1Alpha1Handler) tryLoadStartRequest(ctx context.Context, logger *logrus.Entry, data []byte) (StartRequest, error) { + request, err := h.tryLoadNewStartRequest(ctx, logger, data) + if err == nil { + return request, nil + } + + deprecatedRequest := DeprecatedStartRequest{} + if err := json.Unmarshal(data, &deprecatedRequest); err != nil { + return request, fmt.Errorf("unable to decode post body - Reason: %s", err.Error()) + } + return h.startRequestFromDeprecatedStartRequest(ctx, logger, deprecatedRequest) +} diff --git a/pkg/iut/v1alpha1/errors.go b/pkg/iut/v1alpha1/errors.go new file mode 100644 index 0000000..1a49040 --- /dev/null +++ b/pkg/iut/v1alpha1/errors.go @@ -0,0 +1,43 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "fmt" + "net/http" +) + +// HTTPError is a wrapper around a standard error but also adding HTTP status code. +type HTTPError struct { + Original error + Message string + Code int +} + +// NewHTTPError creates a new HTTPError wrapping another error with it. +func NewHTTPError(e error, httpCode int) *HTTPError { + return &HTTPError{ + Original: e, + Message: e.Error(), + Code: httpCode, + } +} + +// Error is the string representation of HTTPError. +func (e *HTTPError) Error() string { + return fmt.Sprintf("(%d: %s): %s", e.Code, http.StatusText(e.Code), e.Message) +} diff --git a/pkg/iut/v1alpha1/status.go b/pkg/iut/v1alpha1/status.go new file mode 100644 index 0000000..dcecc61 --- /dev/null +++ b/pkg/iut/v1alpha1/status.go @@ -0,0 +1,197 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package v1alpha1 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/eiffel-community/etos-api/internal/iut/checkoutable" + clientv3 "go.etcd.io/etcd/client/v3" +) + +type Status struct { + Status string `json:"status"` + Description string `json:"description"` + Iuts []*checkoutable.Iut `json:"iuts"` + IutReferences []string `json:"iut_references"` + baseKey string + iutsKey string + statusKey string + db *clientv3.Client +} + +// NewStatus will return a new status field. Does not automatically load data from database. +func NewStatus(baseKey string, id string, db *clientv3.Client) *Status { + return &Status{ + baseKey: baseKey, + iutsKey: fmt.Sprintf("%s/provider/%s/iuts", baseKey, id), + statusKey: fmt.Sprintf("%s/provider/%s/status", baseKey, id), + db: db, + } +} + +// MarhslBinary fulfills json marshall interface for StatusResponses +func (s Status) MarshalBinary() ([]byte, error) { + return json.Marshal(s) +} + +// Pending sets status to PENDING and saves it to database. +func (s *Status) Pending(ctx context.Context, description string) error { + s.Status = "PENDING" + return s.Save(ctx) +} + +// Failed sets status to FAILED and saves it to database. +func (s *Status) Failed(ctx context.Context, description string) error { + s.Status = "FAILED" + return s.Save(ctx) +} + +// Done sets status to DONE and saves it to database. +func (s *Status) Done(ctx context.Context, description string) error { + s.Status = "DONE" + return s.Save(ctx) +} + +// AddIUT adds a IUT to Status and saves it to database. +func (s *Status) AddIUT(ctx context.Context, iut *checkoutable.Iut) error { + if err := s.addIut(ctx, iut); err != nil { + return err + } + return s.Save(ctx) +} + +// IUT returns an IUT from the database based on a checkout reference. +func (s Status) Iut(ctx context.Context, reference string) (*checkoutable.Iut, error) { + key := fmt.Sprintf("%s/%s", s.iutsKey, reference) + response, err := s.db.Get(ctx, key) + if err != nil { + return nil, err + } + if len(response.Kvs) == 0 { + return nil, ctx.Err() + } + value := response.Kvs[0].Value + var iut checkoutable.Iut + err = json.Unmarshal(value, &iut) + if err != nil { + return nil, err + } + return &iut, ctx.Err() +} + +// deleteReference deletes a reference from the iutReference slice. +func (s *Status) deleteReference(iut *checkoutable.Iut) { + for index, reference := range s.IutReferences { + if reference == iut.Reference { + newSlice := make([]string, 0) + newSlice = append(newSlice, s.IutReferences[:index]...) + s.IutReferences = append(newSlice, s.IutReferences[index+1:]...) + } + } +} + +// deleteIut deletes an IUT from the iuts slice. +func (s *Status) deleteIut(iutToRemove *checkoutable.Iut) { + for index, iut := range s.Iuts { + if iut.Reference == iutToRemove.Reference { + newSlice := make([]*checkoutable.Iut, 0) + newSlice = append(newSlice, s.Iuts[:index]...) + s.Iuts = append(newSlice, s.Iuts[index+1:]...) + } + } +} + +// RemoveIUT removes an IUT from the database and struct and saves it to database. +func (s *Status) RemoveIUT(ctx context.Context, iut *checkoutable.Iut) error { + if err := s.removeIut(ctx, iut); err != nil { + return err + } + s.deleteIut(iut) + return s.Save(ctx) +} + +// addIut adds an IUT to the struct and saves it to database. +func (s Status) addIut(ctx context.Context, iut *checkoutable.Iut) error { + data, err := iut.MarshalBinary() + if err != nil { + return err + } + _, err = s.db.Put(ctx, fmt.Sprintf("%s/%s", s.iutsKey, iut.Reference), string(data)) + if err != nil { + return err + } + return ctx.Err() +} + +// removeIut removes an IUT from the database. +func (s Status) removeIut(ctx context.Context, iut *checkoutable.Iut) error { + _, err := s.db.Delete(ctx, fmt.Sprintf("%s/%s", s.iutsKey, iut.Reference)) + if err != nil { + return err + } + return ctx.Err() +} + +// Save saves the status struct to the database. It does not save IUTs, they are saved when added. +func (s *Status) Save(ctx context.Context) error { + data, err := s.MarshalBinary() + if err != nil { + return err + } + for _, iut := range s.Iuts { + s.IutReferences = append(s.IutReferences, iut.Reference) + } + if ctx.Err() == nil { + _, err := s.db.Put(ctx, s.statusKey, string(data)) + if err != nil { + return fmt.Errorf("etcd put operation failed while saving status: key=%s, value=%s", s.statusKey, string(data)) + } + } + return ctx.Err() +} + +// Load a status struct from the data in the database. +func (s *Status) Load(ctx context.Context) error { + response, err := s.db.Get(ctx, s.statusKey) + if err != nil { + return err + } + if len(response.Kvs) == 0 { + return fmt.Errorf("no status found in database for %s", s.statusKey) + } + value := response.Kvs[0].Value + err = json.Unmarshal(value, s) + if err != nil { + return err + } + + response, err = s.db.Get(ctx, s.iutsKey, clientv3.WithPrefix()) + if err != nil { + return err + } + for _, kv := range response.Kvs { + var iut checkoutable.Iut + err = json.Unmarshal(kv.Value, &iut) + if err != nil { + return err + } + s.Iuts = append(s.Iuts, &iut) + } + return ctx.Err() +} diff --git a/pkg/iut/v1alpha1/testrunner.go b/pkg/iut/v1alpha1/testrunner.go new file mode 100644 index 0000000..a7b42cb --- /dev/null +++ b/pkg/iut/v1alpha1/testrunner.go @@ -0,0 +1,93 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package v1alpha1 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/eiffel-community/etos-api/internal/iut/checkoutable" + "github.com/sirupsen/logrus" +) + +type Environment struct { + IutInfo string `json:"IUT_INFO,omitempty"` +} + +type Steps struct { + Environment Environment `json:"environment"` + Commands []Command `json:"commands"` +} + +type TestRunner struct { + Steps Steps `json:"steps"` +} + +type Command struct { + Name string `json:"name"` + Parameters []string `json:"parameters"` + Script []string `json:"script"` +} + +type IutInfo struct { + Identity string `json:"identity"` +} + +// AddTestRunnerInfo adds Test runner specific information to the Iut struct +func (h V1Alpha1Handler) AddTestRunnerInfo(ctx context.Context, logger *logrus.Entry, status *Status, iuts []*checkoutable.Iut) []*checkoutable.Iut { + for idx, iut := range iuts { + iutInfo := IutInfo{ + Identity: iut.Identity, + } + + iutInfoJson, err := json.Marshal(iutInfo) + if err != nil { + err := fmt.Errorf("unable to marshall Iut Info json for %s", iut.Identity) + if statusErr := status.Failed(ctx, err.Error()); statusErr != nil { + logger.WithError(statusErr).Error("failed to write failure status to database") + } + } + + var commands []Command + var parameters []string + iut_info_command := Command{ + Name: "iut_info_json", + Parameters: append(parameters, ""), + Script: []string{ + "#!/bin/bash", + "mkdir -p $GLOBAL_ARTIFACT_PATH", + "cat << EOF | python -m json.tool > $TEST_ARTIFACT_PATH/iut_info.json", + string(iutInfoJson), + "EOF", + }, + } + + commands = append(commands, iut_info_command) + //commands = append(commands, h.createDutMetadataCommand(ctx, logger, status, iuts)) + + iuts[idx].TestRunner = TestRunner{ + Steps{ + Environment{ + IutInfo: string(iutInfoJson), + }, + commands, + }, + } + + } + return iuts +} diff --git a/pkg/iut/v1alpha1/utils.go b/pkg/iut/v1alpha1/utils.go new file mode 100644 index 0000000..90a54b3 --- /dev/null +++ b/pkg/iut/v1alpha1/utils.go @@ -0,0 +1,45 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package v1alpha1 + +import ( + "fmt" + "strconv" +) + +// toInt tries to convert the input (float or a string) to an integer +func toInt(v interface{}) (int, error) { + switch v := v.(type) { + case float64: + return int(v), nil + case string: + c, err := strconv.Atoi(v) + if err != nil { + return 0, err + } + return c, nil + default: + return 0, fmt.Errorf("conversion to %T to int not supported", v) + } +} + +// fair calculates and returns a "fair" number of iuts to use given the provided list of Iuts and greed. +func fair(size int, greed float64) int { + if l := int(float64(size) * greed); l > 0 { + return l + } + return 1 +} diff --git a/pkg/iut/v1alpha1/v1alpha1.go b/pkg/iut/v1alpha1/v1alpha1.go new file mode 100644 index 0000000..83b1ceb --- /dev/null +++ b/pkg/iut/v1alpha1/v1alpha1.go @@ -0,0 +1,512 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package v1alpha1 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path" + "regexp" + "runtime" + "strings" + "sync" + "time" + + config "github.com/eiffel-community/etos-api/internal/configs/iut" + "github.com/eiffel-community/etos-api/internal/iut/checkoutable" + "github.com/eiffel-community/etos-api/internal/iut/contextmanager" + "github.com/eiffel-community/etos-api/internal/iut/responses" + "github.com/eiffel-community/etos-api/pkg/iut/application" + clientv3 "go.etcd.io/etcd/client/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "go.opentelemetry.io/otel/trace" + + "github.com/eiffel-community/eiffelevents-sdk-go" + "github.com/google/uuid" + "github.com/julienschmidt/httprouter" + "github.com/package-url/packageurl-go" + "github.com/sirupsen/logrus" +) + +var ( + service_version string + otel_sdk_version string +) + +// BASEREGEX for matching /testrun/tercc-id/provider/iuts/reference. +const BASEREGEX = "/testrun/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/provider/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/iuts" + +type V1Alpha1Application struct { + logger *logrus.Entry + cfg config.Config + database *clientv3.Client + cm *contextmanager.ContextManager + wg *sync.WaitGroup +} + +type V1Alpha1Handler struct { + logger *logrus.Entry + cfg config.Config + database *clientv3.Client + cm *contextmanager.ContextManager + wg *sync.WaitGroup +} + +type StartRequest struct { + MinimumAmount int `json:"minimum_amount"` + MaximumAmount int `json:"maximum_amount"` + ArtifactIdentity string `json:"identity"` + ArtifactID string `json:"artifact_id"` + ArtifactCreated eiffelevents.ArtifactCreatedV3 `json:"artifact_created,omitempty"` + ArtifactPublished eiffelevents.ArtifactPublishedV3 `json:"artifact_published,omitempty"` + TERCC eiffelevents.TestExecutionRecipeCollectionCreatedV4 `json:"tercc,omitempty"` + Context uuid.UUID `json:"context,omitempty"` + Dataset Dataset `json:"dataset,omitempty"` +} + +type Dataset struct { + Greed interface{} `json:"greed"` +} + +type StartResponse struct { + Id uuid.UUID `json:"id"` +} + +type statusResponse struct { + Id uuid.UUID `json:"id"` + Status string `json:"status"` + Description string `json:"description"` +} + +type StopRequest []*checkoutable.Iut + +type StatusRequest struct { + Id uuid.UUID `json:"id"` +} + +// Close does nothing atm. Present for interface coherence +func (a *V1Alpha1Application) Close() { + a.wg.Wait() +} + +// New returns a new V1Alpha1Application object/struct +func New(cfg config.Config, log *logrus.Entry, ctx context.Context, cm *contextmanager.ContextManager, cli *clientv3.Client) application.Application { + return &V1Alpha1Application{ + logger: log, + cfg: cfg, + database: cli, + cm: cm, + wg: &sync.WaitGroup{}, + } +} + +// isOtelEnabled returns true if OpenTelemetry is enabled, otherwise false +func isOtelEnabled() bool { + _, endpointSet := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + return endpointSet +} + +// initTracer initializes the OpenTelemetry instrumentation for trace collection +func (a V1Alpha1Application) initTracer() { + if !isOtelEnabled() { + a.logger.Infof("No OpenTelemetry collector is set. OpenTelemetry traces will not be available.") + return + } + collector := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + a.logger.Infof("Using OpenTelemetry collector: %s", collector) + // Create OTLP exporter to export traces + exporter, err := otlptrace.New(context.Background(), otlptracegrpc.NewClient( + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(collector), + )) + if err != nil { + log.Fatal(err) + } + + // Create a resource with service name attribute + res, err := resource.New(context.Background(), + resource.WithAttributes( + semconv.ServiceNameKey.String("iut-provider"), + semconv.ServiceNamespaceKey.String(os.Getenv("OTEL_SERVICE_NAMESPACE")), + semconv.ServiceVersionKey.String(service_version), + semconv.TelemetrySDKLanguageGo.Key.String("go"), + semconv.TelemetrySDKNameKey.String("opentelemetry"), + semconv.TelemetrySDKVersionKey.String(otel_sdk_version), + ), + ) + if err != nil { + log.Fatal(err) + } + // Create a TraceProvider with the exporter and resource + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + ) + // Set the global TracerProvider + otel.SetTracerProvider(tp) + // Set the global propagator to TraceContext (W3C Trace Context) + otel.SetTextMapPropagator(propagation.TraceContext{}) +} + +// LoadRoutes loads all the v1alpha1 routes. +func (a V1Alpha1Application) LoadRoutes(router *httprouter.Router) { + handler := &V1Alpha1Handler{a.logger, a.cfg, a.database, a.cm, a.wg} + router.GET("/v1alpha1/selftest/ping", handler.Selftest) + router.POST("/start", handler.panicRecovery(handler.timeoutHandler(handler.Start))) + router.GET("/status", handler.panicRecovery(handler.timeoutHandler(handler.Status))) + router.POST("/stop", handler.panicRecovery(handler.timeoutHandler(handler.Stop))) + + a.initTracer() +} + +// getOtelTracer returns the current OpenTelemetry tracer +func (h V1Alpha1Handler) getOtelTracer() trace.Tracer { + return otel.Tracer("iut-provider") +} + +// getOtelContext returns OpenTelemetry context from the given HTTP request object +func (h V1Alpha1Handler) getOtelContext(ctx context.Context, r *http.Request) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) +} + +// recordOtelException records an error to the given span +func (h V1Alpha1Handler) recordOtelException(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} + +// Selftest is a handler to just return 204. +func (h V1Alpha1Handler) Selftest(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + responses.RespondWithError(w, http.StatusNoContent, "") +} + +// Start handles the start request and checks out IUTs and stores the status in a database. +func (h V1Alpha1Handler) Start(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + h.wg.Add(1) + defer h.wg.Done() + ctx := context.Background() + identifier := r.Header.Get("X-Etos-Id") + logger := h.logger.WithField("identifier", identifier).WithContext(ctx) + checkOutID := uuid.New() + startReq, err := h.verifyStartInput(ctx, logger, r) + if err != nil { + logger.Error(err) + sendError(w, err) + return + } + + if startReq.MaximumAmount == 0 { + startReq.MaximumAmount = 100 + } + + ctx = h.getOtelContext(ctx, r) + + _, span := h.getOtelTracer().Start(ctx, "start", trace.WithSpanKind(trace.SpanKindServer)) + defer span.End() + + if isOtelEnabled() && os.Getenv("OTEL_JAEGER_URL") != "" { + traceID := span.SpanContext().TraceID().String() + jaegerURL, _ := url.Parse(os.Getenv("OTEL_JAEGER_URL")) + jaegerURL.Path = path.Join(jaegerURL.Path, traceID) + logger.WithField("user_log", true).Infof("Jaeger trace URL: %s", jaegerURL) + } + + time.Sleep(1 * time.Second) // Allow ETOS environment provider to send logs first. + logger.WithField("user_log", true).Infof("Check out ID: %s", checkOutID) + + if startReq.MaximumAmount == 0 { + startReq.MaximumAmount = 100 + } + + go h.DoCheckout(trace.ContextWithSpan(ctx, span), logger, startReq, checkOutID, identifier) + + span.SetAttributes(attribute.Int("etos.iut_provider.checkout.start_request.maximum_amount", startReq.MaximumAmount)) + span.SetAttributes(attribute.Int("etos.iut_provider.checkout.start_request.minimum_amount", startReq.MinimumAmount)) + span.SetAttributes(attribute.String("etos.iut_provider.checkout.start_request.artifact_id", fmt.Sprintf("%v", startReq.ArtifactID))) + span.SetAttributes(attribute.String("etos.iut_provider.checkout.start_request.artifact_identity", fmt.Sprintf("%v", startReq.ArtifactIdentity))) + span.SetAttributes(attribute.String("etos.iut_provider.checkout.checkout_id", fmt.Sprintf("%v", checkOutID))) + + responses.RespondWithJSON(w, http.StatusOK, StartResponse{Id: checkOutID}) +} + +// Status handles the status request and queries the database for the Iut checkout status +func (h V1Alpha1Handler) Status(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + h.wg.Add(1) + defer h.wg.Done() + // TimeoutHandler adds a Timeout on r.Context(). + ctx := h.getOtelContext(r.Context(), r) + identifier := r.Header.Get("X-Etos-Id") + logger := h.logger.WithField("identifier", identifier).WithContext(ctx) + + statusReq, err := h.verifyStatusInput(ctx, r) + if err != nil { + msg := fmt.Errorf("Failed to verify input for Status() request: %v - Reason: %s", r, err.Error()) + _, span := h.getOtelTracer().Start(ctx, "status", trace.WithSpanKind(trace.SpanKindServer)) + defer span.End() + h.recordOtelException(span, msg) + logger.Error(msg) + sendError(w, msg) + return + } + + key := fmt.Sprintf("/testrun/%s", identifier) + status := NewStatus(key, statusReq.Id.String(), h.database) + if err := status.Load(ctx); err != nil { + msg := fmt.Errorf("Failed to read status, request: %v - Reason: %s", statusReq, err.Error()) + _, span := h.getOtelTracer().Start(ctx, "status", trace.WithSpanKind(trace.SpanKindServer)) + defer span.End() + h.recordOtelException(span, msg) + logger.Error(msg) + failStatus := statusResponse{ + Id: statusReq.Id, + Status: "FAILED", + Description: msg.Error(), + } + responses.RespondWithJSON(w, http.StatusInternalServerError, failStatus) + return + } + + if status.Status != "PENDING" { // avoid creating too many spans while during wait loop + _, span := h.getOtelTracer().Start(ctx, "status", trace.WithSpanKind(trace.SpanKindServer)) + defer span.End() + span.SetAttributes(attribute.String("etos.iut_provider.status.iut_refs", fmt.Sprintf("%s", status.IutReferences))) + span.SetAttributes(attribute.String("etos.iut_provider.status.status", status.Status)) + span.SetAttributes(attribute.String("etos.iut_provider.status.description", status.Description)) + } + responses.RespondWithJSON(w, http.StatusOK, status) +} + +// Stop handles the stop request and checks in all the provided Iuts +func (h V1Alpha1Handler) Stop(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + h.wg.Add(1) + defer h.wg.Done() + // TimeoutHandler adds a Timeout on r.Context(). + ctx := h.getOtelContext(r.Context(), r) + identifier := r.Header.Get("X-Etos-Id") + logger := h.logger.WithField("identifier", identifier).WithContext(ctx) + + _, span := h.getOtelTracer().Start(ctx, "stop", trace.WithSpanKind(trace.SpanKindServer)) + defer span.End() + + iuts, err := h.verifyStopInput(ctx, r) + if err != nil { + msg := fmt.Errorf("Failed to verify input for Stop() request: %v - Reason: %s", r, err.Error()) + h.recordOtelException(span, msg) + sendError(w, msg) + return + } + + iut_refs := "" + refs_regex := "" + for _, iut := range iuts { + iut.AddLogger(logger) + logger.WithField("user_log", true).Infof("Checking in IUT with reference %s", iut.Reference) + if iut_refs != "" { + iut_refs += ", " + refs_regex += "|" + } + iut_refs += iut.Reference + refs_regex += iut.Reference + } + regex := regexp.MustCompile(fmt.Sprintf("%s/(%s)", BASEREGEX, refs_regex)) + + // Due to the way ETOS checks in IUTs (it does not provide the checkout ID), we need to iterate + // over all checkouts for a testrun and check them against the references that we receive from + // ETOS. + key := fmt.Sprintf("/testrun/%s/provider", identifier) + response, err := h.database.Get(ctx, key, clientv3.WithPrefix()) + if err != nil { + msg := fmt.Errorf(fmt.Sprintf("Failed to check-in iuts: %s", iut_refs)) + logger.WithError(err).Error(msg) + h.recordOtelException(span, msg) + responses.RespondWithError(w, http.StatusInternalServerError, msg.Error()) + return + } + statuses := map[string]*Status{} + for _, kv := range response.Kvs { + // Verify that 'ev.Value' is an actual IUT definition and not another + // field in the ETCD database. Since we are prefix searching on /testrun/suiteid/provider/ + // it is very possible that more data will arrive than we are interested in. + if !regex.Match(kv.Key) { + continue + } + splitKey := strings.Split(string(kv.Key), "/") + // We know for a fact that the key is /testrun/suiteid/provider/ID/iuts/reference + // so the 4th key will always be ID. + id := splitKey[4] + _, ok := statuses[id] + if !ok { + key = fmt.Sprintf("/testrun/%s", identifier) + statuses[id] = NewStatus(key, id, h.database) + } + } + + var multiErr error + for _, status := range statuses { + if err = status.Load(ctx); err != nil { + multiErr = errors.Join(multiErr, err) + continue + } + if err = h.checkIn(ctx, logger, iuts, status, false); err != nil { + multiErr = errors.Join(multiErr, err) + continue + } + status.Save(r.Context()) + } + if multiErr != nil { + msg := fmt.Errorf(fmt.Sprintf("Failed to check-in iuts: %s", iut_refs)) + logger.WithError(err).Error(msg) + h.recordOtelException(span, msg) + responses.RespondWithError(w, http.StatusInternalServerError, msg.Error()) + return + } + + iuts_json := make([]map[string]interface{}, len(iuts)) + for i, iut := range iuts { + iuts_json[i] = map[string]interface{}{ + "reference": iut.Reference, + } + } + iuts_json_str, _ := json.Marshal(iuts_json) + + span.SetAttributes(attribute.String("etos.iut_provider.stop.iuts", string(iuts_json_str))) + responses.RespondWithJSON(w, http.StatusNoContent, "") +} + +// verifyStartInput verify input (json body) from a start request +func (h V1Alpha1Handler) verifyStartInput(ctx context.Context, logger *logrus.Entry, r *http.Request) (StartRequest, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return StartRequest{}, NewHTTPError( + fmt.Errorf("failed read request body - Reason: %s", err.Error()), + http.StatusBadRequest, + ) + } + defer r.Body.Close() + + request, err := h.tryLoadStartRequest(ctx, logger, body) + if err != nil { + return request, NewHTTPError(err, http.StatusBadRequest) + } + + if request.ArtifactID == "" || request.ArtifactIdentity == "" { + return request, NewHTTPError( + errors.New("both 'artifact_identity' and 'artifact_id' are required"), + http.StatusBadRequest, + ) + } + + if request.MinimumAmount == 0 { + return request, NewHTTPError( + errors.New("minimumAmount parameter is mandatory"), + http.StatusBadRequest, + ) + } + + _, purlErr := packageurl.FromString(request.ArtifactIdentity) + if purlErr != nil { + return request, NewHTTPError(purlErr, http.StatusBadRequest) + } + + return request, ctx.Err() +} + +// verifyStatusInput verify input (url parameters) from the status request +func (h V1Alpha1Handler) verifyStatusInput(ctx context.Context, r *http.Request) (StatusRequest, error) { + id, err := uuid.Parse(r.URL.Query().Get("id")) + if err != nil { + return StatusRequest{}, NewHTTPError( + fmt.Errorf("Error parsing id parameter in status request - Reason: %s", err.Error()), + http.StatusBadRequest) + } + request := StatusRequest{Id: id} + + return request, ctx.Err() +} + +// verifyStopInput verify input (json body) from the stop request +func (h V1Alpha1Handler) verifyStopInput(ctx context.Context, r *http.Request) (StopRequest, error) { + request := StopRequest{} + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + return request, NewHTTPError(fmt.Errorf("unable to decode post body %+v", err), http.StatusBadRequest) + } + return request, ctx.Err() +} + +// sendError sends an error HTTP response depending on which error has been returned. +func sendError(w http.ResponseWriter, err error) { + httpError, ok := err.(*HTTPError) + if !ok { + responses.RespondWithError(w, http.StatusInternalServerError, fmt.Sprintf("unknown error %+v", err)) + } else { + responses.RespondWithError(w, httpError.Code, httpError.Message) + } +} + +// timeoutHandler will change the request context to a timeout context. +func (h V1Alpha1Handler) timeoutHandler( + fn func(http.ResponseWriter, *http.Request, httprouter.Params), +) func(http.ResponseWriter, *http.Request, httprouter.Params) { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + ctx, cancel := context.WithTimeout(r.Context(), h.cfg.Timeout()) + defer cancel() + newRequest := r.WithContext(ctx) + fn(w, newRequest, ps) + } +} + +// panicRecovery tracks panics from the service, logs them and returns an error response to the user. +func (h V1Alpha1Handler) panicRecovery( + fn func(http.ResponseWriter, *http.Request, httprouter.Params), +) func(http.ResponseWriter, *http.Request, httprouter.Params) { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + defer func() { + if err := recover(); err != nil { + buf := make([]byte, 2048) + n := runtime.Stack(buf, false) + buf = buf[:n] + h.logger.WithField( + "identifier", ps.ByName("identifier"), + ).WithContext( + r.Context(), + ).Errorf("recovering from err %+v\n %s", err, buf) + identifier := ps.ByName("identifier") + responses.RespondWithError( + w, + http.StatusInternalServerError, + fmt.Sprintf("unknown error: contact server admin with id '%s'", identifier), + ) + } + }() + fn(w, r, ps) + } +} diff --git a/pkg/logarea/v1alpha/logarea.go b/pkg/logarea/v1alpha/logarea.go index 63ed96b..3b016c3 100644 --- a/pkg/logarea/v1alpha/logarea.go +++ b/pkg/logarea/v1alpha/logarea.go @@ -25,7 +25,7 @@ import ( "runtime" "time" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/logarea" "github.com/eiffel-community/etos-api/pkg/application" "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" diff --git a/pkg/sse/v1/sse.go b/pkg/sse/v1/sse.go index 194fe6d..5b7ec20 100644 --- a/pkg/sse/v1/sse.go +++ b/pkg/sse/v1/sse.go @@ -25,7 +25,7 @@ import ( "strconv" "time" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/sse" "github.com/eiffel-community/etos-api/internal/kubernetes" "github.com/eiffel-community/etos-api/pkg/application" "github.com/eiffel-community/etos-api/pkg/events" diff --git a/pkg/sse/v1alpha/sse.go b/pkg/sse/v1alpha/sse.go index 3317511..aaf50b2 100644 --- a/pkg/sse/v1alpha/sse.go +++ b/pkg/sse/v1alpha/sse.go @@ -27,7 +27,7 @@ import ( "strconv" "time" - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/sse" "github.com/eiffel-community/etos-api/internal/kubernetes" "github.com/eiffel-community/etos-api/pkg/application" "github.com/eiffel-community/etos-api/pkg/events" diff --git a/test/testconfig/testconfig.go b/test/testconfig/testconfig.go index df5d9fe..d1b9d3a 100644 --- a/test/testconfig/testconfig.go +++ b/test/testconfig/testconfig.go @@ -17,7 +17,7 @@ package testconfig import ( - "github.com/eiffel-community/etos-api/internal/config" + config "github.com/eiffel-community/etos-api/internal/configs/base" ) type cfg struct {