Merge pull request #360 from neicnordic/feat/files-api

Add files api

.github/integration/sda-s3-integration.yml

@@ -276,6 +276,33 @@ services:
       - ./sda/oidc.py:/oidc.py
       - shared:/shared
 
+  api:
+    command: [ sda-api ]
+    container_name: api
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      oidc:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=ingest
+      - BROKER_USER=ingest
+      - BROKER_ROUTINGKEY=ingest
+      - BROKER_ROUTINGERROR=error
+      - DB_PASSWORD=download
+      - DB_USER=download
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    ports:
+      - "8090:8080"
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   integration_test:
     container_name: tester
     command:
@@ -298,7 +325,9 @@ services:
       sync-api:
         condition: service_started
       verify:
-        condition: service_started
+         condition: service_started
+      api:
+         condition: service_started
     environment:
       - PGPASSWORD=rootpasswd
       - STORAGETYPE=s3
@@ -313,4 +342,4 @@ volumes:
   minio_data:
   postgres_data:
   rabbitmq_data:
-  shared:
+  shared:

.github/integration/tests/sda/11_api-getfiles_test.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+# Test the API files endpoint
+token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
+curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token"
+response="$(curl -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
+if [ "$response" != "uploaded" ]; then
+	echo "API returned incorrect value, expected ready got: $response"
+	exit 1
+fi

sda/cmd/api/api.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/lestrrat-go/jwx/v2/jwk"
+	"github.com/neicnordic/sensitive-data-archive/internal/broker"
+	"github.com/neicnordic/sensitive-data-archive/internal/config"
+	"github.com/neicnordic/sensitive-data-archive/internal/database"
+	"github.com/neicnordic/sensitive-data-archive/internal/userauth"
+	log "github.com/sirupsen/logrus"
+)
+
+var Conf *config.Config
+var err error
+var auth *userauth.ValidateFromToken
+
+func main() {
+	Conf, err = config.NewConfig("api")
+	if err != nil {
+		log.Fatal(err)
+	}
+	Conf.API.MQ, err = broker.NewMQ(Conf.Broker)
+	if err != nil {
+		log.Fatal(err)
+	}
+	Conf.API.DB, err = database.NewSDAdb(Conf.Database)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err := setupJwtAuth(); err != nil {
+		log.Fatalf("error when setting up JWT auth, reason %s", err.Error())
+
+	}
+	sigc := make(chan os.Signal, 5)
+	signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	go func() {
+		<-sigc
+		shutdown()
+		os.Exit(0)
+	}()
+
+	srv := setup(Conf)
+	if Conf.API.ServerCert != "" && Conf.API.ServerKey != "" {
+		log.Infof("Starting web server at https://%s:%d", Conf.API.Host, Conf.API.Port)
+		if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil {
+			shutdown()
+			log.Fatalln(err)
+		}
+	} else {
+		log.Infof("Starting web server at http://%s:%d", Conf.API.Host, Conf.API.Port)
+		if err := srv.ListenAndServe(); err != nil {
+			shutdown()
+			log.Fatalln(err)
+		}
+	}
+
+}
+
+func setup(config *config.Config) *http.Server {
+	r := gin.Default()
+	r.GET("/ready", readinessResponse)
+	r.GET("/files", getFiles)
+
+	cfg := &tls.Config{
+		MinVersion:               tls.VersionTLS12,
+		CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
+		PreferServerCipherSuites: true,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		},
+	}
+
+	srv := &http.Server{
+		Addr:              config.API.Host + ":" + fmt.Sprint(config.API.Port),
+		Handler:           r,
+		TLSConfig:         cfg,
+		TLSNextProto:      make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+		ReadHeaderTimeout: 20 * time.Second,
+		ReadTimeout:       5 * time.Minute,
+		WriteTimeout:      20 * time.Second,
+	}
+
+	return srv
+}
+
+func setupJwtAuth() error {
+	auth = userauth.NewValidateFromToken(jwk.NewSet())
+	if Conf.Server.Jwtpubkeyurl != "" {
+		if err := auth.FetchJwtPubKeyURL(Conf.Server.Jwtpubkeyurl); err != nil {
+			return err
+		}
+	}
+	if Conf.Server.Jwtpubkeypath != "" {
+		if err := auth.ReadJwtPubKeyPath(Conf.Server.Jwtpubkeypath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func shutdown() {
+	defer Conf.API.MQ.Channel.Close()
+	defer Conf.API.MQ.Connection.Close()
+	defer Conf.API.DB.Close()
+}
+
+func readinessResponse(c *gin.Context) {
+	statusCode := http.StatusOK
+
+	if Conf.API.MQ.Connection.IsClosed() {
+		statusCode = http.StatusServiceUnavailable
+		newConn, err := broker.NewMQ(Conf.Broker)
+		if err != nil {
+			log.Errorf("failed to reconnect to MQ, reason: %v", err)
+		} else {
+			Conf.API.MQ = newConn
+		}
+	}
+
+	if Conf.API.MQ.Channel.IsClosed() {
+		statusCode = http.StatusServiceUnavailable
+		Conf.API.MQ.Connection.Close()
+		newConn, err := broker.NewMQ(Conf.Broker)
+		if err != nil {
+			log.Errorf("failed to reconnect to MQ, reason: %v", err)
+		} else {
+			Conf.API.MQ = newConn
+		}
+	}
+
+	if DBRes := checkDB(Conf.API.DB, 5*time.Millisecond); DBRes != nil {
+		log.Debugf("DB connection error :%v", DBRes)
+		Conf.API.DB.Reconnect()
+		statusCode = http.StatusServiceUnavailable
+	}
+
+	c.JSON(statusCode, "")
+}
+
+func checkDB(database *database.SDAdb, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	if database.DB == nil {
+		return fmt.Errorf("database is nil")
+	}
+
+	return database.DB.PingContext(ctx)
+}
+
+// getFiles returns the files from the database for a specific user
+func getFiles(c *gin.Context) {
+	c.Writer.Header().Set("Content-Type", "application/json")
+	// Get user ID to extract all files
+	token, err := auth.Authenticate(c.Request)
+	if err != nil {
+		// something went wrong with user token
+		c.JSON(401, err.Error())
+
+		return
+	}
+
+	files, err := Conf.API.DB.GetUserFiles(token.Subject())
+	if err != nil {
+		// something went wrong with querying or parsing rows
+		c.JSON(502, err.Error())
+
+		return
+	}
+
+	// Return response
+	c.JSON(200, files)
+}

sda/cmd/api/api.md

@@ -0,0 +1,19 @@
+# API
+The API service provides data submitters with functionality to control
+their submissions. Users are authenticated with a JWT. 
+
+## Service Description
+
+Endpoints:
+  - `/files`
+
+    1. Parses and validates the JWT token against the public keys, either locally provisioned or from OIDC JWK endpoints.
+    2. The `sub` field from the token is extracted and used as the user's identifier
+    3. All files belonging to this user are extracted from the database, together with their latest status and creation date
+
+     Example:
+      ```bash
+      $ curl 'https://server/files' -H "Authorization: Bearer $token"
+    [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] 
+    ```
+     If the `token` is invalid, 401 is returned.