Add Prometheus metrics for S3 and IAM requests

.github/scripts/cleanup_cosi_resources.sh

@@ -52,12 +52,12 @@ for BUCKET_NAME in $BUCKET_NAMES; do
 done
 
 log_and_run echo "Deleting Bucket Access Class..."
-log_and_run kubectl delete -f cosi-examples/bucketaccessclass.yaml --all || { echo "No BucketAccessClass resources found." | tee -a "$LOG_FILE"; }
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketaccessclass.yaml --all || { echo "No BucketAccessClass resources found." | tee -a "$LOG_FILE"; }
 
 log_and_run echo "Deleting Bucket Class and Bucket Claim..."
-log_and_run kubectl delete -f cosi-examples/bucketclass.yaml || { echo "Bucket Class not found." | tee -a "$LOG_FILE"; }
-log_and_run kubectl delete -f cosi-examples/bucketclaim.yaml || { echo "Bucket Claim not found." | tee -a "$LOG_FILE"; }
-log_and_run kubectl delete -f cosi-examples/bucketclass-delete-on-claim-removal.yaml || { echo "Bucket Class not found." | tee -a "$LOG_FILE"; }
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketclass.yaml || { echo "Bucket Class not found." | tee -a "$LOG_FILE"; }
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketclaim.yaml || { echo "Bucket Claim not found." | tee -a "$LOG_FILE"; }
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketclass-deletion-policy.yaml || { echo "Bucket Class not found." | tee -a "$LOG_FILE"; }
 
 log_and_run echo "Deleting s3-secret-for-cosi secret..."
 log_and_run kubectl delete secret s3-secret-for-cosi --namespace=default || { echo "Secret s3-secret-for-cosi not found." | tee -a "$LOG_FILE"; }

.github/scripts/e2e_tests_brownfield_use_case.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+set -e
+
+LOG_FILE=".github/e2e_tests/artifacts/logs/e2e_tests/brownfield.log"
+mkdir -p "$(dirname "$LOG_FILE")"
+
+HOST_IP=$(hostname -I | awk '{print $1}')
+SECRET_NAME="brownfield-bucket-secret"
+IAM_ENDPOINT="http://$HOST_IP:8600"
+S3_ENDPOINT="http://$HOST_IP:8000"
+BUCKET_NAME="brownfield-bucket"
+NAMESPACE="scality-object-storage"
+REGION="us-west-1"
+
+# Error handling function
+error_handler() {
+  echo "An error occurred during bucket creation tests. Check the log file for details." | tee -a "$LOG_FILE"
+  echo "Failed command: $BASH_COMMAND" | tee -a "$LOG_FILE"
+  exit 1
+}
+
+# Trap errors and call the error handler
+trap 'error_handler' ERR
+
+# Log command execution to the log file for debugging
+log_and_run() {
+  "$@" 2>&1 | tee -a "$LOG_FILE"
+}
+
+
+# Create the bucket fir brownfield scenario
+log_and_run echo "Creating bucket: $BUCKET_NAME"
+log_and_run aws s3api create-bucket --bucket "$BUCKET_NAME" --region $REGION --endpoint-url "$S3_ENDPOINT"
+
+# Check if the bucket exists
+log_and_run echo "Checking if bucket $BUCKET_NAME exists"
+aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket "$BUCKET_NAME"
+log_and_run echo "Bucket $BUCKET_NAME exists!"
+
+log_and_run echo "Applying Bucket Class to use existing bucket..."
+log_and_run kubectl apply -f cosi-examples/brownfield/bucketclass.yaml
+
+log_and_run echo "Manually creating Bucket object with existing bucket..."
+log_and_run kubectl apply -f cosi-examples/brownfield/bucket.yaml
+
+log_and_run echo "Applying Bucket Claim referencing the Bucket object..."
+log_and_run kubectl apply -f cosi-examples/brownfield/bucketclaim.yaml
+
+log_and_run echo "Applying Bucket Access Class..."
+log_and_run kubectl apply -f cosi-examples/brownfield/bucketaccessclass.yaml
+
+log_and_run echo "Applying Bucket Access..."
+log_and_run kubectl apply -f cosi-examples/brownfield/bucketaccess.yaml
+
+log_and_run echo "Verifying brownfield-bucket-secret in the default namespace..."
+SECRET_JSON="$(kubectl get secret "$SECRET_NAME" --namespace "$NAMESPACE" -o json)"
+
+# Decode the Base64 encoded BucketInfo
+BUCKET_INFO_BASE64="$(echo "$SECRET_JSON" | jq -r '.data.BucketInfo')"
+BUCKET_INFO_JSON="$(echo "$BUCKET_INFO_BASE64" | base64 --decode)"
+
+log_and_run echo "Decoded BucketInfo: $BUCKET_INFO_JSON"
+
+# Extract values to verify
+ACTUAL_BUCKET_NAME=$(echo "$BUCKET_INFO_JSON" | jq -r '.spec.bucketName')
+ACTUAL_ENDPOINT=$(echo "$BUCKET_INFO_JSON" | jq -r '.spec.secretS3.endpoint')
+ACTUAL_REGION=$(echo "$BUCKET_INFO_JSON" | jq -r '.spec.secretS3.region')
+ACTUAL_ACCESS_KEY_ID=$(echo "$BUCKET_INFO_JSON" | jq -r '.spec.secretS3.accessKeyID')
+ACTUAL_ACCESS_SECRET_KEY=$(echo "$BUCKET_INFO_JSON" | jq -r '.spec.secretS3.accessSecretKey')
+ACTUAL_PROTOCOLS=$(echo "$BUCKET_INFO_JSON" | jq -c '.spec.protocols')
+
+# Verify bucketName
+if [[ "$ACTUAL_BUCKET_NAME" != "$BUCKET_NAME" ]]; then
+  log_and_run echo "Bucket name mismatch! Expected: $BUCKET_NAME, Found: $ACTUAL_BUCKET_NAME"
+  exit 1
+fi
+
+# Verify endpoint
+EXPECTED_ENDPOINT="$S3_ENDPOINT"
+if [[ "$ACTUAL_ENDPOINT" != "$EXPECTED_ENDPOINT" ]]; then
+  log_and_run echo "Endpoint mismatch! Expected: $EXPECTED_ENDPOINT, Found: $ACTUAL_ENDPOINT"
+  exit 1
+fi
+
+# Verify region
+if [[ "$ACTUAL_REGION" != "$REGION" ]]; then
+  log_and_run echo "Region mismatch! Expected: $REGION, Found: $ACTUAL_REGION"
+  exit 1
+fi
+
+# Verify accessSecretKey exists
+if [[ -z "$ACTUAL_ACCESS_KEY_ID" ]]; then
+  log_and_run echo "AccessSecretKey is empty!"
+  exit 1
+fi
+
+# Verify accessSecretKey exists
+if [[ -z "$ACTUAL_ACCESS_SECRET_KEY" ]]; then
+  log_and_run echo "AccessSecretKey is empty!"
+  exit 1
+fi
+
+# Verify protocol
+EXPECTED_PROTOCOLS='["s3"]'
+if [[ "$ACTUAL_PROTOCOLS" != "$EXPECTED_PROTOCOLS" ]]; then
+  log_and_run echo "Protocols mismatch! Expected: $EXPECTED_PROTOCOLS, Found: $ACTUAL_PROTOCOLS"
+  exit 1
+fi
+
+# cleanup
+log_and_run kubectl delete -f cosi-examples/brownfield/bucketaccess.yaml
+log_and_run kubectl delete -f cosi-examples/brownfield/bucketaccessclass.yaml
+log_and_run kubectl delete -f cosi-examples/brownfield/bucketclaim.yaml
+log_and_run kubectl delete -f cosi-examples/brownfield/bucketclass.yaml
+

.github/scripts/e2e_tests.sh → .github/scripts/e2e_tests_greenfield_use_case.sh

@@ -2,7 +2,7 @@
 set -e
 
 # Define log file for debugging
-LOG_FILE=".github/e2e_tests/artifacts/logs/e2e_tests/bucket_creation_test.log"
+LOG_FILE=".github/e2e_tests/artifacts/logs/e2e_tests/greenfield.log"
 mkdir -p "$(dirname "$LOG_FILE")"  # Ensure the log directory exists
 
 CONTAINER_NAME=s3_and_iam_deployment-iam-1
@@ -83,19 +83,19 @@ EOF
 
 # Step 4: Apply Bucket Class
 log_and_run echo "Applying Bucket Class..."
-log_and_run kubectl apply -f cosi-examples/bucketclass.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketclass.yaml
 
 # Step 5: Apply Bucket Claim
 log_and_run echo "Applying Bucket Claim..."
-log_and_run kubectl apply -f cosi-examples/bucketclaim.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketclaim.yaml
 
 # Step 6: Apply Bucket Access Class
 log_and_run echo "Applying Bucket Access Class..."
-log_and_run kubectl apply -f cosi-examples/bucketaccessclass.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketaccessclass.yaml
 
 # Step 7: Apply Bucket Access
 log_and_run echo "Applying Bucket Access..."
-log_and_run kubectl apply -f cosi-examples/bucketaccess.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketaccess.yaml
 
 # Step 8: Verify Bucket Creation with Retry
 log_and_run echo "Listing all S3 buckets before verification..."
@@ -213,7 +213,7 @@ fi
 
 # Step 11: Delete Bucket Access Resource
 log_and_run echo "Deleting Bucket Access Resource..."
-log_and_run kubectl delete -f cosi-examples/bucketaccess.yaml
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketaccess.yaml
 
 # Step 12: Verify IAM User Deletion
 log_and_run echo "Verifying IAM user '$IAM_USER_NAME' deletion..."
@@ -230,8 +230,8 @@ fi
 # Step 13: Test deletion bucket with deletion policy set
 
 log_and_run echo "Applying Bucket Class with deletion policy and respective Bucket Claim..."
-log_and_run kubectl apply -f cosi-examples/bucketclass-deletion-policy.yaml
-log_and_run kubectl apply -f cosi-examples/bucketclaim-deletion-policy.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketclass-deletion-policy.yaml
+log_and_run kubectl apply -f cosi-examples/greenfield/bucketclaim-deletion-policy.yaml
 
 log_and_run echo "Listing all S3 buckets before deletion..."
 log_and_run aws s3 ls --endpoint-url "$S3_ENDPOINT"
@@ -259,7 +259,7 @@ if [ -z "$BUCKET_TO_BE_DELETED" ]; then
 fi
 
 log_and_run echo "Deleting Bucket Claim..."
-log_and_run kubectl delete -f cosi-examples/bucketclaim-deletion-policy.yaml
+log_and_run kubectl delete -f cosi-examples/greenfield/bucketclaim-deletion-policy.yaml
 
 # Check if the bucket with name $BUCKET_TO_BE_DELETED exists by doing a head bucket.
 # If bucket exists, retry with ATTEMPTS and DELAY. If bucket is not found, test success.

.github/workflows/ci-e2e-tests.yml

@@ -96,10 +96,13 @@ jobs:
         docker save "$CLOUDSERVER_IMAGE" -o /tmp/.docker_cache/cloudserver_image.tar
       shell: bash
 
-    - name: E2E tests for COSI driver using kustomize
+    - name: E2E tests for greenfield use case using kustomize
       run: |
-        pwd
-        .github/scripts/e2e_tests.sh
+        .github/scripts/e2e_tests_greenfield_use_case.sh
+
+    - name: E2E tests for brownfield use case using kustomize
+      run: |
+        .github/scripts/e2e_tests_brownfield_use_case.sh
 
     - name: "Delay completion"
       if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

cmd/scality-cosi-driver/cmd.go

@@ -20,11 +20,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"time"
 
 	"github.com/scality/cosi-driver/pkg/driver"
 	"k8s.io/klog/v2"
 
+	c "github.com/scality/cosi-driver/pkg/constants"
 	"github.com/scality/cosi-driver/pkg/grpcfactory"
+	"github.com/scality/cosi-driver/pkg/metrics"
 )
 
 const (
@@ -33,8 +36,9 @@ const (
 )
 
 var (
-	driverAddress = flag.String("driver-address", "unix:///var/lib/cosi/cosi.sock", "driver address for the socket")
-	driverPrefix  = flag.String("driver-prefix", "", "prefix for COSI driver, e.g. <prefix>.scality.com")
+	driverAddress  = flag.String("driver-address", "unix:///var/lib/cosi/cosi.sock", "driver address for the socket")
+	driverPrefix   = flag.String("driver-prefix", "", "prefix for COSI driver, e.g. <prefix>.scality.com")
+	metricsAddress = flag.String("metrics-address", c.MetricsAddress, "The address to expose Prometheus metrics.")
 )
 
 func init() {
@@ -53,6 +57,11 @@ func init() {
 }
 
 func run(ctx context.Context) error {
+	metricsServer, err := metrics.StartMetricsServer(*metricsAddress)
+	if err != nil {
+		return fmt.Errorf("failed to start metrics server: %w", err)
+	}
+
 	driverName := *driverPrefix + "." + provisionerName
 
 	identityServer, bucketProvisioner, err := driver.CreateDriver(ctx, driverName)
@@ -65,5 +74,12 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("failed to start the provisioner server: %w", err)
 	}
 
-	return server.Run(ctx)
+	err = server.Run(ctx)
+	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer shutdownCancel()
+	if shutdownErr := metricsServer.Shutdown(shutdownCtx); shutdownErr != nil {
+		klog.ErrorS(shutdownErr, "Failed to gracefully shutdown metrics server")
+	}
+
+	return err
 }

cmd/scality-cosi-driver/main.go

@@ -50,5 +50,6 @@ func main() {
 	// Call the run function (defined in cmd.go)
 	if err := run(ctx); err != nil {
 		klog.ErrorS(err, "Scality COSI driver encountered an error, shutting down")
+		os.Exit(1)
 	}
 }

codecov.yml

@@ -48,6 +48,10 @@ component_management:
       name: 🔧 Util Package
       paths:
         - pkg/util/**
+    - component_id: metrics-package
+      name: 📊 Metrics Package
+      paths:
+        - pkg/metrics/**
     - component_id: constants-package
       name: 🔖 Constants Package
       paths:

docs/driver-params.md

@@ -9,7 +9,7 @@ The table below details the configuration parameters for BucketClass, which dete
 | `objectStorageSecretName`         | The name of the Kubernetes secret containing S3 credentials and configuration. | `string`                   | Yes          |
 | `objectStorageSecretNamespace`    | The namespace in which the secret is located (e.g., `default`).                | `string` (e.g., `default`) | Yes          |
 
-[Example](../cosi-examples/bucketclass.yaml)
+[Example](../cosi-examples/greenfield/bucketclass.yaml)
 
 ## Configuration Parameters for Kubernetes secret containing S3 credentials and configuration