Support Google Cloud and other authentication strategies (#9)

Makefile

@@ -50,4 +50,7 @@ generate-api-types:
 
 .PHONY: generate-test-config
 generate-test-config:
-	CONNECTION_URL=http://localhost:9090 go run ./configuration update -d ./tests/configuration --log-level debug
+	CONNECTION_URL=http://localhost:9090 \
+	PROMETHEUS_USERNAME=admin \
+	PROMETHEUS_PASSWORD=test \
+		go run ./configuration update -d ./tests/configuration --log-level debug

README.md

@@ -67,6 +67,8 @@ The connector can detect if you want to request an instant query or range query
 
 The range query mode is default If none of the timestamp operators is set.
 
+The `timestamp` and `value` fields are the result of the instant query. If the request is a range query, `timestamp` and `value` are picked the last item of the `values` series.
+
 #### Common arguments
 
 - `step`: the query resolution step width in duration format or float number of seconds. The step should be explicitly set for range queries. Even though the connector can estimate the approximate step width the result may be empty due to too far interval.
@@ -188,6 +190,62 @@ Execute a raw PromQL query directly. This API should be used by the admin only.
 | prometheus_targets          | [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets)                                 |
 | prometheus_targets_metadata | [/api/v1/targets/metadata](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-target-metadata)       |
 
+## Configuration
+
+### Authentication
+
+#### Basic Authentication
+
+```yaml
+connection_settings:
+  authentication:
+    basic:
+      username:
+        env: PROMETHEUS_USERNAME
+      password:
+        env: PROMETHEUS_PASSWORD
+```
+
+#### HTTP Authorization
+
+```yaml
+connection_settings:
+  authentication:
+    authorization:
+      type:
+        value: Bearer
+      credentials:
+        env: PROMETHEUS_AUTH_TOKEN
+```
+
+#### OAuth2
+
+```yaml
+connection_settings:
+  authentication:
+    oauth2:
+      token_url:
+        value: http://example.com/oauth2/token
+      client_id:
+        env: PROMETHEUS_OAUTH2_CLIENT_ID
+      client_secret:
+        env: PROMETHEUS_OAUTH2_CLIENT_SECRET
+```
+
+#### Google Cloud
+
+The configuration accepts either the Google application credentials JSON string or file path. If the object is empty the client automatically loads the credential file from the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.
+
+```yaml
+connection_settings:
+  authentication:
+    google:
+      # credentials:
+      #   env: GOOGLE_APPLICATION_CREDENTIALS_JSON
+      # credentials_file:
+      #   env: GOOGLE_APPLICATION_CREDENTIALS
+```
+
 ## Development
 
 ### Get started

compose.yaml

@@ -12,6 +12,8 @@ services:
       - local.hasura.dev=host-gateway
     environment:
       CONNECTION_URL: http://prometheus:9090
+      PROMETHEUS_USERNAME: admin
+      PROMETHEUS_PASSWORD: test
       OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://jaeger:4317
       OTEL_METRICS_EXPORTER: prometheus
       HASURA_LOG_LEVEL: debug
@@ -27,6 +29,7 @@ services:
       - "--web.console.libraries=/usr/share/prometheus/console_libraries"
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--web.enable-lifecycle"
+      - "--web.config.file=/etc/prometheus/web.yml"
       - "--enable-feature=promql-experimental-functions"
     ports:
       - 9090:9090

configuration/cli.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"log"
 	"log/slog"
 	"os"
 	"os/signal"
@@ -31,19 +30,22 @@ func main() {
 	defer stop()
 
 	cmd := kong.Parse(&cli, kong.UsageOnError())
-	_, err := initLogger(cli.LogLevel)
+	logger, err := initLogger(cli.LogLevel)
 	if err != nil {
-		log.Fatalf("failed to initialize: %s", err)
+		logger.Error(fmt.Sprintf("failed to initialize: %s", err))
+		os.Exit(1)
 	}
 	switch cmd.Command() {
 	case "update":
 		if err := introspectSchema(ctx, &cli.Update); err != nil {
-			log.Fatalf("failed to update configuration: %s", err)
+			logger.Error(fmt.Sprintf("failed to update configuration: %s", err))
+			os.Exit(1)
 		}
 	case "version":
 		_, _ = fmt.Print(version.BuildVersion)
 	default:
-		log.Fatalf("unknown command <%s>", cmd.Command())
+		logger.Error(fmt.Sprintf("unknown command <%s>", cmd.Command()))
+		os.Exit(1)
 	}
 }
 

configuration/update.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/hasura/ndc-prometheus/connector/client"
 	"github.com/hasura/ndc-prometheus/connector/metadata"
+	"github.com/hasura/ndc-prometheus/connector/types"
 	"github.com/prometheus/common/model"
 	"go.opentelemetry.io/otel"
 	"gopkg.in/yaml.v3"
@@ -41,12 +42,7 @@ func introspectSchema(ctx context.Context, args *UpdateArguments) error {
 		originalConfig = &defaultConfiguration
 	}
 
-	endpoint, err := originalConfig.ConnectionSettings.URL.Get()
-	if err != nil {
-		return err
-	}
-
-	apiClient, err := client.NewClient(endpoint, originalConfig.ConnectionSettings.ToHTTPClientConfig(), clientTracer, nil)
+	apiClient, err := client.NewClient(ctx, originalConfig.ConnectionSettings, clientTracer, nil)
 	if err != nil {
 		return err
 	}
@@ -76,19 +72,22 @@ func introspectSchema(ctx context.Context, args *UpdateArguments) error {
 }
 
 func (uc *updateCommand) updateMetricsMetadata(ctx context.Context) error {
-	metricsInfo, err := uc.Client.API.Metadata(ctx, "", "")
+	metricsInfo, err := uc.Client.Metadata(ctx, "", "10000000")
 	if err != nil {
 		return err
 	}
 
 	newMetrics := map[string]metadata.MetricInfo{}
 	for key, info := range metricsInfo {
+		if len(info) == 0 {
+			continue
+		}
 		if (len(uc.Include) > 0 && !validateRegularExpressions(uc.Include, key)) ||
 			validateRegularExpressions(uc.Exclude, key) ||
 			len(info) == 0 {
 			continue
 		}
-		slog.Debug(key, slog.String("type", "metrics"))
+		slog.Info(key, slog.String("type", string(info[0].Type)))
 		labels, err := uc.getAllLabelsOfMetric(ctx, key)
 		if err != nil {
 			return fmt.Errorf("error when fetching labels for metric `%s`: %s", key, err)
@@ -170,8 +169,8 @@ func (uc *updateCommand) writeConfigFile() error {
 }
 
 var defaultConfiguration = metadata.Configuration{
-	ConnectionSettings: metadata.ClientSettings{
-		URL: metadata.NewEnvironmentVariable("CONNECTION_URL"),
+	ConnectionSettings: client.ClientSettings{
+		URL: types.NewEnvironmentVariable("CONNECTION_URL"),
 	},
 	Generator: metadata.GeneratorSettings{
 		Metrics: metadata.MetricsGeneratorSettings{

connector-definition/configuration.yaml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/hasura/ndc-prometheus/main/jsonschema/configuration.json
 connection_settings:
   url:
-    variable: CONNECTION_URL
+    env: CONNECTION_URL
 generator:
   metrics:
     enabled: true

connector/client/client.go

@@ -2,13 +2,16 @@ package client
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"log/slog"
 	"net/http"
+	"strings"
 	"time"
 
+	"github.com/hasura/ndc-sdk-go/utils"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
-	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/codes"
@@ -26,8 +29,17 @@ type Client struct {
 }
 
 // NewClient creates a new Prometheus client instance
-func NewClient(endpoint string, clientConfig config.HTTPClientConfig, tracer trace.Tracer, timeout *model.Duration) (*Client, error) {
-	httpClient, err := config.NewClientFromConfig(clientConfig, "ndc-prometheus")
+func NewClient(ctx context.Context, cfg ClientSettings, tracer trace.Tracer, timeout *model.Duration) (*Client, error) {
+
+	endpoint, err := cfg.URL.Get()
+	if err != nil {
+		return nil, fmt.Errorf("url: %s", err)
+	}
+	if endpoint == "" {
+		return nil, errors.New("the endpoint setting is empty")
+	}
+
+	httpClient, err := cfg.createHttpClient(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -87,5 +99,19 @@ func createHTTPClient(c api.Client) *httpClient {
 // Do wraps the api.Client with trace context headers injection
 func (ac *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, error) {
 	ac.propagator.Inject(ctx, propagation.HeaderCarrier(req.Header))
-	return ac.Client.Do(ctx, req)
+	r, bs, err := ac.Client.Do(ctx, req)
+	if utils.IsDebug(slog.Default()) {
+		attrs := []any{}
+		if r != nil {
+			attrs = append(attrs, slog.Int("status_code", r.StatusCode))
+		}
+		if len(bs) > 0 {
+			attrs = append(attrs, slog.String("response", string(bs)))
+		}
+		if err != nil {
+			attrs = append(attrs, slog.String("error", err.Error()))
+		}
+		slog.Debug(fmt.Sprintf("%s %s", strings.ToUpper(req.Method), req.RequestURI), attrs...)
+	}
+	return r, bs, err
 }