feat: TKC-1290: add support for custom CA certificates (#5098) (#5135)

* TKC-1290: add support for custom CA certificates

* TKC-1290: fix failing unit tests for custom CA certificates

cmd/api-server/main.go

@@ -189,7 +189,16 @@ func main() {
 		mode = common.ModeAgent
 	}
 	if mode == common.ModeAgent {
-		grpcConn, err = agent.NewGRPCConnection(ctx, cfg.TestkubeProTLSInsecure, cfg.TestkubeProSkipVerify, cfg.TestkubeProURL, log.DefaultLogger)
+		grpcConn, err = agent.NewGRPCConnection(
+			ctx,
+			cfg.TestkubeProTLSInsecure,
+			cfg.TestkubeProSkipVerify,
+			cfg.TestkubeProURL,
+			cfg.TestkubeProCertFile,
+			cfg.TestkubeProKeyFile,
+			cfg.TestkubeProCAFile,
+			log.DefaultLogger,
+		)
 		ui.ExitOnError("error creating gRPC connection", err)
 		defer grpcConn.Close()
 
@@ -514,6 +523,7 @@ func main() {
 		features,
 		logsStream,
 		cfg.TestkubeNamespace,
+		cfg.TestkubeProTLSSecret,
 	)
 
 	slackLoader, err := newSlackLoader(cfg, envs)

cmd/logs/main.go

@@ -103,7 +103,16 @@ func main() {
 	switch mode {
 
 	case common.ModeAgent:
-		grpcConn, err := agent.NewGRPCConnection(ctx, cfg.TestkubeProTLSInsecure, cfg.TestkubeProSkipVerify, cfg.TestkubeProURL+cfg.TestkubeProLogsPath, log)
+		grpcConn, err := agent.NewGRPCConnection(
+			ctx,
+			cfg.TestkubeProTLSInsecure,
+			cfg.TestkubeProSkipVerify,
+			cfg.TestkubeProURL+cfg.TestkubeProLogsPath,
+			cfg.TestkubeProCertFile,
+			cfg.TestkubeProKeyFile,
+			cfg.TestkubeProCAFile,
+			log,
+		)
 		ui.ExitOnError("error creating gRPC connection for logs service", err)
 		defer grpcConn.Close()
 		grpcClient := pb.NewCloudLogsServiceClient(grpcConn)

config/job-container-template.yml

@@ -77,6 +77,11 @@ spec:
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .AgentAPITLSSecret }}
+        - mountPath: /tmp/agent-cert
+          readOnly: true
+          name: {{ .AgentAPITLSSecret }}
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
         - name: artifact-volume
@@ -103,6 +108,11 @@ spec:
         secret:
           secretName: {{ .CertificateSecret }}
       {{- end }}
+      {{- if .AgentAPITLSSecret }}
+      - name: { { .AgentAPITLSSecret } }
+        secret:
+          secretName: {{ .AgentAPITLSSecret }}
+      {{- end }}
       {{- if .ArtifactRequest }}
         {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
       - name: artifact-volume

config/job-template.yml

@@ -17,7 +17,7 @@ spec:
         image: {{ .InitImage }}
         {{- end }}
         imagePullPolicy: IfNotPresent
-        command: 
+        command:
           - "/bin/runner"
           - '{{ .Jsn }}'
         volumeMounts:
@@ -27,6 +27,11 @@ spec:
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .AgentAPITLSSecret }}
+        - mountPath: /tmp/agent-cert
+          readOnly: true
+          name: {{ .AgentAPITLSSecret }}
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
         - name: artifact-volume
@@ -46,7 +51,7 @@ spec:
         {{- end }}
         {{- end }}
       containers:
-      {{ if .Features.LogsV2 -}} 
+      {{ if .Features.LogsV2 -}}
       - name: "{{ .Name }}-logs"
         image: {{ .Registry }}/{{ .LogSidecarImage }}
         env:
@@ -66,7 +71,7 @@ spec:
         image: {{ .Image }}
         {{- end }}
         imagePullPolicy: IfNotPresent
-        command: 
+        command:
           - "/bin/runner"
           - '{{ .Jsn }}'
         volumeMounts:
@@ -76,6 +81,11 @@ spec:
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .AgentAPITLSSecret }}
+        - mountPath: /tmp/agent-cert
+          readOnly: true
+          name: {{ .AgentAPITLSSecret }}
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
         - name: artifact-volume
@@ -102,6 +112,11 @@ spec:
         secret:
           secretName: {{ .CertificateSecret }}
       {{- end }}
+      {{- if .AgentAPITLSSecret }}
+      - name: {{ .AgentAPITLSSecret }}
+        secret:
+          secretName: {{ .AgentAPITLSSecret }}
+      {{- end }}
       {{- if .ArtifactRequest }}
         {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
       - name: artifact-volume

internal/config/config.go

@@ -73,6 +73,10 @@ type Config struct {
 	TestkubeProOrgID                 string        `envconfig:"TESTKUBE_PRO_ORG_ID" default:""`
 	TestkubeProMigrate               string        `envconfig:"TESTKUBE_PRO_MIGRATE" default:"false"`
 	TestkubeProConnectionTimeout     int           `envconfig:"TESTKUBE_PRO_CONNECTION_TIMEOUT" default:"10"`
+	TestkubeProCertFile              string        `envconfig:"TESTKUBE_PRO_CERT_FILE" default:""`
+	TestkubeProKeyFile               string        `envconfig:"TESTKUBE_PRO_KEY_FILE" default:""`
+	TestkubeProCAFile                string        `envconfig:"TESTKUBE_PRO_CA_FILE" default:""`
+	TestkubeProTLSSecret             string        `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""`
 	TestkubeWatcherNamespaces        string        `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""`
 	GraphqlPort                      string        `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
 	TestkubeRegistry                 string        `envconfig:"TESTKUBE_REGISTRY" default:""`

pkg/agent/agent.go

@@ -3,8 +3,10 @@ package agent
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"math"
+	"os"
 	"time"
 
 	"google.golang.org/grpc/keepalive"
@@ -42,13 +44,32 @@ const (
 // buffer up to five messages per worker
 const bufferSizePerWorker = 5
 
-func NewGRPCConnection(ctx context.Context, isInsecure bool, skipVerify bool, server string, logger *zap.SugaredLogger) (*grpc.ClientConn, error) {
+func NewGRPCConnection(
+	ctx context.Context,
+	isInsecure bool,
+	skipVerify bool,
+	server string,
+	certFile, keyFile, caFile string,
+	logger *zap.SugaredLogger,
+) (*grpc.ClientConn, error) {
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	var tlsConfig *tls.Config
+	tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
 	if skipVerify {
 		tlsConfig = &tls.Config{InsecureSkipVerify: true}
+	} else {
+		if certFile != "" && keyFile != "" {
+			if err := clientCert(tlsConfig, certFile, keyFile); err != nil {
+				return nil, err
+			}
+		}
+		if caFile != "" {
+			if err := rootCAs(tlsConfig, caFile); err != nil {
+				return nil, err
+			}
+		}
 	}
+
 	creds := credentials.NewTLS(tlsConfig)
 	if isInsecure {
 		creds = insecure.NewCredentials()
@@ -76,6 +97,35 @@ func NewGRPCConnection(ctx context.Context, isInsecure bool, skipVerify bool, se
 	)
 }
 
+func rootCAs(tlsConfig *tls.Config, file ...string) error {
+	pool := x509.NewCertPool()
+	for _, f := range file {
+		rootPEM, err := os.ReadFile(f)
+		if err != nil || rootPEM == nil {
+			return fmt.Errorf("agent: error loading or parsing rootCA file: %v", err)
+		}
+		ok := pool.AppendCertsFromPEM(rootPEM)
+		if !ok {
+			return fmt.Errorf("agent: failed to parse root certificate from %q", f)
+		}
+	}
+	tlsConfig.RootCAs = pool
+	return nil
+}
+
+func clientCert(tlsConfig *tls.Config, certFile, keyFile string) error {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return fmt.Errorf("agent: error loading client certificate: %v", err)
+	}
+	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
+	if err != nil {
+		return fmt.Errorf("agent: error parsing client certificate: %v", err)
+	}
+	tlsConfig.Certificates = []tls.Certificate{cert}
+	return nil
+}
+
 type Agent struct {
 	client  cloud.TestKubeCloudAPIClient
 	handler fasthttp.RequestHandler

pkg/agent/agent_test.go

@@ -49,7 +49,7 @@ func TestCommandExecution(t *testing.T) {
 		atomic.AddInt32(&msgCnt, 1)
 	}
 
-	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger)
+	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger)
 	ui.ExitOnError("error creating gRPC connection", err)
 	defer grpcConn.Close()
 

pkg/agent/events_test.go

@@ -47,7 +47,7 @@ func TestEventLoop(t *testing.T) {
 
 	logger, _ := zap.NewDevelopment()
 
-	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger)
+	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger)
 	ui.ExitOnError("error creating gRPC connection", err)
 	defer grpcConn.Close()
 

pkg/agent/logs_test.go

@@ -45,7 +45,7 @@ func TestLogStream(t *testing.T) {
 		fmt.Fprintf(ctx, "Hi there! RequestURI is %q", ctx.RequestURI())
 	}
 
-	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger)
+	grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger)
 	ui.ExitOnError("error creating gRPC connection", err)
 	defer grpcConn.Close()
 

pkg/envs/variables.go

@@ -48,6 +48,9 @@ type Params struct {
 	ProAPIURL                 string `envconfig:"RUNNER_PRO_API_URL"`                           // RUNNER_PRO_API_URL
 	ProConnectionTimeoutSec   int    `envconfig:"RUNNER_PRO_CONNECTION_TIMEOUT" default:"10"`   // RUNNER_PRO_CONNECTION_TIMEOUT
 	ProAPISkipVerify          bool   `envconfig:"RUNNER_PRO_API_SKIP_VERIFY" default:"false"`   // RUNNER_PRO_API_SKIP_VERIFY
+	ProAPICertFile            string `envconfig:"RUNNER_PRO_API_CERT_FILE"`                     // RUNNER_PRO_API_CERT_FILE
+	ProAPIKeyFile             string `envconfig:"RUNNER_PRO_API_KEY_FILE"`                      // RUNNER_PRO_API_KEY_FILE
+	ProAPICAFile              string `envconfig:"RUNNER_PRO_API_CA_FILE"`                       // RUNNER_PRO_API_CA_FILE
 	SlavesConfigs             string `envconfig:"RUNNER_SLAVES_CONFIGS"`                        // RUNNER_SLAVES_CONFIGS
 }
 

pkg/executor/client/common.go

@@ -23,19 +23,20 @@ const (
 )
 
 type ExecuteOptions struct {
-	ID                   string
-	TestName             string
-	Namespace            string
-	TestSpec             testsv3.TestSpec
-	ExecutorName         string
-	ExecutorSpec         executorv1.ExecutorSpec
-	Request              testkube.ExecutionRequest
-	Sync                 bool
-	Labels               map[string]string
-	UsernameSecret       *testkube.SecretRef
-	TokenSecret          *testkube.SecretRef
+	ID                string
+	TestName          string
+	Namespace         string
+	TestSpec          testsv3.TestSpec
+	ExecutorName      string
+	ExecutorSpec      executorv1.ExecutorSpec
+	Request           testkube.ExecutionRequest
+	Sync              bool
+	Labels            map[string]string
+	UsernameSecret    *testkube.SecretRef
+	TokenSecret       *testkube.SecretRef
+	CertificateSecret string
+	// AgentAPITLSSecret is a secret name that contains TLS certificate for Agent (gRPC) API
 	AgentAPITLSSecret    string
-	CertificateSecret    string
 	ImagePullSecretNames []string
 	Features             featureflags.FeatureFlags
 }

pkg/executor/client/job.go

@@ -172,6 +172,7 @@ type JobOptions struct {
 	UsernameSecret        *testkube.SecretRef
 	TokenSecret           *testkube.SecretRef
 	CertificateSecret     string
+	AgentAPITLSSecret     string
 	Variables             map[string]testkube.Variable
 	ActiveDeadlineSeconds int64
 	ServiceAccountName    string
@@ -1021,6 +1022,9 @@ func NewJobOptions(log *zap.SugaredLogger, templatesClient templatesv1.Interface
 		}
 	}
 
+	// used for adding custom certificates for Agent (gRPC) API
+	jobOptions.AgentAPITLSSecret = options.AgentAPITLSSecret
+
 	return
 }
 

pkg/executor/common.go

@@ -127,6 +127,18 @@ var RunnerEnvVars = []corev1.EnvVar{
 		Name:  "RUNNER_PRO_CONNECTION_TIMEOUT",
 		Value: getOr("TESTKUBE_PRO_CONNECTION_TIMEOUT", "10"),
 	},
+	{
+		Name:  "RUNNER_PRO_API_CERT_FILE",
+		Value: os.Getenv("TESTKUBE_PRO_CERT_FILE"),
+	},
+	{
+		Name:  "RUNNER_PRO_API_KEY_FILE",
+		Value: os.Getenv("TESTKUBE_PRO_KEY_FILE"),
+	},
+	{
+		Name:  "RUNNER_PRO_API_CA_FILE",
+		Value: os.Getenv("TESTKUBE_PRO_CA_FILE"),
+	},
 	{
 		Name:  "RUNNER_DASHBOARD_URI",
 		Value: os.Getenv("TESTKUBE_DASHBOARD_URI"),

pkg/executor/containerexecutor/containerexecutor_test.go

@@ -166,6 +166,9 @@ func TestNewExecutorJobSpecWithArgs(t *testing.T) {
 		{Name: "RUNNER_CLOUD_API_TLS_INSECURE", Value: "false"}, // DEPRECATED
 		{Name: "RUNNER_CLOUD_API_SKIP_VERIFY", Value: "false"},  // DEPRECATED
 		{Name: "RUNNER_CLUSTERID", Value: ""},
+		{Name: "RUNNER_PRO_API_CERT_FILE", Value: ""},
+		{Name: "RUNNER_PRO_API_KEY_FILE", Value: ""},
+		{Name: "RUNNER_PRO_API_CA_FILE", Value: ""},
 		{Name: "CI", Value: "1"},
 		{Name: "key", Value: "value"},
 		{Name: "aa", Value: "bb"},

pkg/executor/scraper/factory/factory.go

@@ -102,7 +102,16 @@ func getRemoteStorageUploader(ctx context.Context, params envs.Params) (uploader
 	output.PrintLogf(
 		"%s Uploading artifacts using Remote Storage Uploader (timeout:%ds, agentInsecure:%v, agentSkipVerify: %v, url: %s, scraperSkipVerify: %v)",
 		ui.IconCheckMark, params.ProConnectionTimeoutSec, params.ProAPITLSInsecure, params.ProAPISkipVerify, params.ProAPIURL, params.SkipVerify)
-	grpcConn, err := agent.NewGRPCConnection(ctxTimeout, params.ProAPITLSInsecure, params.ProAPISkipVerify, params.ProAPIURL, log.DefaultLogger)
+	grpcConn, err := agent.NewGRPCConnection(
+		ctxTimeout,
+		params.ProAPITLSInsecure,
+		params.ProAPISkipVerify,
+		params.ProAPIURL,
+		params.ProAPICertFile,
+		params.ProAPIKeyFile,
+		params.ProAPICAFile,
+		log.DefaultLogger,
+	)
 	if err != nil {
 		return nil, err
 	}

pkg/logs/adapter/cloud_test.go

@@ -36,7 +36,7 @@ func TestCloudAdapter(t *testing.T) {
 		id := "id1"
 
 		// and connection
-		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger)
+		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger)
 		assert.NoError(t, err)
 		defer grpcConn.Close()
 
@@ -78,7 +78,7 @@ func TestCloudAdapter(t *testing.T) {
 		id3 := "id3"
 
 		// and connection
-		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger)
+		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger)
 		assert.NoError(t, err)
 		defer grpcConn.Close()
 		grpcClient := pb.NewCloudLogsServiceClient(grpcConn)
@@ -127,7 +127,7 @@ func TestCloudAdapter(t *testing.T) {
 		id := "id1M"
 
 		// and grpc connetion to the server
-		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger)
+		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger)
 		assert.NoError(t, err)
 		defer grpcConn.Close()
 
@@ -161,7 +161,7 @@ func TestCloudAdapter(t *testing.T) {
 		ctx := context.Background()
 
 		// and grpc connetion to the server
-		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger)
+		grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger)
 		assert.NoError(t, err)
 		defer grpcConn.Close()