diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index e6e59ab9b4d..883d27f4551 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -189,7 +189,16 @@ func main() { mode = common.ModeAgent } if mode == common.ModeAgent { - grpcConn, err = agent.NewGRPCConnection(ctx, cfg.TestkubeProTLSInsecure, cfg.TestkubeProSkipVerify, cfg.TestkubeProURL, log.DefaultLogger) + grpcConn, err = agent.NewGRPCConnection( + ctx, + cfg.TestkubeProTLSInsecure, + cfg.TestkubeProSkipVerify, + cfg.TestkubeProURL, + cfg.TestkubeProCertFile, + cfg.TestkubeProKeyFile, + cfg.TestkubeProCAFile, + log.DefaultLogger, + ) ui.ExitOnError("error creating gRPC connection", err) defer grpcConn.Close() @@ -514,6 +523,7 @@ func main() { features, logsStream, cfg.TestkubeNamespace, + cfg.TestkubeProTLSSecret, ) slackLoader, err := newSlackLoader(cfg, envs) diff --git a/cmd/logs/main.go b/cmd/logs/main.go index 15a67f9889c..aaac9e8699b 100644 --- a/cmd/logs/main.go +++ b/cmd/logs/main.go @@ -103,7 +103,16 @@ func main() { switch mode { case common.ModeAgent: - grpcConn, err := agent.NewGRPCConnection(ctx, cfg.TestkubeProTLSInsecure, cfg.TestkubeProSkipVerify, cfg.TestkubeProURL+cfg.TestkubeProLogsPath, log) + grpcConn, err := agent.NewGRPCConnection( + ctx, + cfg.TestkubeProTLSInsecure, + cfg.TestkubeProSkipVerify, + cfg.TestkubeProURL+cfg.TestkubeProLogsPath, + cfg.TestkubeProCertFile, + cfg.TestkubeProKeyFile, + cfg.TestkubeProCAFile, + log, + ) ui.ExitOnError("error creating gRPC connection for logs service", err) defer grpcConn.Close() grpcClient := pb.NewCloudLogsServiceClient(grpcConn) diff --git a/config/job-container-template.yml b/config/job-container-template.yml index f3372643357..93249d45aed 100644 --- a/config/job-container-template.yml +++ b/config/job-container-template.yml @@ -77,6 +77,11 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .AgentAPITLSSecret }} + - mountPath: /tmp/agent-cert + readOnly: true + name: {{ .AgentAPITLSSecret }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume @@ -103,6 +108,11 @@ spec: secret: secretName: {{ .CertificateSecret }} {{- end }} + {{- if .AgentAPITLSSecret }} + - name: { { .AgentAPITLSSecret } } + secret: + secretName: {{ .AgentAPITLSSecret }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume diff --git a/config/job-template.yml b/config/job-template.yml index 4525c298616..e4ca84162df 100644 --- a/config/job-template.yml +++ b/config/job-template.yml @@ -17,7 +17,7 @@ spec: image: {{ .InitImage }} {{- end }} imagePullPolicy: IfNotPresent - command: + command: - "/bin/runner" - '{{ .Jsn }}' volumeMounts: @@ -27,6 +27,11 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .AgentAPITLSSecret }} + - mountPath: /tmp/agent-cert + readOnly: true + name: {{ .AgentAPITLSSecret }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume @@ -46,7 +51,7 @@ spec: {{- end }} {{- end }} containers: - {{ if .Features.LogsV2 -}} + {{ if .Features.LogsV2 -}} - name: "{{ .Name }}-logs" image: {{ .Registry }}/{{ .LogSidecarImage }} env: @@ -66,7 +71,7 @@ spec: image: {{ .Image }} {{- end }} imagePullPolicy: IfNotPresent - command: + command: - "/bin/runner" - '{{ .Jsn }}' volumeMounts: @@ -76,6 +81,11 @@ spec: - name: {{ .CertificateSecret }} mountPath: /etc/certs {{- end }} + {{- if .AgentAPITLSSecret }} + - mountPath: /tmp/agent-cert + readOnly: true + name: {{ .AgentAPITLSSecret }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume @@ -102,6 +112,11 @@ spec: secret: secretName: {{ .CertificateSecret }} {{- end }} + {{- if .AgentAPITLSSecret }} + - name: {{ .AgentAPITLSSecret }} + secret: + secretName: {{ .AgentAPITLSSecret }} + {{- end }} {{- if .ArtifactRequest }} {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume diff --git a/internal/config/config.go b/internal/config/config.go index 38f5c964955..f51a93de4f0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -73,6 +73,10 @@ type Config struct { TestkubeProOrgID string `envconfig:"TESTKUBE_PRO_ORG_ID" default:""` TestkubeProMigrate string `envconfig:"TESTKUBE_PRO_MIGRATE" default:"false"` TestkubeProConnectionTimeout int `envconfig:"TESTKUBE_PRO_CONNECTION_TIMEOUT" default:"10"` + TestkubeProCertFile string `envconfig:"TESTKUBE_PRO_CERT_FILE" default:""` + TestkubeProKeyFile string `envconfig:"TESTKUBE_PRO_KEY_FILE" default:""` + TestkubeProCAFile string `envconfig:"TESTKUBE_PRO_CA_FILE" default:""` + TestkubeProTLSSecret string `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""` TestkubeWatcherNamespaces string `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""` GraphqlPort string `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"` TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""` diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 84676975829..fc10230d5f6 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -3,8 +3,10 @@ package agent import ( "context" "crypto/tls" + "crypto/x509" "fmt" "math" + "os" "time" "google.golang.org/grpc/keepalive" @@ -42,13 +44,32 @@ const ( // buffer up to five messages per worker const bufferSizePerWorker = 5 -func NewGRPCConnection(ctx context.Context, isInsecure bool, skipVerify bool, server string, logger *zap.SugaredLogger) (*grpc.ClientConn, error) { +func NewGRPCConnection( + ctx context.Context, + isInsecure bool, + skipVerify bool, + server string, + certFile, keyFile, caFile string, + logger *zap.SugaredLogger, +) (*grpc.ClientConn, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - var tlsConfig *tls.Config + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} if skipVerify { tlsConfig = &tls.Config{InsecureSkipVerify: true} + } else { + if certFile != "" && keyFile != "" { + if err := clientCert(tlsConfig, certFile, keyFile); err != nil { + return nil, err + } + } + if caFile != "" { + if err := rootCAs(tlsConfig, caFile); err != nil { + return nil, err + } + } } + creds := credentials.NewTLS(tlsConfig) if isInsecure { creds = insecure.NewCredentials() @@ -76,6 +97,35 @@ func NewGRPCConnection(ctx context.Context, isInsecure bool, skipVerify bool, se ) } +func rootCAs(tlsConfig *tls.Config, file ...string) error { + pool := x509.NewCertPool() + for _, f := range file { + rootPEM, err := os.ReadFile(f) + if err != nil || rootPEM == nil { + return fmt.Errorf("agent: error loading or parsing rootCA file: %v", err) + } + ok := pool.AppendCertsFromPEM(rootPEM) + if !ok { + return fmt.Errorf("agent: failed to parse root certificate from %q", f) + } + } + tlsConfig.RootCAs = pool + return nil +} + +func clientCert(tlsConfig *tls.Config, certFile, keyFile string) error { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return fmt.Errorf("agent: error loading client certificate: %v", err) + } + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return fmt.Errorf("agent: error parsing client certificate: %v", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + return nil +} + type Agent struct { client cloud.TestKubeCloudAPIClient handler fasthttp.RequestHandler diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 0dedd5ff6a1..b2ac1de903d 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -49,7 +49,7 @@ func TestCommandExecution(t *testing.T) { atomic.AddInt32(&msgCnt, 1) } - grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger) ui.ExitOnError("error creating gRPC connection", err) defer grpcConn.Close() diff --git a/pkg/agent/events_test.go b/pkg/agent/events_test.go index dc2c716c5fc..4efc81e3b28 100644 --- a/pkg/agent/events_test.go +++ b/pkg/agent/events_test.go @@ -47,7 +47,7 @@ func TestEventLoop(t *testing.T) { logger, _ := zap.NewDevelopment() - grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger) ui.ExitOnError("error creating gRPC connection", err) defer grpcConn.Close() diff --git a/pkg/agent/logs_test.go b/pkg/agent/logs_test.go index c38b125e05e..0273811ec8c 100644 --- a/pkg/agent/logs_test.go +++ b/pkg/agent/logs_test.go @@ -45,7 +45,7 @@ func TestLogStream(t *testing.T) { fmt.Fprintf(ctx, "Hi there! RequestURI is %q", ctx.RequestURI()) } - grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(context.Background(), true, false, url, "", "", "", log.DefaultLogger) ui.ExitOnError("error creating gRPC connection", err) defer grpcConn.Close() diff --git a/pkg/envs/variables.go b/pkg/envs/variables.go index 5c9fd06aa7f..1afb930fd50 100644 --- a/pkg/envs/variables.go +++ b/pkg/envs/variables.go @@ -48,6 +48,9 @@ type Params struct { ProAPIURL string `envconfig:"RUNNER_PRO_API_URL"` // RUNNER_PRO_API_URL ProConnectionTimeoutSec int `envconfig:"RUNNER_PRO_CONNECTION_TIMEOUT" default:"10"` // RUNNER_PRO_CONNECTION_TIMEOUT ProAPISkipVerify bool `envconfig:"RUNNER_PRO_API_SKIP_VERIFY" default:"false"` // RUNNER_PRO_API_SKIP_VERIFY + ProAPICertFile string `envconfig:"RUNNER_PRO_API_CERT_FILE"` // RUNNER_PRO_API_CERT_FILE + ProAPIKeyFile string `envconfig:"RUNNER_PRO_API_KEY_FILE"` // RUNNER_PRO_API_KEY_FILE + ProAPICAFile string `envconfig:"RUNNER_PRO_API_CA_FILE"` // RUNNER_PRO_API_CA_FILE SlavesConfigs string `envconfig:"RUNNER_SLAVES_CONFIGS"` // RUNNER_SLAVES_CONFIGS } diff --git a/pkg/executor/client/common.go b/pkg/executor/client/common.go index a923f65d4df..7ca6279a126 100644 --- a/pkg/executor/client/common.go +++ b/pkg/executor/client/common.go @@ -23,19 +23,20 @@ const ( ) type ExecuteOptions struct { - ID string - TestName string - Namespace string - TestSpec testsv3.TestSpec - ExecutorName string - ExecutorSpec executorv1.ExecutorSpec - Request testkube.ExecutionRequest - Sync bool - Labels map[string]string - UsernameSecret *testkube.SecretRef - TokenSecret *testkube.SecretRef + ID string + TestName string + Namespace string + TestSpec testsv3.TestSpec + ExecutorName string + ExecutorSpec executorv1.ExecutorSpec + Request testkube.ExecutionRequest + Sync bool + Labels map[string]string + UsernameSecret *testkube.SecretRef + TokenSecret *testkube.SecretRef + CertificateSecret string + // AgentAPITLSSecret is a secret name that contains TLS certificate for Agent (gRPC) API AgentAPITLSSecret string - CertificateSecret string ImagePullSecretNames []string Features featureflags.FeatureFlags } diff --git a/pkg/executor/client/job.go b/pkg/executor/client/job.go index 524fdf842c3..d72ae3e09ec 100644 --- a/pkg/executor/client/job.go +++ b/pkg/executor/client/job.go @@ -172,6 +172,7 @@ type JobOptions struct { UsernameSecret *testkube.SecretRef TokenSecret *testkube.SecretRef CertificateSecret string + AgentAPITLSSecret string Variables map[string]testkube.Variable ActiveDeadlineSeconds int64 ServiceAccountName string @@ -1021,6 +1022,9 @@ func NewJobOptions(log *zap.SugaredLogger, templatesClient templatesv1.Interface } } + // used for adding custom certificates for Agent (gRPC) API + jobOptions.AgentAPITLSSecret = options.AgentAPITLSSecret + return } diff --git a/pkg/executor/common.go b/pkg/executor/common.go index 0a281c8dca2..29edaaaa339 100644 --- a/pkg/executor/common.go +++ b/pkg/executor/common.go @@ -127,6 +127,18 @@ var RunnerEnvVars = []corev1.EnvVar{ Name: "RUNNER_PRO_CONNECTION_TIMEOUT", Value: getOr("TESTKUBE_PRO_CONNECTION_TIMEOUT", "10"), }, + { + Name: "RUNNER_PRO_API_CERT_FILE", + Value: os.Getenv("TESTKUBE_PRO_CERT_FILE"), + }, + { + Name: "RUNNER_PRO_API_KEY_FILE", + Value: os.Getenv("TESTKUBE_PRO_KEY_FILE"), + }, + { + Name: "RUNNER_PRO_API_CA_FILE", + Value: os.Getenv("TESTKUBE_PRO_CA_FILE"), + }, { Name: "RUNNER_DASHBOARD_URI", Value: os.Getenv("TESTKUBE_DASHBOARD_URI"), diff --git a/pkg/executor/containerexecutor/containerexecutor_test.go b/pkg/executor/containerexecutor/containerexecutor_test.go index c753cd31cd3..219329a60c6 100644 --- a/pkg/executor/containerexecutor/containerexecutor_test.go +++ b/pkg/executor/containerexecutor/containerexecutor_test.go @@ -166,6 +166,9 @@ func TestNewExecutorJobSpecWithArgs(t *testing.T) { {Name: "RUNNER_CLOUD_API_TLS_INSECURE", Value: "false"}, // DEPRECATED {Name: "RUNNER_CLOUD_API_SKIP_VERIFY", Value: "false"}, // DEPRECATED {Name: "RUNNER_CLUSTERID", Value: ""}, + {Name: "RUNNER_PRO_API_CERT_FILE", Value: ""}, + {Name: "RUNNER_PRO_API_KEY_FILE", Value: ""}, + {Name: "RUNNER_PRO_API_CA_FILE", Value: ""}, {Name: "CI", Value: "1"}, {Name: "key", Value: "value"}, {Name: "aa", Value: "bb"}, diff --git a/pkg/executor/scraper/factory/factory.go b/pkg/executor/scraper/factory/factory.go index ef29fcd942c..1b8d89bff00 100644 --- a/pkg/executor/scraper/factory/factory.go +++ b/pkg/executor/scraper/factory/factory.go @@ -102,7 +102,16 @@ func getRemoteStorageUploader(ctx context.Context, params envs.Params) (uploader output.PrintLogf( "%s Uploading artifacts using Remote Storage Uploader (timeout:%ds, agentInsecure:%v, agentSkipVerify: %v, url: %s, scraperSkipVerify: %v)", ui.IconCheckMark, params.ProConnectionTimeoutSec, params.ProAPITLSInsecure, params.ProAPISkipVerify, params.ProAPIURL, params.SkipVerify) - grpcConn, err := agent.NewGRPCConnection(ctxTimeout, params.ProAPITLSInsecure, params.ProAPISkipVerify, params.ProAPIURL, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection( + ctxTimeout, + params.ProAPITLSInsecure, + params.ProAPISkipVerify, + params.ProAPIURL, + params.ProAPICertFile, + params.ProAPIKeyFile, + params.ProAPICAFile, + log.DefaultLogger, + ) if err != nil { return nil, err } diff --git a/pkg/logs/adapter/cloud_test.go b/pkg/logs/adapter/cloud_test.go index fb639323997..54a8866c805 100644 --- a/pkg/logs/adapter/cloud_test.go +++ b/pkg/logs/adapter/cloud_test.go @@ -36,7 +36,7 @@ func TestCloudAdapter(t *testing.T) { id := "id1" // and connection - grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger) assert.NoError(t, err) defer grpcConn.Close() @@ -78,7 +78,7 @@ func TestCloudAdapter(t *testing.T) { id3 := "id3" // and connection - grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger) assert.NoError(t, err) defer grpcConn.Close() grpcClient := pb.NewCloudLogsServiceClient(grpcConn) @@ -127,7 +127,7 @@ func TestCloudAdapter(t *testing.T) { id := "id1M" // and grpc connetion to the server - grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger) assert.NoError(t, err) defer grpcConn.Close() @@ -161,7 +161,7 @@ func TestCloudAdapter(t *testing.T) { ctx := context.Background() // and grpc connetion to the server - grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, log.DefaultLogger) + grpcConn, err := agent.NewGRPCConnection(ctx, true, true, ts.Url, "", "", "", log.DefaultLogger) assert.NoError(t, err) defer grpcConn.Close() diff --git a/pkg/logs/config/logs_config.go b/pkg/logs/config/logs_config.go index 89b820cdb68..4586d2a6ef2 100644 --- a/pkg/logs/config/logs_config.go +++ b/pkg/logs/config/logs_config.go @@ -13,6 +13,9 @@ type Config struct { TestkubeProURL string `envconfig:"TESTKUBE_PRO_URL" default:""` TestkubeProLogsPath string `envconfig:"TESTKUBE_PRO_LOGS_PATH" default:"/logs"` TestkubeProTLSInsecure bool `envconfig:"TESTKUBE_PRO_TLS_INSECURE" default:"false"` + TestkubeProCertFile string `envconfig:"TESTKUBE_PRO_CERT_FILE" default:""` + TestkubeProKeyFile string `envconfig:"TESTKUBE_PRO_KEY_FILE" default:""` + TestkubeProCAFile string `envconfig:"TESTKUBE_PRO_CA_FILE" default:""` TestkubeProWorkerCount int `envconfig:"TESTKUBE_PRO_WORKER_COUNT" default:"50"` TestkubeProLogStreamWorkerCount int `envconfig:"TESTKUBE_PRO_LOG_STREAM_WORKER_COUNT" default:"25"` TestkubeProSkipVerify bool `envconfig:"TESTKUBE_PRO_SKIP_VERIFY" default:"false"` diff --git a/pkg/scheduler/service.go b/pkg/scheduler/service.go index a7e4550417c..8a4ea3c25f8 100644 --- a/pkg/scheduler/service.go +++ b/pkg/scheduler/service.go @@ -45,6 +45,7 @@ type Scheduler struct { logsStream logsclient.Stream subscriptionChecker checktcl.SubscriptionChecker namespace string + agentAPITLSSecret string } func NewScheduler( @@ -68,6 +69,7 @@ func NewScheduler( featureFlags featureflags.FeatureFlags, logsStream logsclient.Stream, namespace string, + agentAPITLSSecret string, ) *Scheduler { return &Scheduler{ metrics: metrics, @@ -90,6 +92,7 @@ func NewScheduler( featureFlags: featureFlags, logsStream: logsStream, namespace: namespace, + agentAPITLSSecret: agentAPITLSSecret, } } diff --git a/pkg/scheduler/test_scheduler.go b/pkg/scheduler/test_scheduler.go index ccbfefb6666..18860f72ded 100644 --- a/pkg/scheduler/test_scheduler.go +++ b/pkg/scheduler/test_scheduler.go @@ -557,6 +557,7 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe UsernameSecret: usernameSecret, TokenSecret: tokenSecret, CertificateSecret: certificateSecret, + AgentAPITLSSecret: s.agentAPITLSSecret, ImagePullSecretNames: imagePullSecrets, Features: s.featureFlags, }, nil diff --git a/pkg/triggers/executor_test.go b/pkg/triggers/executor_test.go index 907d5b39c19..1673a20cd26 100644 --- a/pkg/triggers/executor_test.go +++ b/pkg/triggers/executor_test.go @@ -127,6 +127,7 @@ func TestExecute(t *testing.T) { featureflags.FeatureFlags{}, mockLogsStream, "", + "", ) s := &Service{ triggerStatus: make(map[statusKey]*triggerStatus), diff --git a/pkg/triggers/service_test.go b/pkg/triggers/service_test.go index 290beb0ce5b..13389131cd1 100644 --- a/pkg/triggers/service_test.go +++ b/pkg/triggers/service_test.go @@ -140,6 +140,7 @@ func TestService_Run(t *testing.T) { featureflags.FeatureFlags{}, mockLogsStream, "", + "", ) mockLeaseBackend := NewMockLeaseBackend(mockCtrl)