feat: inject custom ca (#5269)

* feat: add runner custom ca variable * feat: add runner ca certificate to default job template * feat: add runner ca certificate to job template * feat: add runner ca certificate to job scraper template * feat: add runner ca certificate to slave pod template
kubeshop · Apr 3, 2024 · e515775 · e515775
1 parent 8358103
commit e515775
Show file tree

Hide file tree

Showing 13 changed files with 152 additions and 42 deletions.
diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go
@@ -533,6 +533,7 @@ func main() {
 		logsStream,
 		cfg.TestkubeNamespace,
 		cfg.TestkubeProTLSSecret,
+		cfg.TestkubeProRunnerCustomCASecret,
 	)
 	if mode == common.ModeAgent {
 		sched.WithSubscriptionChecker(subscriptionChecker)

diff --git a/config/job-scraper-template.yml b/config/job-scraper-template.yml
@@ -46,16 +46,33 @@ spec:
         command:
           - "/bin/runner"
           - '{{ .Jsn }}'
+        {{- if .RunnerCustomCASecret }}
+        env:
+          - name: SSL_CERT_DIR
+            value: /etc/testkube/certs
+        {{- end }}
+        volumeMounts:
+        {{- if .RunnerCustomCASecret }}
+          - name: {{ .RunnerCustomCASecret }}
+            mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+            readOnly: true
+            subPath: ca.crt
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if .ArtifactRequest.VolumeMountPath }}
-        volumeMounts:
           - name: artifact-volume
             mountPath: {{ .ArtifactRequest.VolumeMountPath }}
           {{- end }}
         {{- end }}
+      volumes:
+      {{- if .RunnerCustomCASecret }}
+        - name: {{ .RunnerCustomCASecret }}
+          secret:
+            secretName: {{ .RunnerCustomCASecret }}
+            defaultMode: 420
+      {{- end }}
       {{- if .ArtifactRequest }}
         {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
-      volumes:
         - name: artifact-volume
           persistentVolumeClaim:
             claimName: {{ .Name }}-pvc

diff --git a/config/job-template.yml b/config/job-template.yml
@@ -20,13 +20,24 @@ spec:
         command:
           - "/bin/runner"
           - '{{ .Jsn }}'
+        {{- if .RunnerCustomCASecret }}
+        env:
+          - name: SSL_CERT_DIR
+            value: /etc/testkube/certs
+        {{- end }}
         volumeMounts:
         - name: data-volume
           mountPath: /data
         {{- if .CertificateSecret }}
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .RunnerCustomCASecret }}
+        - name: {{ .RunnerCustomCASecret }}
+          mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+          readOnly: true
+          subPath: ca.crt
+        {{- end }}
         {{- if .AgentAPITLSSecret }}
         - mountPath: /tmp/agent-cert
           readOnly: true
@@ -86,13 +97,24 @@ spec:
         command:
           - "/bin/runner"
           - '{{ .Jsn }}'
+        {{- if .RunnerCustomCASecret }}
+        env:
+          - name: SSL_CERT_DIR
+            value: /etc/testkube/certs
+        {{- end }}
         volumeMounts:
         - name: data-volume
           mountPath: /data
         {{- if .CertificateSecret }}
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .RunnerCustomCASecret }}
+        - name: {{ .RunnerCustomCASecret }}
+          mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+          readOnly: true
+          subPath: ca.crt
+        {{- end }}
         {{- if .AgentAPITLSSecret }}
         - mountPath: /tmp/agent-cert
           readOnly: true
@@ -124,6 +146,12 @@ spec:
         secret:
           secretName: {{ .CertificateSecret }}
       {{- end }}
+      {{- if .RunnerCustomCASecret }}
+      - name: {{ .RunnerCustomCASecret }}
+        secret:
+          secretName: {{ .RunnerCustomCASecret }}
+          defaultMode: 420
+      {{- end }}
       {{- if .AgentAPITLSSecret }}
       - name: {{ .AgentAPITLSSecret }}
         secret:

diff --git a/config/slave-pod-template.yml b/config/slave-pod-template.yml
@@ -25,13 +25,24 @@ spec:
     command:
       - "/bin/runner"
       - '{{ .Jsn }}'
+    {{- if .RunnerCustomCASecret }}
+    env:
+      - name: SSL_CERT_DIR
+        value: /etc/testkube/certs
+    {{- end }}
     volumeMounts:
     - name: data-volume
       mountPath: /data
     {{- if .CertificateSecret }}
     - name: {{ .CertificateSecret }}
       mountPath: /etc/certs
     {{- end }}
+    {{- if .RunnerCustomCASecret }}
+    - name: {{ .RunnerCustomCASecret }}
+      mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+      readOnly: true
+      subPath: ca.crt
+    {{- end }}
     {{- if .ArtifactRequest }}
       {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
     - name: artifact-volume
@@ -159,6 +170,12 @@ spec:
     secret:
       secretName: {{ .CertificateSecret }}
   {{- end }}
+  {{- if .RunnerCustomCASecret }}
+  - name: {{ .RunnerCustomCASecret }}
+    secret:
+      secretName: {{ .RunnerCustomCASecret }}
+      defaultMode: 420
+  {{- end }}
   {{- if .ArtifactRequest }}
     {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
   - name: artifact-volume

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -75,33 +75,34 @@ type Config struct {
 	TestkubeProConnectionTimeout                int           `envconfig:"TESTKUBE_PRO_CONNECTION_TIMEOUT" default:"10"`
 	TestkubeProCertFile                         string        `envconfig:"TESTKUBE_PRO_CERT_FILE" default:""`
 	TestkubeProKeyFile                          string        `envconfig:"TESTKUBE_PRO_KEY_FILE" default:""`
-	TestkubeProCAFile                           string        `envconfig:"TESTKUBE_PRO_CA_FILE" default:""`
-	TestkubeProTLSSecret                        string        `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""`
-	TestkubeWatcherNamespaces                   string        `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""`
-	GraphqlPort                                 string        `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
-	TestkubeRegistry                            string        `envconfig:"TESTKUBE_REGISTRY" default:""`
-	TestkubePodStartTimeout                     time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"`
-	CDEventsTarget                              string        `envconfig:"CDEVENTS_TARGET" default:""`
-	TestkubeDashboardURI                        string        `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
-	DisableReconciler                           bool          `envconfig:"DISABLE_RECONCILER" default:"false"`
-	TestkubeClusterName                         string        `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
-	CompressArtifacts                           bool          `envconfig:"COMPRESSARTIFACTS" default:"false"`
-	TestkubeHelmchartVersion                    string        `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
-	DebugListenAddr                             string        `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
-	EnableDebugServer                           bool          `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
-	EnableSecretsEndpoint                       bool          `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
-	DisableMongoMigrations                      bool          `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
-	Debug                                       bool          `envconfig:"DEBUG" default:"false"`
-	EnableImageDataPersistentCache              bool          `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
-	ImageDataPersistentCacheKey                 string        `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
-	LogServerGrpcAddress                        string        `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
-	LogServerSecure                             bool          `envconfig:"LOG_SERVER_SECURE" default:"false"`
-	LogServerSkipVerify                         bool          `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
-	LogServerCertFile                           string        `envconfig:"LOG_SERVER_CERT_FILE" default:""`
-	LogServerKeyFile                            string        `envconfig:"LOG_SERVER_KEY_FILE" default:""`
-	LogServerCAFile                             string        `envconfig:"LOG_SERVER_CA_FILE" default:""`
-	DisableSecretCreation                       bool          `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
-	TestkubeExecutionNamespaces                 string        `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
+
+	TestkubeProTLSSecret            string        `envconfig:"TESTKUBE_PRO_TLS_SECRET" default:""`
+	TestkubeProRunnerCustomCASecret string        `envconfig:"TESTKUBE_PRO_RUNNER_CUSTOM_CA_SECRET" default:""`
+	TestkubeWatcherNamespaces       string        `envconfig:"TESTKUBE_WATCHER_NAMESPACES" default:""`
+	GraphqlPort                     string        `envconfig:"TESTKUBE_GRAPHQL_PORT" default:"8070"`
+	TestkubeRegistry                string        `envconfig:"TESTKUBE_REGISTRY" default:""`
+	TestkubePodStartTimeout         time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"`
+	CDEventsTarget                  string        `envconfig:"CDEVENTS_TARGET" default:""`
+	TestkubeDashboardURI            string        `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
+	DisableReconciler               bool          `envconfig:"DISABLE_RECONCILER" default:"false"`
+	TestkubeClusterName             string        `envconfig:"TESTKUBE_CLUSTER_NAME" default:""`
+	CompressArtifacts               bool          `envconfig:"COMPRESSARTIFACTS" default:"false"`
+	TestkubeHelmchartVersion        string        `envconfig:"TESTKUBE_HELMCHART_VERSION" default:""`
+	DebugListenAddr                 string        `envconfig:"DEBUG_LISTEN_ADDR" default:"0.0.0.0:1337"`
+	EnableDebugServer               bool          `envconfig:"ENABLE_DEBUG_SERVER" default:"false"`
+	EnableSecretsEndpoint           bool          `envconfig:"ENABLE_SECRETS_ENDPOINT" default:"false"`
+	DisableMongoMigrations          bool          `envconfig:"DISABLE_MONGO_MIGRATIONS" default:"false"`
+	Debug                           bool          `envconfig:"DEBUG" default:"false"`
+	EnableImageDataPersistentCache  bool          `envconfig:"TESTKUBE_ENABLE_IMAGE_DATA_PERSISTENT_CACHE" default:"false"`
+	ImageDataPersistentCacheKey     string        `envconfig:"TESTKUBE_IMAGE_DATA_PERSISTENT_CACHE_KEY" default:"testkube-image-cache"`
+	LogServerGrpcAddress            string        `envconfig:"LOG_SERVER_GRPC_ADDRESS" default:":9090"`
+	LogServerSecure                 bool          `envconfig:"LOG_SERVER_SECURE" default:"false"`
+	LogServerSkipVerify             bool          `envconfig:"LOG_SERVER_SKIP_VERIFY" default:"false"`
+	LogServerCertFile               string        `envconfig:"LOG_SERVER_CERT_FILE" default:""`
+	LogServerKeyFile                string        `envconfig:"LOG_SERVER_KEY_FILE" default:""`
+	LogServerCAFile                 string        `envconfig:"LOG_SERVER_CA_FILE" default:""`
+	DisableSecretCreation           bool          `envconfig:"DISABLE_SECRET_CREATION" default:"false"`
+	TestkubeExecutionNamespaces     string        `envconfig:"TESTKUBE_EXECUTION_NAMESPACES" default:""`
 
 	// DEPRECATED: Use TestkubeProAPIKey instead
 	TestkubeCloudAPIKey string `envconfig:"TESTKUBE_CLOUD_API_KEY" default:""`
@@ -119,6 +120,13 @@ type Config struct {
 	TestkubeCloudOrgID string `envconfig:"TESTKUBE_CLOUD_ORG_ID" default:""`
 	// DEPRECATED: Use TestkubeProMigrate instead
 	TestkubeCloudMigrate string `envconfig:"TESTKUBE_CLOUD_MIGRATE" default:"false"`
+
+	// TestkubeProCAFile is meant to provide a custom CA when making a TLS connection to
+	// the agent API.
+	//
+	// Deprecated: Instead mount a CA file into a directory and specify the diretory
+	// path with the SSL_CERT_DIR environment variable.
+	TestkubeProCAFile string `envconfig:"TESTKUBE_PRO_CA_FILE" default:""`
 }
 
 func Get() (*Config, error) {

diff --git a/pkg/executor/client/common.go b/pkg/executor/client/common.go
@@ -23,18 +23,19 @@ const (
 )
 
 type ExecuteOptions struct {
-	ID                string
-	TestName          string
-	Namespace         string
-	TestSpec          testsv3.TestSpec
-	ExecutorName      string
-	ExecutorSpec      executorv1.ExecutorSpec
-	Request           testkube.ExecutionRequest
-	Sync              bool
-	Labels            map[string]string
-	UsernameSecret    *testkube.SecretRef
-	TokenSecret       *testkube.SecretRef
-	CertificateSecret string
+	ID                   string
+	TestName             string
+	Namespace            string
+	TestSpec             testsv3.TestSpec
+	ExecutorName         string
+	ExecutorSpec         executorv1.ExecutorSpec
+	Request              testkube.ExecutionRequest
+	Sync                 bool
+	Labels               map[string]string
+	UsernameSecret       *testkube.SecretRef
+	TokenSecret          *testkube.SecretRef
+	RunnerCustomCASecret string
+	CertificateSecret    string
 	// AgentAPITLSSecret is a secret name that contains TLS certificate for Agent (gRPC) API
 	AgentAPITLSSecret    string
 	ImagePullSecretNames []string

diff --git a/pkg/executor/client/job.go b/pkg/executor/client/job.go
@@ -171,6 +171,7 @@ type JobOptions struct {
 	HTTPSProxy            string
 	UsernameSecret        *testkube.SecretRef
 	TokenSecret           *testkube.SecretRef
+	RunnerCustomCASecret  string
 	CertificateSecret     string
 	AgentAPITLSSecret     string
 	Variables             map[string]testkube.Variable
@@ -614,6 +615,7 @@ func NewJobOptionsFromExecutionOptions(options ExecuteOptions) JobOptions {
 		HTTPSProxy:            options.Request.HttpsProxy,
 		UsernameSecret:        options.UsernameSecret,
 		TokenSecret:           options.TokenSecret,
+		RunnerCustomCASecret:  options.RunnerCustomCASecret,
 		CertificateSecret:     options.CertificateSecret,
 		ActiveDeadlineSeconds: options.Request.ActiveDeadlineSeconds,
 		JobTemplateExtensions: options.Request.JobTemplate,

diff --git a/pkg/executor/containerexecutor/containerexecutor.go b/pkg/executor/containerexecutor/containerexecutor.go
@@ -167,6 +167,7 @@ type JobOptions struct {
 	HTTPSProxy                string
 	UsernameSecret            *testkube.SecretRef
 	TokenSecret               *testkube.SecretRef
+	RunnerCustomCASecret      string
 	CertificateSecret         string
 	AgentAPITLSSecret         string
 	Variables                 map[string]testkube.Variable
@@ -683,6 +684,7 @@ func NewJobOptionsFromExecutionOptions(options client.ExecuteOptions) *JobOption
 		HTTPSProxy:                options.Request.HttpsProxy,
 		UsernameSecret:            options.UsernameSecret,
 		TokenSecret:               options.TokenSecret,
+		RunnerCustomCASecret:      options.RunnerCustomCASecret,
 		CertificateSecret:         options.CertificateSecret,
 		AgentAPITLSSecret:         options.AgentAPITLSSecret,
 		ActiveDeadlineSeconds:     options.Request.ActiveDeadlineSeconds,

diff --git a/pkg/executor/containerexecutor/templates/job.tmpl b/pkg/executor/containerexecutor/templates/job.tmpl
@@ -22,13 +22,24 @@ spec:
         command:
           - "/bin/runner"
           - '{{ .Jsn }}'
+        {{- if .RunnerCustomCASecret }}
+        env:
+          - name: SSL_CERT_DIR
+            value: /etc/testkube/certs
+        {{- end }}
         volumeMounts:
         - name: data-volume
           mountPath: /data
         {{- if .CertificateSecret }}
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .RunnerCustomCASecret }}
+        - name: {{ .RunnerCustomCASecret }}
+          mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+          readOnly: true
+          subPath: ca.crt
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if .ArtifactRequest.VolumeMountPath }}
         - name: artifact-volume
@@ -49,7 +60,7 @@ spec:
         {{- end }}
       {{- end }}
       containers:
-      {{ if .Features.LogsV2 -}} 
+      {{ if .Features.LogsV2 -}}
       - name: "{{ .Name }}-logs"
         image: {{ .Registry }}/{{ .LogSidecarImage }}
         env:
@@ -84,13 +95,24 @@ spec:
         {{- if .WorkingDir }}
         workingDir: {{ .WorkingDir }}
         {{- end }}
+        {{- if .RunnerCustomCASecret }}
+        env:
+          - name: SSL_CERT_DIR
+            value: /etc/testkube/certs
+        {{- end }}
         volumeMounts:
         - name: data-volume
           mountPath: /data
         {{- if .CertificateSecret }}
         - name: {{ .CertificateSecret }}
           mountPath: /etc/certs
         {{- end }}
+        {{- if .RunnerCustomCASecret }}
+        - name: {{ .RunnerCustomCASecret }}
+          mountPath: /etc/testkube/certs/testkube-custom-ca.pem
+          readOnly: true
+          subPath: ca.crt
+        {{- end }}
         {{- if .ArtifactRequest }}
           {{- if .ArtifactRequest.VolumeMountPath }}
         - name: artifact-volume
@@ -117,6 +139,12 @@ spec:
         secret:
           secretName: {{ .CertificateSecret }}
       {{- end }}
+      {{- if .RunnerCustomCASecret }}
+      - name: {{ .RunnerCustomCASecret }}
+        secret:
+          secretName: {{ .RunnerCustomCASecret }}
+          defaultMode: 420
+      {{- end }}
       {{- if .ArtifactRequest }}
         {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }}
       - name: artifact-volume

diff --git a/pkg/scheduler/service.go b/pkg/scheduler/service.go
@@ -46,6 +46,7 @@ type Scheduler struct {
 	subscriptionChecker       checktcl.SubscriptionChecker
 	namespace                 string
 	agentAPITLSSecret         string
+	runnerCustomCASecret      string
 }
 
 func NewScheduler(
@@ -70,6 +71,7 @@ func NewScheduler(
 	logsStream logsclient.Stream,
 	namespace string,
 	agentAPITLSSecret string,
+	runnerCustomCASecret string,
 ) *Scheduler {
 	return &Scheduler{
 		metrics:                   metrics,
@@ -93,6 +95,7 @@ func NewScheduler(
 		logsStream:                logsStream,
 		namespace:                 namespace,
 		agentAPITLSSecret:         agentAPITLSSecret,
+		runnerCustomCASecret:      runnerCustomCASecret,
 	}
 }
 

diff --git a/pkg/scheduler/test_scheduler.go b/pkg/scheduler/test_scheduler.go
@@ -556,6 +556,7 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe
 		Labels:               testCR.Labels,
 		UsernameSecret:       usernameSecret,
 		TokenSecret:          tokenSecret,
+		RunnerCustomCASecret: s.runnerCustomCASecret,
 		CertificateSecret:    certificateSecret,
 		AgentAPITLSSecret:    s.agentAPITLSSecret,
 		ImagePullSecretNames: imagePullSecrets,

diff --git a/pkg/triggers/executor_test.go b/pkg/triggers/executor_test.go
@@ -128,6 +128,7 @@ func TestExecute(t *testing.T) {
 		mockLogsStream,
 		"",
 		"",
+		"",
 	)
 	s := &Service{
 		triggerStatus:    make(map[statusKey]*triggerStatus),