From f22b33c67add7b9deaadb359cf047213602c3d56 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Mon, 26 Aug 2024 17:14:20 +0200 Subject: [PATCH] simplification Signed-off-by: Jan Wozniak --- .../interceptor/e2e-test/tls/deployment.yaml | 8 + interceptor/main.go | 36 +-- .../interceptor_multi_tls_test.go | 207 ------------------ .../interceptor_tls/interceptor_tls_test.go | 7 +- tests/utils/setup_test.go | 3 + 5 files changed, 37 insertions(+), 224 deletions(-) delete mode 100644 tests/checks/interceptor_multi_tls/interceptor_multi_tls_test.go diff --git a/config/interceptor/e2e-test/tls/deployment.yaml b/config/interceptor/e2e-test/tls/deployment.yaml index 0b8365248..2f7a4437d 100644 --- a/config/interceptor/e2e-test/tls/deployment.yaml +++ b/config/interceptor/e2e-test/tls/deployment.yaml @@ -18,13 +18,21 @@ spec: value: "/certs/tls.crt" - name: KEDA_HTTP_PROXY_TLS_KEY_PATH value: "/certs/tls.key" + - name: KEDA_HTTP_PROXY_TLS_CERT_STORE_PATHS + value: "/additional-certs" - name: KEDA_HTTP_PROXY_TLS_PORT value: "8443" volumeMounts: - readOnly: true mountPath: "/certs" name: certs + - readOnly: true + mountPath: "/additional-certs/abc-certs" + name: abc-certs volumes: - name: certs secret: secretName: keda-tls + - name: abc-certs + secret: + secretName: abc-certs diff --git a/interceptor/main.go b/interceptor/main.go index 11e1ac3dc..52114692b 100644 --- a/interceptor/main.go +++ b/interceptor/main.go @@ -15,6 +15,7 @@ import ( "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -235,18 +236,18 @@ func runMetricsServer( } // addCert adds a certificate to the map of certificates based on the certificate's SANs -func addCert(m map[string]tls.Certificate, certPath, keyPath string, logger logr.Logger) error { +func addCert(m map[string]tls.Certificate, certPath, keyPath string, logger logr.Logger) (*tls.Certificate, error) { cert, err := tls.LoadX509KeyPair(certPath, keyPath) if err != nil { - return fmt.Errorf("error loading certificate and key: %w", err) + return nil, fmt.Errorf("error loading certificate and key: %w", err) } if cert.Leaf == nil { if len(cert.Certificate) == 0 { - return fmt.Errorf("no certificate found in certificate chain") + return nil, fmt.Errorf("no certificate found in certificate chain") } cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) if err != nil { - return fmt.Errorf("error parsing certificate: %w", err) + return nil, fmt.Errorf("error parsing certificate: %w", err) } } for _, d := range cert.Leaf.DNSNames { @@ -261,21 +262,24 @@ func addCert(m map[string]tls.Certificate, certPath, keyPath string, logger logr logger.Info("adding certificate", "uri", uri.String()) m[uri.String()] = cert } - return nil + return &cert, nil } // makeCertGetter creates a function that returns a certificate based on the server name // The matching between request and certificate is performed by comparing TLS/SNI server name with x509 SANs -func makeCertGetter(tlsConfig map[string]string, logger logr.Logger) (func(*tls.ClientHelloInfo) (*tls.Certificate, error), error) { +func makeCertGetter(tlsConfig map[string]string, logger logr.Logger) (func(*tls.ClientHelloInfo) (*tls.Certificate, error), []tls.Certificate, error) { certPath := tlsConfig["certificatePath"] keyPath := tlsConfig["keyPath"] certStorePaths := tlsConfig["certstorePaths"] + var defaultCert *tls.Certificate uriDomainsToCerts := make(map[string]tls.Certificate) if certPath != "" && keyPath != "" { - if err := addCert(uriDomainsToCerts, certPath, keyPath, logger); err != nil { - return nil, fmt.Errorf("error adding certificate and key: %w", err) + cert, err := addCert(uriDomainsToCerts, certPath, keyPath, logger) + if err != nil { + return nil, nil, fmt.Errorf("error adding certificate and key: %w", err) } + defaultCert = cert } if certStorePaths != "" { @@ -307,7 +311,7 @@ func makeCertGetter(tlsConfig map[string]string, logger logr.Logger) (func(*tls. return nil }) if err != nil { - return nil, fmt.Errorf("error walking certificate store: %w", err) + return nil, nil, fmt.Errorf("error walking certificate store: %w", err) } } @@ -315,10 +319,10 @@ func makeCertGetter(tlsConfig map[string]string, logger logr.Logger) (func(*tls. logger.Info("adding certificate", "certID", certID, "certPath", certPath) keyPath, ok := keyFiles[certID] if !ok { - return nil, fmt.Errorf("no key found for certificate %s", certPath) + return nil, nil, fmt.Errorf("no key found for certificate %s", certPath) } - if err := addCert(uriDomainsToCerts, certPath, keyPath, logger); err != nil { - return nil, fmt.Errorf("error adding certificate %s: %w", certPath, err) + if _, err := addCert(uriDomainsToCerts, certPath, keyPath, logger); err != nil { + return nil, nil, fmt.Errorf("error adding certificate %s: %w", certPath, err) } } } @@ -327,8 +331,11 @@ func makeCertGetter(tlsConfig map[string]string, logger logr.Logger) (func(*tls. if cert, ok := uriDomainsToCerts[hello.ServerName]; ok { return &cert, nil } + if defaultCert != nil { + return defaultCert, nil + } return nil, fmt.Errorf("no certificate found for %s", hello.ServerName) - }, nil + }, maps.Values(uriDomainsToCerts), nil } func runProxyServer( @@ -352,12 +359,13 @@ func runProxyServer( tlsCfg := tls.Config{} if tlsEnabled { - certGetter, err := makeCertGetter(tlsConfig, logger) + certGetter, defaultCert, err := makeCertGetter(tlsConfig, logger) if err != nil { logger.Error(fmt.Errorf("error creating certGetter for proxy server"), "error", err) os.Exit(1) } tlsCfg.GetCertificate = certGetter + tlsCfg.Certificates = defaultCert } var upstreamHandler http.Handler diff --git a/tests/checks/interceptor_multi_tls/interceptor_multi_tls_test.go b/tests/checks/interceptor_multi_tls/interceptor_multi_tls_test.go deleted file mode 100644 index 1a4b34f82..000000000 --- a/tests/checks/interceptor_multi_tls/interceptor_multi_tls_test.go +++ /dev/null @@ -1,207 +0,0 @@ -//go:build e2e -// +build e2e - -package interceptor_tls_test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - . "github.com/kedacore/http-add-on/tests/helper" -) - -const ( - testName = "interceptor-tls-test" -) - -var ( - testNamespace = fmt.Sprintf("%s-ns", testName) - deploymentName = fmt.Sprintf("%s-deployment", testName) - serviceName = fmt.Sprintf("%s-service", testName) - clientName = fmt.Sprintf("%s-client", testName) - httpScaledObjectName = fmt.Sprintf("%s-http-so", testName) - host = testName - minReplicaCount = 0 - maxReplicaCount = 1 -) - -type templateData struct { - TestNamespace string - DeploymentName string - ServiceName string - ClientName string - HTTPScaledObjectName string - Host string - MinReplicas int - MaxReplicas int -} - -const ( - serviceTemplate = ` -apiVersion: v1 -kind: Service -metadata: - name: {{.ServiceName}} - namespace: {{.TestNamespace}} - labels: - app: {{.DeploymentName}} -spec: - ports: - - port: 8080 - targetPort: http - protocol: TCP - name: http - - port: 8443 - targetPort: https - protocol: TCP - name: https - selector: - app: {{.DeploymentName}} -` - - deploymentTemplate = ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{.DeploymentName}} - namespace: {{.TestNamespace}} - labels: - app: {{.DeploymentName}} -spec: - replicas: 0 - selector: - matchLabels: - app: {{.DeploymentName}} - template: - metadata: - labels: - app: {{.DeploymentName}} - spec: - containers: - - name: {{.DeploymentName}} - image: registry.k8s.io/e2e-test-images/agnhost:2.45 - args: - - netexec - - --http-port - - "8443" - - --tls-cert-store-paths - - /certs - ports: - - name: http - containerPort: 8080 - protocol: TCP - - name: https - containerPort: 8443 - protocol: TCP - volumeMounts: - - readOnly: true - mountPath: "/certs/test" - name: certs-test - - readOnly: true - mountPath: "/certs/abc" - name: certs-abc - readinessProbe: - httpGet: - path: / - port: https - scheme: HTTPS - volumes: - - name: certs-test - secret: - secretName: test-tls - - name: certs-abc - secret: - secretName: abc-tls -` - - httpScaledObjectTemplate = ` -kind: HTTPScaledObject -apiVersion: http.keda.sh/v1alpha1 -metadata: - name: {{.HTTPScaledObjectName}} - namespace: {{.TestNamespace}} -spec: - hosts: - - {{.Host}} - targetPendingRequests: 100 - scaledownPeriod: 10 - scaleTargetRef: - name: {{.DeploymentName}} - service: {{.ServiceName}} - port: 8443 - replicas: - min: {{ .MinReplicas }} - max: {{ .MaxReplicas }} -` - - clientTemplate = ` -apiVersion: v1 -kind: Pod -metadata: - name: {{.ClientName}} - namespace: {{.TestNamespace}} -spec: - containers: - - name: {{.ClientName}} - image: curlimages/curl - command: - - sh - - -c - - "exec tail -f /dev/null"` -) - -func TestInterceptorTLS(t *testing.T) { - // setup - t.Log("--- setting up ---") - - // create kubernetes resources - kc := GetKubernetesClient(t) - data, templates := getTemplateData() - CreateKubernetesResources(t, kc, testNamespace, data, templates) - - // setup certs - _, err := ExecuteCommand(fmt.Sprintf("kubectl -n %s create secret tls test-tls --cert ../../../certs/tls.crt --key ../../../certs/tls.key", testNamespace)) - require.NoErrorf(t, err, "could not create tls cert secret in %s namespace - %s", testNamespace, err) - _, err = ExecuteCommand(fmt.Sprintf("kubectl -n %s create secret tls abc-tls --cert ../../../certs/abc.tls.crt --key ../../../certs/abc.tls.key", testNamespace)) - require.NoErrorf(t, err, "could not create tls cert secret in %s namespace - %s", testNamespace, err) - - // wait for test pod to start - assert.True(t, WaitForAllPodRunningInNamespace(t, kc, testNamespace, 10, 2), - "test client count should be available after 20 seconds") - - // send test request and validate response body - sendRequest(t) - - // cleanup - DeleteKubernetesResources(t, testNamespace, data, templates) -} - -func sendRequest(t *testing.T) { - t.Log("--- sending request ---") - - stdout, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, fmt.Sprintf("curl -k --resolve %v:8443:keda-http-add-on-interceptor-proxy.keda https://%v:8443/echo?msg=tls_test", host, host)) - require.NoErrorf(t, err, "could not run command on test client pod - %s", err) - - assert.Equal(t, "tls_test", stdout, fmt.Sprintf("incorrect response body from test request: expected %s, got %s", "tls_test", stdout)) -} - -func getTemplateData() (templateData, []Template) { - return templateData{ - TestNamespace: testNamespace, - DeploymentName: deploymentName, - ServiceName: serviceName, - ClientName: clientName, - HTTPScaledObjectName: httpScaledObjectName, - Host: host, - MinReplicas: minReplicaCount, - MaxReplicas: maxReplicaCount, - }, []Template{ - {Name: "deploymentTemplate", Config: deploymentTemplate}, - {Name: "serviceNameTemplate", Config: serviceTemplate}, - {Name: "clientTemplate", Config: clientTemplate}, - {Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate}, - } -} diff --git a/tests/checks/interceptor_tls/interceptor_tls_test.go b/tests/checks/interceptor_tls/interceptor_tls_test.go index 133791729..eb0a851de 100644 --- a/tests/checks/interceptor_tls/interceptor_tls_test.go +++ b/tests/checks/interceptor_tls/interceptor_tls_test.go @@ -170,14 +170,15 @@ func TestInterceptorTLS(t *testing.T) { sendRequest(t) // cleanup - DeleteKubernetesResources(t, testNamespace, data, templates) + //DeleteKubernetesResources(t, testNamespace, data, templates) } func sendRequest(t *testing.T) { t.Log("--- sending request ---") - stdout, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, fmt.Sprintf("curl -k --resolve %v:8443:keda-http-add-on-interceptor-proxy.keda https://%v:8443/echo?msg=tls_test", host, host)) - require.NoErrorf(t, err, "could not run command on test client pod - %s", err) + cmd := fmt.Sprintf("curl -k -H 'Host: %s' https://keda-http-add-on-interceptor-proxy.keda:8443/echo?msg=tls_test", host) + stdout, stderr, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd) + require.NoErrorf(t, err, "could not run command %q on test client pod - %s\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout, stderr) assert.Equal(t, "tls_test", stdout, fmt.Sprintf("incorrect response body from test request: expected %s, got %s", "tls_test", stdout)) } diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 1b30db30e..e7e52dcc5 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -251,6 +251,9 @@ func TestSetupTLSConfiguration(t *testing.T) { _, err = ExecuteCommand("kubectl -n keda create secret tls keda-tls --cert ../../certs/tls.crt --key ../../certs/tls.key") require.NoErrorf(t, err, "could not create tls cert secret in keda namespace - %s", err) + + _, err = ExecuteCommand("kubectl -n keda create secret tls abc-certs --cert ../../certs/abc.tls.crt --key ../../certs/abc.tls.key") + require.NoErrorf(t, err, "could not create tls cert secret in keda namespace - %s", err) } func TestDeployKEDAHttpAddOn(t *testing.T) {