Skip to content

Commit

Permalink
Disable TLS
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgrandl
rgrandl committed Jun 6, 2024
1 parent 73d0819 commit d7386c6
Show file tree
Hide file tree
Showing 9 changed files with 188 additions and 90 deletions.
7 changes: 4 additions & 3 deletions cmd/weaver-gke/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (
)

var (
controllerFlags = flag.NewFlagSet("controller", flag.ContinueOnError)
controllerPort = controllerFlags.Int("port", 0, "Controller port")
controllerFlags = flag.NewFlagSet("controller", flag.ContinueOnError)
controllerPort = controllerFlags.Int("port", 0, "Controller port")
controllerMtlsEnabled = controllerFlags.Bool("mtls", false, "Whether controller uses MTLS")
)

var controllerCmd = tool.Command{
Expand All @@ -37,7 +38,7 @@ var controllerCmd = tool.Command{
Flags:
-h, --help Print this help message.`,
Fn: func(ctx context.Context, args []string) error {
return gke.RunController(ctx, *controllerPort)
return gke.RunController(ctx, *controllerPort, *controllerMtlsEnabled)
},
Hidden: true,
}
7 changes: 4 additions & 3 deletions cmd/weaver-gke/distributor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (
)

var (
distributorFlags = flag.NewFlagSet("distributor", flag.ContinueOnError)
distributorPort = distributorFlags.Int("port", 0, "Distributor port")
distributorFlags = flag.NewFlagSet("distributor", flag.ContinueOnError)
distributorPort = distributorFlags.Int("port", 0, "Distributor port")
distributorMtlsEnabled = distributorFlags.Bool("mtls", false, "Whether distributor uses MTLS")
)

var distributorCmd = tool.Command{
Expand All @@ -37,7 +38,7 @@ var distributorCmd = tool.Command{
Flags:
-h, --help Print this help message.`,
Fn: func(ctx context.Context, args []string) error {
return gke.RunDistributor(ctx, *distributorPort)
return gke.RunDistributor(ctx, *distributorPort, *distributorMtlsEnabled)
},
Hidden: true,
}
7 changes: 4 additions & 3 deletions cmd/weaver-gke/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (
)

var (
managerFlags = flag.NewFlagSet("manager", flag.ContinueOnError)
managerPort = managerFlags.Int("port", 0, "Manager port")
managerFlags = flag.NewFlagSet("manager", flag.ContinueOnError)
managerPort = managerFlags.Int("port", 0, "Manager port")
managerMtlsEnabled = managerFlags.Bool("mtls", false, "Whether manager uses MTLS")
)

var managerCmd = tool.Command{
Expand All @@ -37,7 +38,7 @@ var managerCmd = tool.Command{
Flags:
-h, --help Print this help message.`,
Fn: func(ctx context.Context, args []string) error {
return gke.RunManager(ctx, *managerPort)
return gke.RunManager(ctx, *managerPort, *managerMtlsEnabled)
},
Hidden: true,
}
34 changes: 26 additions & 8 deletions internal/gke/babysitter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ package gke

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"log/slog"
"net"
Expand Down Expand Up @@ -137,16 +139,24 @@ func RunBabysitter(ctx context.Context) error {
return metricsExporter.Export(ctx, metrics, cfg.Telemetry.Metrics.AutoGenerateMetrics)
}

caCert, getSelfCert, err := getPodCerts()
if err != nil {
return err
var caCert *x509.Certificate
var getSelfCert func() ([]byte, []byte, error)
if meta.MtlsEnabled {
caCert, getSelfCert, err = getPodCerts()
if err != nil {
return err
}
}

// Create an unique http client to the manager, that will be reused across all
// the http requests to the manager.
var tlsConfig *tls.Config
if meta.MtlsEnabled {
tlsConfig = mtls.ClientTLSConfig(meta.Project, caCert, getSelfCert, "manager")
}
m := &manager.HttpClient{
Addr: cfg.ManagerAddr,
Client: makeHttpClient(mtls.ClientTLSConfig(meta.Project, caCert, getSelfCert, "manager")),
Client: makeHttpClient(tlsConfig),
}
mux := http.NewServeMux()
host, err := os.Hostname()
Expand All @@ -158,17 +168,25 @@ func RunBabysitter(ctx context.Context) error {
if err != nil {
return err
}
selfAddr := fmt.Sprintf("https://%s", lis.Addr())
var selfAddr string
if meta.MtlsEnabled {
selfAddr = fmt.Sprintf("https://%s", lis.Addr())
} else {
selfAddr = fmt.Sprintf("http://%s", lis.Addr())
}
_, err = babysitter.Start(ctx, logger, cfg, replicaSet, meta.Project, meta.PodName, internalAddress, mux, selfAddr, m, caCert, getSelfCert, logSaver, traceSaver, metricSaver)
if err != nil {
return err
}

server := &http.Server{
Handler: mux,
TLSConfig: mtls.ServerTLSConfig(meta.Project, caCert, getSelfCert, "manager", "distributor"),
Handler: mux,
}
if meta.MtlsEnabled {
server.TLSConfig = mtls.ServerTLSConfig(meta.Project, caCert, getSelfCert, "manager", "distributor")
return server.ServeTLS(lis, "", "")
}
return server.ServeTLS(lis, "", "")
return server.Serve(lis)
}

// gkeConfigFromEnv reads config.GKEConfig from the Service Weaver internal
Expand Down
21 changes: 16 additions & 5 deletions internal/gke/container.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions internal/gke/container.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,5 @@ message ContainerMetadata {
string container_name = 7; // Kubernetes container name
string app = 8; // Kubernetes app label
config.Telemetry telemetry = 9; // Options to configure the telemetry
bool mtls_enabled = 10; // Whether MTLS should be enabled
}
78 changes: 47 additions & 31 deletions internal/gke/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,10 @@ func prepareProject(ctx context.Context, config CloudConfig, cfg *config.GKEConf
}

// Setup the Certificate Authority.
if err := ensureCA(ctx, config); err != nil {
return nil, "", err
if cfg.Mtls {
if err := ensureCA(ctx, config); err != nil {
return nil, "", err
}
}

// Ensure the Service Weaver configuration cluster is setup.
Expand Down Expand Up @@ -481,8 +483,14 @@ func buildRolloutRequest(cfg *config.GKEConfig) *controller.RolloutRequest {
for _, region := range cfg.Regions {
// NOTE: distributor address must be resolvable from anywhere inside
// the project's VPC.
distributorAddr :=
fmt.Sprintf("https://distributor.%s.svc.%s-%s:80", namespaceName, applicationClusterName, region)
var distributorAddr string
if cfg.Mtls {
distributorAddr =
fmt.Sprintf("https://distributor.%s.svc.%s-%s:80", namespaceName, applicationClusterName, region)
} else {
distributorAddr =
fmt.Sprintf("http://distributor.%s.svc.%s-%s:80", namespaceName, applicationClusterName, region)
}
req.Locations = append(req.Locations, &controller.RolloutRequest_Location{
Name: region,
DistributorAddr: distributorAddr,
Expand Down Expand Up @@ -1042,7 +1050,7 @@ func ensureConfigCluster(ctx context.Context, config CloudConfig, cfg *config.GK
if err != nil {
return nil, "", err
}
cluster, err := ensureManagedCluster(ctx, config, name, region)
cluster, err := ensureManagedCluster(ctx, config, name, region, cfg.Mtls)
if err != nil {
return nil, "", err
}
Expand Down Expand Up @@ -1110,7 +1118,7 @@ func ensureConfigCluster(ctx context.Context, config CloudConfig, cfg *config.GK
// It returns the cluster information and the IP address of the gateway that
// routes internal traffic to Service Weaver applications in the cluster.
func ensureApplicationCluster(ctx context.Context, config CloudConfig, cfg *config.GKEConfig, region string) (*ClusterInfo, string, error) {
cluster, err := ensureManagedCluster(ctx, config, applicationClusterName, region)
cluster, err := ensureManagedCluster(ctx, config, applicationClusterName, region, cfg.Mtls)
if err != nil {
return nil, "", err
}
Expand Down Expand Up @@ -1254,7 +1262,7 @@ func ensureApplicationCluster(ctx context.Context, config CloudConfig, cfg *conf

// ensureManagedCluster ensures that a Service Weaver managed cluster is available
// and running in the given region.
func ensureManagedCluster(ctx context.Context, config CloudConfig, name, region string) (*ClusterInfo, error) {
func ensureManagedCluster(ctx context.Context, config CloudConfig, name, region string, mtlsEnabled bool) (*ClusterInfo, error) {
exists, err := hasCluster(ctx, config, name, region)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1380,14 +1388,16 @@ func ensureManagedCluster(ctx context.Context, config CloudConfig, name, region
return nil, err
}

// Setup the workload certificate config in the cluster.
if err := ensureWorkloadCertificateConfig(ctx, cluster); err != nil {
return nil, err
}
if mtlsEnabled {
// Setup the workload certificate config in the cluster.
if err := ensureWorkloadCertificateConfig(ctx, cluster); err != nil {
return nil, err
}

// Setup the trust config in the cluster.
if err := ensureTrustConfig(ctx, cluster); err != nil {
return nil, err
// Setup the trust config in the cluster.
if err := ensureTrustConfig(ctx, cluster); err != nil {
return nil, err
}
}

// Scale down resources used by system services.
Expand Down Expand Up @@ -1848,18 +1858,18 @@ func ensureWeaverServices(ctx context.Context, config CloudConfig, cfg *config.G
if err != nil {
return err
}
if err := ensureController(ctx, config, name, region, cfg.Telemetry, toolImageURL); err != nil {
if err := ensureController(ctx, config, name, region, cfg.Telemetry, cfg.Mtls, toolImageURL); err != nil {
return err
}
for _, region := range cfg.Regions {
cluster, err := GetClusterInfo(ctx, config, applicationClusterName, region)
if err != nil {
return err
}
if err := ensureDistributor(ctx, cluster, cfg.Telemetry, toolImageURL); err != nil {
if err := ensureDistributor(ctx, cluster, cfg.Telemetry, cfg.Mtls, toolImageURL); err != nil {
return err
}
if err := ensureManager(ctx, cluster, cfg.Telemetry, toolImageURL); err != nil {
if err := ensureManager(ctx, cluster, cfg.Telemetry, cfg.Mtls, toolImageURL); err != nil {
return err
}
}
Expand All @@ -1868,13 +1878,13 @@ func ensureWeaverServices(ctx context.Context, config CloudConfig, cfg *config.G

// ensureController ensures that a controller is running in the config cluster.
func ensureController(ctx context.Context, config CloudConfig, clusterName, region string,
telemetry *config.Telemetry, toolImageURL string) error {
telemetry *config.Telemetry, mtlsEnabled bool, toolImageURL string) error {
cluster, err := GetClusterInfo(ctx, config, clusterName, region)
if err != nil {
return err
}
const name = "controller"
if err := ensureNannyDeployment(ctx, cluster, name, controllerKubeServiceAccount, telemetry, toolImageURL); err != nil {
if err := ensureNannyDeployment(ctx, cluster, name, controllerKubeServiceAccount, telemetry, mtlsEnabled, toolImageURL); err != nil {
return err
}
if err := ensureNannyVerticalPodAutoscaler(ctx, cluster, name); err != nil {
Expand All @@ -1885,9 +1895,9 @@ func ensureController(ctx context.Context, config CloudConfig, clusterName, regi

// ensureDistributor ensures that a distributor is running in the given cluster.
func ensureDistributor(ctx context.Context, cluster *ClusterInfo,
telemetry *config.Telemetry, toolImageURL string) error {
telemetry *config.Telemetry, mtlsEnabled bool, toolImageURL string) error {
const name = "distributor"
if err := ensureNannyDeployment(ctx, cluster, name, distributorKubeServiceAccount, telemetry, toolImageURL); err != nil {
if err := ensureNannyDeployment(ctx, cluster, name, distributorKubeServiceAccount, telemetry, mtlsEnabled, toolImageURL); err != nil {
return err
}
if err := ensureNannyVerticalPodAutoscaler(ctx, cluster, name); err != nil {
Expand All @@ -1898,9 +1908,9 @@ func ensureDistributor(ctx context.Context, cluster *ClusterInfo,

// ensureManager ensures that a manager is running in the given cluster.
func ensureManager(ctx context.Context, cluster *ClusterInfo,
telemetry *config.Telemetry, toolImageURL string) error {
telemetry *config.Telemetry, mtlsEnabled bool, toolImageURL string) error {
const name = "manager"
if err := ensureNannyDeployment(ctx, cluster, name, managerKubeServiceAccount, telemetry, toolImageURL); err != nil {
if err := ensureNannyDeployment(ctx, cluster, name, managerKubeServiceAccount, telemetry, mtlsEnabled, toolImageURL); err != nil {
return err
}
if err := ensureNannyVerticalPodAutoscaler(ctx, cluster, name); err != nil {
Expand All @@ -1912,7 +1922,7 @@ func ensureManager(ctx context.Context, cluster *ClusterInfo,
// ensureNannyDeployment ensures that a nanny deployment with the given name
// and service account is running in the given cluster.
func ensureNannyDeployment(ctx context.Context, cluster *ClusterInfo, name string,
serviceAccount string, telemetry *config.Telemetry, toolImageURL string) error {
serviceAccount string, telemetry *config.Telemetry, mtlsEnabled bool, toolImageURL string) error {
meta := ContainerMetadata{
Project: cluster.CloudConfig.Project,
ClusterName: cluster.Name,
Expand All @@ -1921,6 +1931,7 @@ func ensureNannyDeployment(ctx context.Context, cluster *ClusterInfo, name strin
ContainerName: nannyContainerName,
App: name,
Telemetry: telemetry,
MtlsEnabled: mtlsEnabled,
}
metaStr, err := proto.ToEnv(&meta)
if err != nil {
Expand Down Expand Up @@ -1950,6 +1961,16 @@ func ensureNannyDeployment(ctx context.Context, cluster *ClusterInfo, name strin
}
return oldTag < newTag, nil
}

objectMetaTmpl := metav1.ObjectMeta{
Labels: map[string]string{"app": name},
}
if mtlsEnabled {
objectMetaTmpl.Annotations = map[string]string{
"security.cloud.google.com/use-workload-certificates": "",
}
}

return patchDeployment(ctx, cluster, patchOptions{}, shouldUpdate, &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -1960,20 +1981,15 @@ func ensureNannyDeployment(ctx context.Context, cluster *ClusterInfo, name strin
MatchLabels: map[string]string{"app": name},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"app": name},
Annotations: map[string]string{
"security.cloud.google.com/use-workload-certificates": "",
},
},
ObjectMeta: objectMetaTmpl,
Spec: v1.PodSpec{
PriorityClassName: controlPriorityClassName,
Containers: []v1.Container{
{
Name: name,
Image: toolImageURL,
Args: []string{
fmt.Sprintf("/weaver/weaver-gke %s --port=%d", name, nannyServingPort),
fmt.Sprintf("/weaver/weaver-gke %s --port=%d --mtls=%v", name, nannyServingPort, mtlsEnabled),
},
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
Expand Down
Loading

0 comments on commit d7386c6

Please sign in to comment.