Add basic bootstrap user support (#1513)

This adds a user kubernetes-controller that functions as a bootstrap user any time SASL auth is enabled.

Some things to note:

1. To use admin_api_require_auth successfully you either have to do a new install or
2. Go through an upgrade to this version of the helm chart and then enable admin_api_require_auth
3. There are some places where we're still sourcing admin credentials via reading from a mounted users file. I haven't fixed that up yet, but can do so in the future to unify around leveraging the same user for internal management of the cluster.

.gitignore

@@ -22,6 +22,8 @@ charts/redpanda/ci/21-eks-tiered-storage-with-creds-values.yaml
 # Dep Chart Lock file
 Chart.lock
 
+charts/redpanda/local.yaml
+
 charts/redpanda/templates/redpanda-license.yaml
 charts/redpanda/templates/external-service.yaml
 charts/redpanda/templates/external-tls-secret.yaml

charts/redpanda/README.md

@@ -110,9 +110,25 @@ Authentication settings. For details, see the [SASL documentation](https://docs.
 **Default:**
 
 ```
-{"sasl":{"enabled":false,"mechanism":"SCRAM-SHA-512","secretRef":"redpanda-users","users":[]}}
+{"sasl":{"bootstrapUser":{"mechanism":"SCRAM-SHA-256"},"enabled":false,"mechanism":"SCRAM-SHA-512","secretRef":"redpanda-users","users":[]}}
 ```
 
+### [auth.sasl.bootstrapUser](https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=auth.sasl.bootstrapUser)
+
+Details about how to create the bootstrap user for the cluster. The secretKeyRef is optionally specified. If it is specified, the chart will use a password written to that secret when creating the "kubernetes-controller" bootstrap user. If it is unspecified, then the secret will be generated and stored in the secret "releasename"-bootstrap-user, with the key "password".
+
+**Default:**
+
+```
+{"mechanism":"SCRAM-SHA-256"}
+```
+
+### [auth.sasl.bootstrapUser.mechanism](https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=auth.sasl.bootstrapUser.mechanism)
+
+The authentication mechanism to use for the bootstrap user. Options are `SCRAM-SHA-256` and `SCRAM-SHA-512`.
+
+**Default:** `"SCRAM-SHA-256"`
+
 ### [auth.sasl.enabled](https://artifacthub.io/packages/helm/redpanda-data/redpanda?modal=values&path=auth.sasl.enabled)
 
 Enable SASL authentication. If you enable SASL authentication, you must provide a Secret in `auth.sasl.secretRef`.

charts/redpanda/chart_template_test.go

@@ -71,6 +71,8 @@ func TestTemplate(t *testing.T) {
 					// jwtSecret defaults to a random string. Can't have that
 					// in snapshot testing so set it to a static value.
 					"console.secret.login.jwtSecret=SECRETKEY",
+					// the bootstrap user password has the same issues as jwtSecret
+					"auth.sasl.bootstrapUser.password=changeme",
 				},
 			})
 

charts/redpanda/chart_test.go

@@ -155,6 +155,61 @@ func TestChart(t *testing.T) {
 		assert.JSONEq(t, string(schemaBytes), retrievedSchema)
 		assert.NoErrorf(t, httpProxyListenerTest(ctx, rpk), "HTTP Proxy listener sub test failed")
 	})
+
+	t.Run("admin api auth required", func(t *testing.T) {
+		ctx := testutil.Context(t)
+
+		env := h.Namespaced(t)
+
+		partial := redpanda.PartialValues{
+			External:      &redpanda.PartialExternalConfig{Enabled: ptr.To(false)},
+			ClusterDomain: ptr.To("cluster.local"),
+			Config: &redpanda.PartialConfig{
+				Cluster: redpanda.PartialClusterConfig{
+					"admin_api_require_auth": true,
+				},
+			},
+			Auth: &redpanda.PartialAuth{
+				SASL: &redpanda.PartialSASLAuth{
+					Enabled: ptr.To(true),
+					Users: []redpanda.PartialSASLUser{{
+						Name:      ptr.To("superuser"),
+						Password:  ptr.To("superpassword"),
+						Mechanism: ptr.To("SCRAM-SHA-512"),
+					}},
+				},
+			},
+		}
+
+		r, err := rand.Int(rand.Reader, new(big.Int).SetInt64(1799999999))
+		require.NoError(t, err)
+
+		chartReleaseName := fmt.Sprintf("chart-%d", r.Int64())
+		rpRelease := env.Install(ctx, redpandaChart, helm.InstallOptions{
+			Values:    partial,
+			Name:      chartReleaseName,
+			Namespace: env.Namespace(),
+		})
+
+		rpk := Client{Ctl: env.Ctl(), Release: &rpRelease}
+
+		dot := &helmette.Dot{
+			Values:  *helmette.UnmarshalInto[*helmette.Values](partial),
+			Release: helmette.Release{Name: rpRelease.Name, Namespace: rpRelease.Namespace},
+			Chart: helmette.Chart{
+				Name: "redpanda",
+			},
+		}
+
+		cleanup, err := rpk.ExposeRedpandaCluster(ctx, dot, w, wErr)
+		if cleanup != nil {
+			t.Cleanup(cleanup)
+		}
+		require.NoError(t, err)
+
+		assert.NoErrorf(t, kafkaListenerTest(ctx, rpk), "Kafka listener sub test failed")
+		assert.NoErrorf(t, adminListenerTest(ctx, rpk), "Admin listener sub test failed")
+	})
 }
 
 func TieredStorageStatic(t *testing.T) redpanda.PartialValues {

charts/redpanda/post_install_upgrade_job.go

@@ -82,7 +82,7 @@ func PostInstallUpgradeJob(dot *helmette.Dot) *batchv1.Job {
 						{
 							Name:      PostInstallContainerName,
 							Image:     fmt.Sprintf("%s:%s", values.Image.Repository, Tag(dot)),
-							Env:       PostInstallUpgradeEnvironmentVariables(dot),
+							Env:       rpkEnvVars(dot, PostInstallUpgradeEnvironmentVariables(dot)),
 							Command:   []string{"bash", "-c"},
 							Args:      []string{},
 							Resources: ptr.Deref(values.PostInstallJob.Resources, corev1.ResourceRequirements{}),
@@ -138,16 +138,18 @@ func PostInstallUpgradeJob(dot *helmette.Dot) *batchv1.Job {
 
 		// Second: For each environment variable with the prefix RPK
 		// ("${!RPK_@}"), use `rpk redpanda config set` to update the exported
-		// config.
+		// config, ignoring any authentication environment variables.
 
 		// Lots of Bash Jargon here:
 		//     "${KEY#*RPK_}" => Strip the RPK_ prefix from KEY.
 		//     "${config,,}" => config.toLower()
 		//     "${!KEY}" => Dynamic variable resolution. ie: What is the value of the variable with a name equal to the value of $KEY?
 
 		`for KEY in "${!RPK_@}"; do`,
-		`  config="${KEY#*RPK_}"`,
-		`  rpk redpanda config set --config /tmp/cfg.yml "${config,,}" "${!KEY}"`,
+		`  if ! [[ "$KEY" =~ ^(RPK_USER|RPK_PASS|RPK_SASL_MECHANISM)$ ]]; then`,
+		`    config="${KEY#*RPK_}"`,
+		`    rpk redpanda config set --config /tmp/cfg.yml "${config,,}" "${!KEY}"`,
+		`  fi`,
 		`done`,
 		``, ``,
 
@@ -232,7 +234,7 @@ func PostInstallUpgradeEnvironmentVariables(dot *helmette.Dot) []corev1.EnvVar {
 
 		// cloud_storage_cache_size can be represented as Resource.Quantity that why value can be converted
 		// from value with SI suffix to bytes number.
-		if k == "cloud_storage_cache_size" && v != nil {
+		if k == "cloud_storage_cache_size" {
 			envars = append(envars, corev1.EnvVar{
 				Name:  fmt.Sprintf("RPK_%s", helmette.Upper(k)),
 				Value: helmette.ToJSON(helmette.UnmarshalInto[*resource.Quantity](v).Value()),

charts/redpanda/post_upgrade_job.go

@@ -62,7 +62,7 @@ func PostUpgrade(dot *helmette.Dot) *batchv1.Job {
 							Image:   fmt.Sprintf("%s:%s", values.Image.Repository, Tag(dot)),
 							Command: []string{"/bin/bash", "-c"},
 							Args:    []string{PostUpgradeJobScript(dot)},
-							Env:     values.PostUpgradeJob.ExtraEnv,
+							Env:     rpkEnvVars(dot, values.PostUpgradeJob.ExtraEnv),
 							EnvFrom: values.PostUpgradeJob.ExtraEnvFrom,
 							SecurityContext: ptr.To(helmette.MergeTo[corev1.SecurityContext](
 								ptr.Deref(values.PostUpgradeJob.SecurityContext, corev1.SecurityContext{}),

charts/redpanda/secrets.go

@@ -39,6 +39,9 @@ func Secrets(dot *helmette.Dot) []*corev1.Secret {
 	if fsValidator := SecretFSValidator(dot); fsValidator != nil {
 		secrets = append(secrets, fsValidator)
 	}
+	if bootstrapUser := SecretBootstrapUser(dot); bootstrapUser != nil {
+		secrets = append(secrets, bootstrapUser)
+	}
 	return secrets
 }
 
@@ -103,7 +106,7 @@ func SecretSTSLifecycle(dot *helmette.Dot) *corev1.Secret {
 			`  # Setup and export SASL bootstrap-user`,
 			`  IFS=":" read -r USER_NAME PASSWORD MECHANISM < <(grep "" $(find /etc/secrets/users/* -print))`,
 			fmt.Sprintf(`  MECHANISM=${MECHANISM:-%s}`, helmette.Dig(dot.Values.AsMap(), "SCRAM-SHA-512", "auth", "sasl", "mechanism")),
-			`  rpk acl user create ${USER_NAME} --password=${PASSWORD} --mechanism ${MECHANISM} || true`,
+			`  rpk acl user create ${USER_NAME} -p {PASSWORD} --mechanism ${MECHANISM} || true`,
 		)
 	}
 	postStartSh = append(postStartSh,
@@ -204,6 +207,44 @@ func SecretSASLUsers(dot *helmette.Dot) *corev1.Secret {
 	}
 }
 
+func SecretBootstrapUser(dot *helmette.Dot) *corev1.Secret {
+	values := helmette.Unwrap[Values](dot.Values)
+	if !values.Auth.SASL.Enabled || values.Auth.SASL.BootstrapUser.SecretKeyRef != nil {
+		return nil
+	}
+
+	secretName := fmt.Sprintf("%s-bootstrap-user", Fullname(dot))
+
+	if dot.Release.IsUpgrade {
+		if existing, ok := helmette.Lookup[corev1.Secret](dot, dot.Release.Namespace, secretName); ok {
+			return existing
+		}
+	}
+
+	password := helmette.RandAlphaNum(32)
+
+	userPassword := values.Auth.SASL.BootstrapUser.Password
+	if userPassword != nil {
+		password = *userPassword
+	}
+
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: dot.Release.Namespace,
+			Labels:    FullLabels(dot),
+		},
+		Type: corev1.SecretTypeOpaque,
+		StringData: map[string]string{
+			"password": password,
+		},
+	}
+}
+
 func SecretConfigWatcher(dot *helmette.Dot) *corev1.Secret {
 	values := helmette.Unwrap[Values](dot.Values)
 
@@ -269,7 +310,7 @@ func SecretConfigWatcher(dot *helmette.Dot) *corev1.Secret {
 			`  process_users() {`,
 			`    USERS_DIR=${1-"/etc/secrets/users"}`,
 			`    USERS_FILE=$(find ${USERS_DIR}/* -print)`,
-			`    USERS_LIST=""`,
+			`    USERS_LIST="kubernetes-controller"`,
 			`    READ_LIST_SUCCESS=0`,
 			`    # Read line by line, handle a missing EOL at the end of file`,
 			`    while read p || [ -n "$p" ] ; do`,
@@ -283,7 +324,7 @@ func SecretConfigWatcher(dot *helmette.Dot) *corev1.Secret {
 			`      fi`,
 			`      echo "Creating user ${USER_NAME}..."`,
 			fmt.Sprintf(`      MECHANISM=${MECHANISM:-%s}`, helmette.Dig(dot.Values.AsMap(), "SCRAM-SHA-512", "auth", "sasl", "mechanism")),
-			`      creation_result=$(rpk acl user create ${USER_NAME} --password=${PASSWORD} --mechanism ${MECHANISM} 2>&1) && creation_result_exit_code=$? || creation_result_exit_code=$?  # On a non-success exit code`,
+			`      creation_result=$(rpk acl user create ${USER_NAME} -p ${PASSWORD} --mechanism ${MECHANISM} 2>&1) && creation_result_exit_code=$? || creation_result_exit_code=$?  # On a non-success exit code`,
 			`      if [[ $creation_result_exit_code -ne 0 ]]; then`,
 			`        # Check if the stderr contains "User already exists"`,
 			`        # this error occurs when password has changed`,
@@ -297,7 +338,7 @@ func SecretConfigWatcher(dot *helmette.Dot) *corev1.Secret {
 			`            break`,
 			`          fi`,
 			`          # Now we update the user`,
-			`          update_result=$(rpk acl user create ${USER_NAME} --password=${PASSWORD} --mechanism ${MECHANISM} 2>&1) && update_result_exit_code=$? || update_result_exit_code=$?  # On a non-success exit code`,
+			`          update_result=$(rpk acl user create ${USER_NAME} -p ${PASSWORD} --mechanism ${MECHANISM} 2>&1) && update_result_exit_code=$? || update_result_exit_code=$?  # On a non-success exit code`,
 			`          if [[ $update_result_exit_code -ne 0 ]]; then`,
 			`            echo "updating user ${USER_NAME} failed: ${update_result}"`,
 			`            READ_LIST_SUCCESS=1`,
@@ -330,6 +371,17 @@ func SecretConfigWatcher(dot *helmette.Dot) *corev1.Secret {
 			`    fi`,
 			`  }`,
 			``,
+			`  # before we do anything ensure we have the bootstrap user`,
+			`  echo "Ensuring bootstrap user ${RPK_USER}..."`,
+			`  creation_result=$(rpk acl user create ${RPK_USER} -p ${RPK_PASS} --mechanism ${RPK_SASL_MECHANISM} 2>&1) && creation_result_exit_code=$? || creation_result_exit_code=$?  # On a non-success exit code`,
+			`  if [[ $creation_result_exit_code -ne 0 ]]; then`,
+			`    if [[ $creation_result == *"User already exists"* ]]; then`,
+			`      echo "Bootstrap user already created"`,
+			`    else`,
+			`      echo "error creating user ${RPK_USER}: ${creation_result}"`,
+			`    fi`,
+			`  fi`,
+			``,
 			`  # first time processing`,
 			`  process_users $USERS_DIR`,
 			``,

charts/redpanda/statefulset.go

@@ -488,7 +488,7 @@ func statefulSetInitContainerConfigurator(dot *helmette.Dot) *corev1.Container {
 			`-c`,
 			`trap "exit 0" TERM; exec $CONFIGURATOR_SCRIPT "${SERVICE_NAME}" "${KUBERNETES_NODE_NAME}" & wait $!`,
 		},
-		Env: []corev1.EnvVar{
+		Env: rpkEnvVars(dot, []corev1.EnvVar{
 			{
 				Name:  "CONFIGURATOR_SCRIPT",
 				Value: "/etc/secrets/configurator/scripts/configurator.sh",
@@ -521,7 +521,7 @@ func statefulSetInitContainerConfigurator(dot *helmette.Dot) *corev1.Container {
 					},
 				},
 			},
-		},
+		}),
 		SecurityContext: ptr.To(ContainerSecurityContext(dot)),
 		VolumeMounts: append(append(CommonMounts(dot),
 			templateToVolumeMounts(dot, values.Statefulset.InitContainers.Configurator.ExtraVolumeMounts)...),
@@ -562,7 +562,7 @@ func statefulSetContainerRedpanda(dot *helmette.Dot) *corev1.Container {
 	container := &corev1.Container{
 		Name:  Name(dot),
 		Image: fmt.Sprintf(`%s:%s`, values.Image.Repository, Tag(dot)),
-		Env:   statefulSetRedpandaEnv(),
+		Env:   bootstrapEnvVars(dot, statefulSetRedpandaEnv()),
 		Lifecycle: &corev1.Lifecycle{
 			// finish the lifecycle scripts with "true" to prevent them from terminating the pod prematurely
 			PostStart: &corev1.LifecycleHandler{
@@ -799,6 +799,7 @@ func statefulSetContainerConfigWatcher(dot *helmette.Dot) *corev1.Container {
 			`-c`,
 			`trap "exit 0" TERM; exec /etc/secrets/config-watcher/scripts/sasl-user.sh & wait $!`,
 		},
+		Env:             rpkEnvVars(dot, nil),
 		Resources:       helmette.UnmarshalInto[corev1.ResourceRequirements](values.Statefulset.SideCars.ConfigWatcher.Resources),
 		SecurityContext: values.Statefulset.SideCars.ConfigWatcher.SecurityContext,
 		VolumeMounts: append(
@@ -855,6 +856,22 @@ func statefulSetContainerControllers(dot *helmette.Dot) *corev1.Container {
 	}
 }
 
+func rpkEnvVars(dot *helmette.Dot, envVars []corev1.EnvVar) []corev1.EnvVar {
+	values := helmette.Unwrap[Values](dot.Values)
+	if values.Auth.SASL != nil && values.Auth.SASL.Enabled {
+		return append(envVars, values.Auth.SASL.BootstrapUser.RpkEnvironment(Fullname(dot))...)
+	}
+	return envVars
+}
+
+func bootstrapEnvVars(dot *helmette.Dot, envVars []corev1.EnvVar) []corev1.EnvVar {
+	values := helmette.Unwrap[Values](dot.Values)
+	if values.Auth.SASL != nil && values.Auth.SASL.Enabled {
+		return append(envVars, values.Auth.SASL.BootstrapUser.BootstrapEnvironment(Fullname(dot))...)
+	}
+	return envVars
+}
+
 func templateToVolumeMounts(dot *helmette.Dot, template string) []corev1.VolumeMount {
 	result := helmette.Tpl(template, dot)
 	return helmette.UnmarshalYamlArray[corev1.VolumeMount](result)