115 privateimagehubs dockerfile (#998)

* use additionalDockerAuth for build using privateimagehubs credentials * rename docker auth struct * grant pipeline runner access to privateimagehub secret * update charts * mount and use private image hub auth with buildah --------- Co-authored-by: Nils Gustav Stråbø <[email protected]>
equinor · Dec 5, 2023 · 263454d · 263454d
1 parent cdbabd9
commit 263454d
Show file tree

Hide file tree

Showing 10 changed files with 219 additions and 146 deletions.
diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: radix-operator
-version: 1.25.9
-appVersion: 1.45.9
+version: 1.26.0
+appVersion: 1.46.0
 kubeVersion: ">=1.24.0"
 description: Radix Operator
 keywords:

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	dario.cat/mergo v1.0.0
-	github.com/equinor/radix-common v1.5.0
+	github.com/equinor/radix-common v1.6.2
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.3.0
 	github.com/pkg/errors v0.9.1

diff --git a/go.sum b/go.sum
@@ -72,8 +72,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/equinor/radix-common v1.5.0 h1:z5hQHlKG2x16/NnV4b9ynf9n5ZageYUewE4MANdA96Y=
-github.com/equinor/radix-common v1.5.0/go.mod h1:UZ69U56VFtTxABi5JjGdaqn9Df5ilfTTqzUQ0riofVM=
+github.com/equinor/radix-common v1.6.2 h1:856A/XcdDwzejmHbbuuWra/xFB6iu8uksEAme/M1v7c=
+github.com/equinor/radix-common v1.6.2/go.mod h1:9hHvudaiqmoIjCqKlsW14jMj8qU/b/wMXUwkffd9MUw=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=

diff --git a/pipeline-runner/steps/build_acr.go b/pipeline-runner/steps/build_acr.go
@@ -3,6 +3,7 @@ package steps
 import (
 	"encoding/json"
 	"fmt"
+	"path"
 	"strings"
 	"time"
 
@@ -22,7 +23,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const buildSecretsMountPath = "/build-secrets"
+const (
+	buildSecretsMountPath        = "/build-secrets"
+	privateImageHubMountPath     = "/radix-private-image-hubs"
+	buildahRegistryAuthFile      = "/home/build/auth.json"
+	azureServicePrincipleContext = "/radix-image-builder/.azure"
+)
 
 type void struct{}
 
@@ -33,14 +39,13 @@ func createACRBuildJob(rr *v1.RadixRegistration, pipelineInfo *model.PipelineInf
 	branch := pipelineInfo.PipelineArguments.Branch
 	imageTag := pipelineInfo.PipelineArguments.ImageTag
 	jobName := pipelineInfo.PipelineArguments.JobName
-
 	initContainers := git.CloneInitContainers(rr.Spec.CloneURL, branch, pipelineInfo.PipelineArguments.ContainerSecurityContext)
 	buildContainers := createACRBuildContainers(appName, pipelineInfo, buildSecrets)
 	timestamp := time.Now().Format("20060102150405")
 	defaultMode, backOffLimit := int32(256), int32(0)
-
 	componentImagesAnnotation, _ := json.Marshal(pipelineInfo.BuildComponentImages)
 	hash := strings.ToLower(utils.RandStringStrSeed(5, pipelineInfo.PipelineArguments.JobName))
+
 	annotations := radixannotations.ForClusterAutoscalerSafeToEvict(false)
 	buildPodSecurityContext := &pipelineInfo.PipelineArguments.PodSecurityContext
 	if isUsingBuildKit(pipelineInfo) {
@@ -109,6 +114,14 @@ func getACRBuildJobVolumes(defaultMode *int32, buildSecrets []corev1.EnvVar) []c
 				},
 			},
 		},
+		{
+			Name: defaults.PrivateImageHubSecretName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: defaults.PrivateImageHubSecretName,
+				},
+			},
+		},
 	}
 
 	if len(buildSecrets) > 0 {
@@ -127,40 +140,36 @@ func getACRBuildJobVolumes(defaultMode *int32, buildSecrets []corev1.EnvVar) []c
 }
 
 func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo, buildSecrets []corev1.EnvVar) []corev1.Container {
+	var containers []corev1.Container
 	imageTag := pipelineInfo.PipelineArguments.ImageTag
 	pushImage := pipelineInfo.PipelineArguments.PushImage
-	buildContainerSecContext := &pipelineInfo.PipelineArguments.ContainerSecurityContext
-	var containerCommand []string
-
 	clusterType := pipelineInfo.PipelineArguments.Clustertype
 	clusterName := pipelineInfo.PipelineArguments.Clustername
 	containerRegistry := pipelineInfo.PipelineArguments.ContainerRegistry
-	imageBuilder := fmt.Sprintf("%s/%s", containerRegistry, pipelineInfo.PipelineArguments.ImageBuilder)
 	subscriptionId := pipelineInfo.PipelineArguments.SubscriptionId
 	branch := pipelineInfo.PipelineArguments.Branch
 	targetEnvs := strings.Join(pipelineInfo.TargetEnvironments, ",")
-	secretMountsArgsString := ""
+	firstPartContainerRegistry := strings.Split(containerRegistry, ".")[0]
 
+	imageBuilder := fmt.Sprintf("%s/%s", containerRegistry, pipelineInfo.PipelineArguments.ImageBuilder)
+	buildContainerSecContext := &pipelineInfo.PipelineArguments.ContainerSecurityContext
+	var secretMountsArgsString string
 	if isUsingBuildKit(pipelineInfo) {
 		imageBuilder = pipelineInfo.PipelineArguments.BuildKitImageBuilder
 		buildContainerSecContext = getBuildContainerSecContext()
 		secretMountsArgsString = getSecretArgs(buildSecrets)
 	}
 
-	gitCommitHash := pipelineInfo.GitCommitHash
-	gitTags := pipelineInfo.GitTags
-
-	var containers []corev1.Container
-	azureServicePrincipleContext := "/radix-image-builder/.azure"
-	firstPartContainerRegistry := strings.Split(containerRegistry, ".")[0]
 	var push string
-	var useCache string
 	if pushImage {
 		push = "--push"
 	}
+
+	var useCache string
 	if !pipelineInfo.PipelineArguments.UseCache {
 		useCache = "--no-cache"
 	}
+
 	distinctBuildContainers := make(map[string]void)
 	for _, componentImage := range pipelineInfo.BuildComponentImages {
 		if _, exists := distinctBuildContainers[componentImage.ContainerName]; exists {
@@ -174,7 +183,6 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo,
 		clusterTypeImage := utils.GetImagePath(containerRegistry, appName, componentImage.ImageName, fmt.Sprintf("%s-%s", clusterType, imageTag))
 		clusterNameImage := utils.GetImagePath(containerRegistry, appName, componentImage.ImageName, fmt.Sprintf("%s-%s", clusterName, imageTag))
 		containerImageRepositoryName := utils.GetRepositoryName(appName, componentImage.ImageName)
-
 		envVars := []corev1.EnvVar{
 			{
 				Name:  "DOCKER_FILE_NAME",
@@ -198,7 +206,7 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo,
 			},
 			{
 				Name:  "AZURE_CREDENTIALS",
-				Value: fmt.Sprintf("%s/sp_credentials.json", azureServicePrincipleContext),
+				Value: path.Join(azureServicePrincipleContext, "sp_credentials.json"),
 			},
 			{
 				Name:  "SUBSCRIPTION_ID",
@@ -235,83 +243,97 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo,
 			},
 			{
 				Name:  defaults.RadixCommitHashEnvironmentVariable,
-				Value: gitCommitHash,
+				Value: pipelineInfo.GitCommitHash,
 			},
 			{
 				Name:  defaults.RadixGitTagsEnvironmentVariable,
-				Value: gitTags,
+				Value: pipelineInfo.GitTags,
 			},
-			// buildah specific env vars
-			{
-				Name: "BUILDAH_USERNAME",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName},
-						Key:                  "username",
+		}
+		if isUsingBuildKit(pipelineInfo) {
+			envVars = append(envVars, []corev1.EnvVar{
+				{
+					Name: "BUILDAH_USERNAME",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName},
+							Key:                  "username",
+						},
 					},
 				},
-			},
-			{
-				Name: "BUILDAH_PASSWORD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName},
-						Key:                  "password",
+				{
+					Name: "BUILDAH_PASSWORD",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName},
+							Key:                  "password",
+						},
 					},
 				},
-			},
-			{
-				Name: "BUILDAH_CACHE_USERNAME",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName},
-						Key:                  "username",
+				{
+					Name: "BUILDAH_CACHE_USERNAME",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName},
+							Key:                  "username",
+						},
 					},
 				},
-			},
-			{
-				Name: "BUILDAH_CACHE_PASSWORD",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName},
-						Key:                  "password",
+				{
+					Name: "BUILDAH_CACHE_PASSWORD",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName},
+							Key:                  "password",
+						},
 					},
 				},
-			},
+				{
+					Name:  "REGISTRY_AUTH_FILE",
+					Value: buildahRegistryAuthFile,
+				},
+			}...)
 		}
-
 		envVars = append(envVars, buildSecrets...)
 
-		container := corev1.Container{
-			Name:            componentImage.ContainerName,
-			Image:           imageBuilder,
-			ImagePullPolicy: corev1.PullAlways,
-			Env:             envVars,
-			VolumeMounts:    getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext, buildSecrets),
-			SecurityContext: buildContainerSecContext,
-		}
+		var command []string
 		if isUsingBuildKit(pipelineInfo) {
 			cacheImagePath := utils.GetImageCachePath(pipelineInfo.PipelineArguments.AppContainerRegistry, pipelineInfo.RadixApplication.Name)
 			useBuildCache := pipelineInfo.RadixApplication.Spec.Build.UseBuildCache == nil || *pipelineInfo.RadixApplication.Spec.Build.UseBuildCache
 			cacheContainerRegistry := pipelineInfo.PipelineArguments.AppContainerRegistry // Store application cache in the App Registry
+			command = getBuildahContainerCommand(containerRegistry, secretMountsArgsString, componentImage.Context, componentImage.Dockerfile, componentImage.ImagePath, clusterTypeImage, clusterNameImage, cacheContainerRegistry, cacheImagePath, useBuildCache, pushImage)
+		}
 
-			containerCommand = getBuildahContainerCommand(containerRegistry, secretMountsArgsString, componentImage.Context, componentImage.Dockerfile, componentImage.ImagePath, clusterTypeImage, clusterNameImage, cacheContainerRegistry, cacheImagePath, useBuildCache, pushImage)
-			container.Command = containerCommand
-			container.Resources.Requests = map[corev1.ResourceName]resource.Quantity{
-				corev1.ResourceCPU:    resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsCPU),
-				corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsMemory),
-			}
-			container.Resources.Limits = map[corev1.ResourceName]resource.Quantity{
-				corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesLimitsMemory),
+		var resources corev1.ResourceRequirements
+		if isUsingBuildKit(pipelineInfo) {
+			resources = corev1.ResourceRequirements{
+				Requests: map[corev1.ResourceName]resource.Quantity{
+					corev1.ResourceCPU:    resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsCPU),
+					corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsMemory),
+				},
+				Limits: map[corev1.ResourceName]resource.Quantity{
+					corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesLimitsMemory),
+				},
 			}
 		}
+
+		container := corev1.Container{
+			Name:            componentImage.ContainerName,
+			Image:           imageBuilder,
+			Command:         command,
+			ImagePullPolicy: corev1.PullAlways,
+			Env:             envVars,
+			VolumeMounts:    getBuildAcrJobContainerVolumeMounts(buildSecrets, isUsingBuildKit(pipelineInfo)),
+			SecurityContext: buildContainerSecContext,
+			Resources:       resources,
+		}
 		containers = append(containers, container)
 	}
 
 	return containers
 }
 
-func getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext string, buildSecrets []corev1.EnvVar) []corev1.VolumeMount {
+func getBuildAcrJobContainerVolumeMounts(buildSecrets []corev1.EnvVar, mountPrivateImageHubAuth bool) []corev1.VolumeMount {
 	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      git.BuildContextVolumeName,
@@ -323,27 +345,33 @@ func getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext string, bu
 			ReadOnly:  true,
 		},
 	}
-	if len(buildSecrets) == 0 {
-		return volumeMounts
-	}
-	volumeMounts = append(volumeMounts,
-		corev1.VolumeMount{
-			Name:      defaults.BuildSecretsName,
-			MountPath: buildSecretsMountPath,
-			ReadOnly:  true,
+
+	if mountPrivateImageHubAuth {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      defaults.PrivateImageHubSecretName,
+			MountPath: privateImageHubMountPath,
 		})
+	}
+
+	if len(buildSecrets) > 0 {
+		volumeMounts = append(volumeMounts,
+			corev1.VolumeMount{
+				Name:      defaults.BuildSecretsName,
+				MountPath: buildSecretsMountPath,
+				ReadOnly:  true,
+			})
+	}
+
 	return volumeMounts
 }
 
 func getBuildahContainerCommand(containerImageRegistry, secretArgsString, context, dockerFileName, imageTag, clusterTypeImageTag, clusterNameImageTag, cacheContainerImageRegistry, cacheImagePath string, useBuildCache, pushImage bool) []string {
-
 	commandList := commandbuilder.NewCommandList()
-
+	commandList.AddStrCmd("cp %s %s", path.Join(privateImageHubMountPath, ".dockerconfigjson"), buildahRegistryAuthFile)
 	commandList.AddStrCmd("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s", containerImageRegistry)
 	if useBuildCache {
 		commandList.AddStrCmd("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s", cacheContainerImageRegistry)
 	}
-
 	buildah := commandbuilder.NewCommand("/usr/bin/buildah build")
 	commandList.AddCmd(buildah)