From 263454d46d1bed5e33f79cad9bd7f200f5d3a861 Mon Sep 17 00:00:00 2001 From: Anneli Aune <78496131+anneliawa@users.noreply.github.com> Date: Tue, 5 Dec 2023 08:41:23 +0100 Subject: [PATCH] 115 privateimagehubs dockerfile (#998) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use additionalDockerAuth for build using privateimagehubs credentials * rename docker auth struct * grant pipeline runner access to privateimagehub secret * update charts * mount and use private image hub auth with buildah --------- Co-authored-by: Nils Gustav Stråbø --- charts/radix-operator/Chart.yaml | 4 +- go.mod | 2 +- go.sum | 4 +- pipeline-runner/steps/build_acr.go | 182 ++++++++++-------- pipeline-runner/steps/build_test.go | 46 ++++- .../applicationconfig/applicationconfig.go | 13 -- .../applicationconfig_test.go | 37 +++- pkg/apis/applicationconfig/imagehubsecret.go | 57 +++--- pkg/apis/applicationconfig/role.go | 18 +- pkg/apis/applicationconfig/rolebinding.go | 2 +- 10 files changed, 219 insertions(+), 146 deletions(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index ef0ddbd51..8465b2b01 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator -version: 1.25.9 -appVersion: 1.45.9 +version: 1.26.0 +appVersion: 1.46.0 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/go.mod b/go.mod index d69150403..33a965a3a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( dario.cat/mergo v1.0.0 - github.com/equinor/radix-common v1.5.0 + github.com/equinor/radix-common v1.6.2 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 491c66727..0ea2a57cd 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/equinor/radix-common v1.5.0 h1:z5hQHlKG2x16/NnV4b9ynf9n5ZageYUewE4MANdA96Y= -github.com/equinor/radix-common v1.5.0/go.mod h1:UZ69U56VFtTxABi5JjGdaqn9Df5ilfTTqzUQ0riofVM= +github.com/equinor/radix-common v1.6.2 h1:856A/XcdDwzejmHbbuuWra/xFB6iu8uksEAme/M1v7c= +github.com/equinor/radix-common v1.6.2/go.mod h1:9hHvudaiqmoIjCqKlsW14jMj8qU/b/wMXUwkffd9MUw= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= diff --git a/pipeline-runner/steps/build_acr.go b/pipeline-runner/steps/build_acr.go index 66a123954..4e51567ea 100644 --- a/pipeline-runner/steps/build_acr.go +++ b/pipeline-runner/steps/build_acr.go @@ -3,6 +3,7 @@ package steps import ( "encoding/json" "fmt" + "path" "strings" "time" @@ -22,7 +23,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const buildSecretsMountPath = "/build-secrets" +const ( + buildSecretsMountPath = "/build-secrets" + privateImageHubMountPath = "/radix-private-image-hubs" + buildahRegistryAuthFile = "/home/build/auth.json" + azureServicePrincipleContext = "/radix-image-builder/.azure" +) type void struct{} @@ -33,14 +39,13 @@ func createACRBuildJob(rr *v1.RadixRegistration, pipelineInfo *model.PipelineInf branch := pipelineInfo.PipelineArguments.Branch imageTag := pipelineInfo.PipelineArguments.ImageTag jobName := pipelineInfo.PipelineArguments.JobName - initContainers := git.CloneInitContainers(rr.Spec.CloneURL, branch, pipelineInfo.PipelineArguments.ContainerSecurityContext) buildContainers := createACRBuildContainers(appName, pipelineInfo, buildSecrets) timestamp := time.Now().Format("20060102150405") defaultMode, backOffLimit := int32(256), int32(0) - componentImagesAnnotation, _ := json.Marshal(pipelineInfo.BuildComponentImages) hash := strings.ToLower(utils.RandStringStrSeed(5, pipelineInfo.PipelineArguments.JobName)) + annotations := radixannotations.ForClusterAutoscalerSafeToEvict(false) buildPodSecurityContext := &pipelineInfo.PipelineArguments.PodSecurityContext if isUsingBuildKit(pipelineInfo) { @@ -109,6 +114,14 @@ func getACRBuildJobVolumes(defaultMode *int32, buildSecrets []corev1.EnvVar) []c }, }, }, + { + Name: defaults.PrivateImageHubSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: defaults.PrivateImageHubSecretName, + }, + }, + }, } if len(buildSecrets) > 0 { @@ -127,40 +140,36 @@ func getACRBuildJobVolumes(defaultMode *int32, buildSecrets []corev1.EnvVar) []c } func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo, buildSecrets []corev1.EnvVar) []corev1.Container { + var containers []corev1.Container imageTag := pipelineInfo.PipelineArguments.ImageTag pushImage := pipelineInfo.PipelineArguments.PushImage - buildContainerSecContext := &pipelineInfo.PipelineArguments.ContainerSecurityContext - var containerCommand []string - clusterType := pipelineInfo.PipelineArguments.Clustertype clusterName := pipelineInfo.PipelineArguments.Clustername containerRegistry := pipelineInfo.PipelineArguments.ContainerRegistry - imageBuilder := fmt.Sprintf("%s/%s", containerRegistry, pipelineInfo.PipelineArguments.ImageBuilder) subscriptionId := pipelineInfo.PipelineArguments.SubscriptionId branch := pipelineInfo.PipelineArguments.Branch targetEnvs := strings.Join(pipelineInfo.TargetEnvironments, ",") - secretMountsArgsString := "" + firstPartContainerRegistry := strings.Split(containerRegistry, ".")[0] + imageBuilder := fmt.Sprintf("%s/%s", containerRegistry, pipelineInfo.PipelineArguments.ImageBuilder) + buildContainerSecContext := &pipelineInfo.PipelineArguments.ContainerSecurityContext + var secretMountsArgsString string if isUsingBuildKit(pipelineInfo) { imageBuilder = pipelineInfo.PipelineArguments.BuildKitImageBuilder buildContainerSecContext = getBuildContainerSecContext() secretMountsArgsString = getSecretArgs(buildSecrets) } - gitCommitHash := pipelineInfo.GitCommitHash - gitTags := pipelineInfo.GitTags - - var containers []corev1.Container - azureServicePrincipleContext := "/radix-image-builder/.azure" - firstPartContainerRegistry := strings.Split(containerRegistry, ".")[0] var push string - var useCache string if pushImage { push = "--push" } + + var useCache string if !pipelineInfo.PipelineArguments.UseCache { useCache = "--no-cache" } + distinctBuildContainers := make(map[string]void) for _, componentImage := range pipelineInfo.BuildComponentImages { if _, exists := distinctBuildContainers[componentImage.ContainerName]; exists { @@ -174,7 +183,6 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo, clusterTypeImage := utils.GetImagePath(containerRegistry, appName, componentImage.ImageName, fmt.Sprintf("%s-%s", clusterType, imageTag)) clusterNameImage := utils.GetImagePath(containerRegistry, appName, componentImage.ImageName, fmt.Sprintf("%s-%s", clusterName, imageTag)) containerImageRepositoryName := utils.GetRepositoryName(appName, componentImage.ImageName) - envVars := []corev1.EnvVar{ { Name: "DOCKER_FILE_NAME", @@ -198,7 +206,7 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo, }, { Name: "AZURE_CREDENTIALS", - Value: fmt.Sprintf("%s/sp_credentials.json", azureServicePrincipleContext), + Value: path.Join(azureServicePrincipleContext, "sp_credentials.json"), }, { Name: "SUBSCRIPTION_ID", @@ -235,83 +243,97 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo, }, { Name: defaults.RadixCommitHashEnvironmentVariable, - Value: gitCommitHash, + Value: pipelineInfo.GitCommitHash, }, { Name: defaults.RadixGitTagsEnvironmentVariable, - Value: gitTags, + Value: pipelineInfo.GitTags, }, - // buildah specific env vars - { - Name: "BUILDAH_USERNAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}, - Key: "username", + } + if isUsingBuildKit(pipelineInfo) { + envVars = append(envVars, []corev1.EnvVar{ + { + Name: "BUILDAH_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}, + Key: "username", + }, }, }, - }, - { - Name: "BUILDAH_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}, - Key: "password", + { + Name: "BUILDAH_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}, + Key: "password", + }, }, }, - }, - { - Name: "BUILDAH_CACHE_USERNAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}, - Key: "username", + { + Name: "BUILDAH_CACHE_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}, + Key: "username", + }, }, }, - }, - { - Name: "BUILDAH_CACHE_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}, - Key: "password", + { + Name: "BUILDAH_CACHE_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}, + Key: "password", + }, }, }, - }, + { + Name: "REGISTRY_AUTH_FILE", + Value: buildahRegistryAuthFile, + }, + }...) } - envVars = append(envVars, buildSecrets...) - container := corev1.Container{ - Name: componentImage.ContainerName, - Image: imageBuilder, - ImagePullPolicy: corev1.PullAlways, - Env: envVars, - VolumeMounts: getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext, buildSecrets), - SecurityContext: buildContainerSecContext, - } + var command []string if isUsingBuildKit(pipelineInfo) { cacheImagePath := utils.GetImageCachePath(pipelineInfo.PipelineArguments.AppContainerRegistry, pipelineInfo.RadixApplication.Name) useBuildCache := pipelineInfo.RadixApplication.Spec.Build.UseBuildCache == nil || *pipelineInfo.RadixApplication.Spec.Build.UseBuildCache cacheContainerRegistry := pipelineInfo.PipelineArguments.AppContainerRegistry // Store application cache in the App Registry + command = getBuildahContainerCommand(containerRegistry, secretMountsArgsString, componentImage.Context, componentImage.Dockerfile, componentImage.ImagePath, clusterTypeImage, clusterNameImage, cacheContainerRegistry, cacheImagePath, useBuildCache, pushImage) + } - containerCommand = getBuildahContainerCommand(containerRegistry, secretMountsArgsString, componentImage.Context, componentImage.Dockerfile, componentImage.ImagePath, clusterTypeImage, clusterNameImage, cacheContainerRegistry, cacheImagePath, useBuildCache, pushImage) - container.Command = containerCommand - container.Resources.Requests = map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceCPU: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsCPU), - corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsMemory), - } - container.Resources.Limits = map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesLimitsMemory), + var resources corev1.ResourceRequirements + if isUsingBuildKit(pipelineInfo) { + resources = corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsCPU), + corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsMemory), + }, + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceMemory: resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesLimitsMemory), + }, } } + + container := corev1.Container{ + Name: componentImage.ContainerName, + Image: imageBuilder, + Command: command, + ImagePullPolicy: corev1.PullAlways, + Env: envVars, + VolumeMounts: getBuildAcrJobContainerVolumeMounts(buildSecrets, isUsingBuildKit(pipelineInfo)), + SecurityContext: buildContainerSecContext, + Resources: resources, + } containers = append(containers, container) } return containers } -func getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext string, buildSecrets []corev1.EnvVar) []corev1.VolumeMount { +func getBuildAcrJobContainerVolumeMounts(buildSecrets []corev1.EnvVar, mountPrivateImageHubAuth bool) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: git.BuildContextVolumeName, @@ -323,27 +345,33 @@ func getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext string, bu ReadOnly: true, }, } - if len(buildSecrets) == 0 { - return volumeMounts - } - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: defaults.BuildSecretsName, - MountPath: buildSecretsMountPath, - ReadOnly: true, + + if mountPrivateImageHubAuth { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: defaults.PrivateImageHubSecretName, + MountPath: privateImageHubMountPath, }) + } + + if len(buildSecrets) > 0 { + volumeMounts = append(volumeMounts, + corev1.VolumeMount{ + Name: defaults.BuildSecretsName, + MountPath: buildSecretsMountPath, + ReadOnly: true, + }) + } + return volumeMounts } func getBuildahContainerCommand(containerImageRegistry, secretArgsString, context, dockerFileName, imageTag, clusterTypeImageTag, clusterNameImageTag, cacheContainerImageRegistry, cacheImagePath string, useBuildCache, pushImage bool) []string { - commandList := commandbuilder.NewCommandList() - + commandList.AddStrCmd("cp %s %s", path.Join(privateImageHubMountPath, ".dockerconfigjson"), buildahRegistryAuthFile) commandList.AddStrCmd("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s", containerImageRegistry) if useBuildCache { commandList.AddStrCmd("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s", cacheContainerImageRegistry) } - buildah := commandbuilder.NewCommand("/usr/bin/buildah build") commandList.AddCmd(buildah) diff --git a/pipeline-runner/steps/build_test.go b/pipeline-runner/steps/build_test.go index 32c949a6e..348d8c2a2 100644 --- a/pipeline-runner/steps/build_test.go +++ b/pipeline-runner/steps/build_test.go @@ -161,6 +161,7 @@ func (s *buildTestSuite) Test_BuildDeploy_JobSpecAndDeploymentConsistent() { {Name: git.BuildContextVolumeName}, {Name: git.GitSSHKeyVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: git.GitSSHKeyVolumeName, DefaultMode: pointers.Ptr[int32](256)}}}, {Name: defaults.AzureACRServicePrincipleSecretName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.AzureACRServicePrincipleSecretName}}}, + {Name: defaults.PrivateImageHubSecretName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.PrivateImageHubSecretName}}}, } s.ElementsMatch(expectedVolumes, job.Spec.Template.Spec.Volumes) @@ -204,10 +205,6 @@ func (s *buildTestSuite) Test_BuildDeploy_JobSpecAndDeploymentConsistent() { {Name: "TARGET_ENVIRONMENTS", Value: "dev"}, {Name: "RADIX_GIT_COMMIT_HASH", Value: gitHash}, {Name: "RADIX_GIT_TAGS", Value: gitTags}, - {Name: "BUILDAH_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}}, - {Name: "BUILDAH_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}}, - {Name: "BUILDAH_CACHE_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}}, - {Name: "BUILDAH_CACHE_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}}, } s.ElementsMatch(expectedEnv, job.Spec.Template.Spec.Containers[0].Env) @@ -1334,7 +1331,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_WithBuildSecrets() { jobs, _ := s.kubeClient.BatchV1().Jobs(utils.GetAppNamespace(appName)).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := jobs.Items[0] - s.Len(job.Spec.Template.Spec.Volumes, 4) + s.Len(job.Spec.Template.Spec.Volumes, 5) expectedVolumes := []corev1.Volume{ {Name: defaults.BuildSecretsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.BuildSecretsName}}}, } @@ -1355,6 +1352,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_WithBuildSecrets() { func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() { appName, rjName, compName, sourceFolder, dockerFile := "anyapp", "anyrj", "c1", "../path1/./../../path2", "anydockerfile" prepareConfigMapName := "preparecm" + gitConfigMapName, gitHash, gitTags := "gitcm", "githash", "gittags" rr := utils.ARadixRegistration().WithName(appName).BuildRR() _, _ = s.radixClient.RadixV1().RadixRegistrations().Create(context.Background(), rr, metav1.CreateOptions{}) rj := utils.ARadixBuildDeployJob().WithJobName(rjName).WithAppName(appName).BuildRJ() @@ -1366,8 +1364,10 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() { WithComponent(utils.NewApplicationComponentBuilder().WithPort("any", 8080).WithName(compName).WithDockerfileName(dockerFile).WithSourceFolder(sourceFolder)). BuildRA() s.Require().NoError(internaltest.CreatePreparePipelineConfigMapResponse(s.kubeClient, prepareConfigMapName, appName, ra, nil)) + s.Require().NoError(internaltest.CreateGitInfoConfigMapResponse(s.kubeClient, gitConfigMapName, appName, gitHash, gitTags)) pipeline := model.PipelineInfo{ PipelineArguments: model.PipelineArguments{ + PipelineType: "build-deploy", Branch: "main", JobName: rjName, BuildKitImageBuilder: "anybuildkitimage:tag", @@ -1379,6 +1379,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() { Builder: model.Builder{ResourcesLimitsMemory: "100M", ResourcesRequestsCPU: "50m", ResourcesRequestsMemory: "50M"}, }, RadixConfigMapName: prepareConfigMapName, + GitConfigMapName: gitConfigMapName, } jobWaiter := internalwait.NewMockJobCompletionWaiter(s.ctrl) jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) @@ -1396,6 +1397,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() { s.Equal(pipeline.PipelineArguments.BuildKitImageBuilder, job.Spec.Template.Spec.Containers[0].Image) expectedBuildCmd := strings.Join( []string{ + "cp /radix-private-image-hubs/.dockerconfigjson /home/build/auth.json && ", fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && ", pipeline.PipelineArguments.ContainerRegistry), fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s && ", pipeline.PipelineArguments.AppContainerRegistry), "/usr/bin/buildah build --storage-driver=overlay --isolation=chroot --jobs 0 ", @@ -1413,6 +1415,34 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() { ) expectedCommand := []string{"/bin/bash", "-c", expectedBuildCmd} s.Equal(expectedCommand, job.Spec.Template.Spec.Containers[0].Command) + expectedEnv := []corev1.EnvVar{ + {Name: "DOCKER_FILE_NAME", Value: dockerFile}, + {Name: "DOCKER_REGISTRY", Value: pipeline.PipelineArguments.ContainerRegistry}, + {Name: "IMAGE", Value: fmt.Sprintf("%s/%s-%s:%s", pipeline.PipelineArguments.ContainerRegistry, appName, compName, pipeline.PipelineArguments.ImageTag)}, + {Name: "CONTEXT", Value: "/workspace/path2/"}, + {Name: "PUSH", Value: ""}, + {Name: "AZURE_CREDENTIALS", Value: "/radix-image-builder/.azure/sp_credentials.json"}, + {Name: "SUBSCRIPTION_ID", Value: pipeline.PipelineArguments.SubscriptionId}, + {Name: "CLUSTERTYPE_IMAGE", Value: fmt.Sprintf("%s/%s-%s:%s-%s", pipeline.PipelineArguments.ContainerRegistry, appName, compName, pipeline.PipelineArguments.Clustertype, pipeline.PipelineArguments.ImageTag)}, + {Name: "CLUSTERNAME_IMAGE", Value: fmt.Sprintf("%s/%s-%s:%s-%s", pipeline.PipelineArguments.ContainerRegistry, appName, compName, pipeline.PipelineArguments.Clustername, pipeline.PipelineArguments.ImageTag)}, + {Name: "REPOSITORY_NAME", Value: fmt.Sprintf("%s-%s", appName, compName)}, + {Name: "CACHE", Value: "--no-cache"}, + {Name: "RADIX_ZONE", Value: pipeline.PipelineArguments.RadixZone}, + {Name: "BRANCH", Value: pipeline.PipelineArguments.Branch}, + {Name: "TARGET_ENVIRONMENTS", Value: "dev"}, + {Name: "RADIX_GIT_COMMIT_HASH", Value: gitHash}, + {Name: "RADIX_GIT_TAGS", Value: gitTags}, + {Name: "BUILDAH_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}}, + {Name: "BUILDAH_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}}, + {Name: "BUILDAH_CACHE_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}}, + {Name: "BUILDAH_CACHE_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}}, + {Name: "REGISTRY_AUTH_FILE", Value: "/home/build/auth.json"}, + } + s.ElementsMatch(expectedEnv, job.Spec.Template.Spec.Containers[0].Env) + expectedVolumeMounts := []corev1.VolumeMount{ + {Name: defaults.PrivateImageHubSecretName, MountPath: "/radix-private-image-hubs"}, + } + s.Subset(job.Spec.Template.Spec.Containers[0].VolumeMounts, expectedVolumeMounts) } func (s *buildTestSuite) Test_BuildJobSpec_BuildKit_PushImage() { @@ -1460,6 +1490,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit_PushImage() { s.Equal(pipeline.PipelineArguments.BuildKitImageBuilder, job.Spec.Template.Spec.Containers[0].Image) expectedBuildCmd := strings.Join( []string{ + "cp /radix-private-image-hubs/.dockerconfigjson /home/build/auth.json && ", fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && ", pipeline.PipelineArguments.ContainerRegistry), fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s && ", pipeline.PipelineArguments.AppContainerRegistry), "/usr/bin/buildah build --storage-driver=overlay --isolation=chroot --jobs 0 ", @@ -1526,13 +1557,13 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit_WithBuildSecrets() { jobs, _ := s.kubeClient.BatchV1().Jobs(utils.GetAppNamespace(appName)).List(context.Background(), metav1.ListOptions{}) s.Require().Len(jobs.Items, 1) job := jobs.Items[0] - s.Len(job.Spec.Template.Spec.Volumes, 4) + s.Len(job.Spec.Template.Spec.Volumes, 5) expectedVolumes := []corev1.Volume{ {Name: defaults.BuildSecretsName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: defaults.BuildSecretsName}}}, } s.Subset(job.Spec.Template.Spec.Volumes, expectedVolumes) s.Require().Len(job.Spec.Template.Spec.Containers, 1) - s.Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 3) + s.Len(job.Spec.Template.Spec.Containers[0].VolumeMounts, 4) expectedVolumeMounts := []corev1.VolumeMount{ {Name: defaults.BuildSecretsName, MountPath: "/build-secrets", ReadOnly: true}, } @@ -1540,6 +1571,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit_WithBuildSecrets() { s.Equal(pipeline.PipelineArguments.BuildKitImageBuilder, job.Spec.Template.Spec.Containers[0].Image) expectedBuildCmd := strings.Join( []string{ + "cp /radix-private-image-hubs/.dockerconfigjson /home/build/auth.json && ", fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && ", pipeline.PipelineArguments.ContainerRegistry), fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s && ", pipeline.PipelineArguments.AppContainerRegistry), "/usr/bin/buildah build --storage-driver=overlay --isolation=chroot --jobs 0 ", diff --git a/pkg/apis/applicationconfig/applicationconfig.go b/pkg/apis/applicationconfig/applicationconfig.go index f5a359822..24b801d3e 100644 --- a/pkg/apis/applicationconfig/applicationconfig.go +++ b/pkg/apis/applicationconfig/applicationconfig.go @@ -7,8 +7,6 @@ import ( "reflect" "strings" - "github.com/equinor/radix-operator/pkg/apis/defaults" - "github.com/equinor/radix-operator/pkg/apis/utils/branch" "github.com/equinor/radix-operator/pkg/apis/kube" @@ -168,17 +166,6 @@ func (app *ApplicationConfig) OnSync() error { return err } - err = utils.GrantAppReaderAccessToSecret(app.kubeutil, app.registration, defaults.PrivateImageHubReaderRoleName, defaults.PrivateImageHubSecretName) - if err != nil { - log.Warnf("failed to grant reader access to private image hub secret %v", err) - } - - err = utils.GrantAppAdminAccessToSecret(app.kubeutil, app.registration, defaults.PrivateImageHubSecretName, defaults.PrivateImageHubSecretName) - if err != nil { - log.Warnf("failed to grant access to private image hub secret %v", err) - return err - } - err = app.syncBuildSecrets() if err != nil { log.Errorf("Failed to create build secrets. %v", err) diff --git a/pkg/apis/applicationconfig/applicationconfig_test.go b/pkg/apis/applicationconfig/applicationconfig_test.go index ad138804b..cc8c1d2eb 100644 --- a/pkg/apis/applicationconfig/applicationconfig_test.go +++ b/pkg/apis/applicationconfig/applicationconfig_test.go @@ -7,6 +7,8 @@ import ( "log" "testing" + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -392,20 +394,43 @@ func Test_AppReaderBuildSecretsRoleAndRoleBindingExists(t *testing.T) { assert.False(t, roleBindingByNameExists("radix-app-reader-build-secrets", rolebindings)) } -func Test_AppReaderPrivateImageHubRoleAndRoleBindingExists(t *testing.T) { +func Test_PrivateImageHubRoleAndRoleBindingExists(t *testing.T) { tu, client, kubeUtil, radixclient := setupTest() + adminGroups, readerGroups := []string{"admin1", "admin2"}, []string{"reader1", "reader2"} applyApplicationWithSync(tu, client, kubeUtil, radixclient, utils.ARadixApplication(). + WithRadixRegistration(utils.ARadixRegistration().WithAdGroups(adminGroups).WithReaderAdGroups(readerGroups)). WithAppName("any-app"). - WithEnvironment("dev", "master"). - WithBuildSecrets("secret1", "secret2")) + WithEnvironment("dev", "master")) - roles, _ := client.RbacV1().Roles("any-app-app").List(context.TODO(), metav1.ListOptions{}) - assert.True(t, roleByNameExists("radix-private-image-hubs-reader", roles)) + type testSpec struct { + roleName string + expectedVerbs []string + expectedSubjects []string + } + tests := []testSpec{ + {roleName: "radix-private-image-hubs-reader", expectedVerbs: []string{"get", "list", "watch"}, expectedSubjects: readerGroups}, + {roleName: "radix-private-image-hubs", expectedVerbs: []string{"get", "list", "watch", "update", "patch", "delete"}, expectedSubjects: adminGroups}, + } + roles, _ := client.RbacV1().Roles("any-app-app").List(context.TODO(), metav1.ListOptions{}) rolebindings, _ := client.RbacV1().RoleBindings("any-app-app").List(context.TODO(), metav1.ListOptions{}) - assert.True(t, roleBindingByNameExists("radix-private-image-hubs-reader", rolebindings)) + + for _, test := range tests { + t.Run(test.roleName, func(t *testing.T) { + expectedRules := []rbacv1.PolicyRule{ + {Verbs: test.expectedVerbs, Resources: []string{"secrets"}, APIGroups: []string{""}, ResourceNames: []string{"radix-private-image-hubs"}}, + } + assert.True(t, roleByNameExists(test.roleName, roles)) + assert.ElementsMatch(t, expectedRules, getRoleByName(test.roleName, roles).Rules) + assert.True(t, roleBindingByNameExists(test.roleName, rolebindings)) + expectedRoleRef := rbacv1.RoleRef{Kind: "Role", APIGroup: "rbac.authorization.k8s.io", Name: test.roleName} + assert.Equal(t, expectedRoleRef, getRoleBindingByName(test.roleName, rolebindings).RoleRef) + actualSubjectNames := slice.Map(getRoleBindingByName(test.roleName, rolebindings).Subjects, func(s rbacv1.Subject) string { return s.Name }) + assert.ElementsMatch(t, test.expectedSubjects, actualSubjectNames) + }) + } } func Test_WithPrivateImageHubSet_SecretsCorrectly_Added(t *testing.T) { client, _ := applyRadixAppWithPrivateImageHub(radixv1.PrivateImageHubEntries{ diff --git a/pkg/apis/applicationconfig/imagehubsecret.go b/pkg/apis/applicationconfig/imagehubsecret.go index 18ef54d3a..08798c916 100644 --- a/pkg/apis/applicationconfig/imagehubsecret.go +++ b/pkg/apis/applicationconfig/imagehubsecret.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" + "github.com/equinor/radix-common/pkg/docker" + "github.com/equinor/radix-operator/pkg/apis/defaults" "github.com/equinor/radix-operator/pkg/apis/kube" v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -74,8 +76,8 @@ func (app *ApplicationConfig) GetPendingPrivateImageHubSecrets() ([]string, erro } func (app *ApplicationConfig) syncPrivateImageHubSecrets() error { - ns := utils.GetAppNamespace(app.config.Name) - secret, err := app.kubeutil.GetSecret(ns, defaults.PrivateImageHubSecretName) + namespace := utils.GetAppNamespace(app.config.Name) + secret, err := app.kubeutil.GetSecret(namespace, defaults.PrivateImageHubSecretName) if err != nil && !errors.IsNotFound(err) { log.Warnf("failed to get private image hub secret %v", err) return err @@ -112,7 +114,7 @@ func (app *ApplicationConfig) syncPrivateImageHubSecrets() error { imageHubs[server] = currentConfig } } else { - imageHubs[server] = secretImageHub{ + imageHubs[server] = docker.Credential{ Username: config.Username, Email: config.Email, } @@ -125,7 +127,24 @@ func (app *ApplicationConfig) syncPrivateImageHubSecrets() error { return err } } - return applyPrivateImageHubSecret(app.kubeutil, ns, app.config.Name, secretValue) + err = applyPrivateImageHubSecret(app.kubeutil, namespace, app.config.Name, secretValue) + if err != nil { + return nil + } + + err = utils.GrantAppReaderAccessToSecret(app.kubeutil, app.registration, defaults.PrivateImageHubReaderRoleName, defaults.PrivateImageHubSecretName) + if err != nil { + log.Warnf("failed to grant reader access to private image hub secret %v", err) + return err + } + + err = utils.GrantAppAdminAccessToSecret(app.kubeutil, app.registration, defaults.PrivateImageHubSecretName, defaults.PrivateImageHubSecretName) + if err != nil { + log.Warnf("failed to grant access to private image hub secret %v", err) + return err + } + + return nil } // applyPrivateImageHubSecret create a private image hub secret based on SecretTypeDockerConfigJson @@ -156,27 +175,9 @@ func applyPrivateImageHubSecret(kubeutil *kube.Kube, ns, appName string, secretV return nil } -// represent the secret of type docker-config -type secretImageHubsJSON struct { - Auths secretImageHubs `json:"auths"` -} - -type secretImageHubs map[string]secretImageHub - -type secretImageHub struct { - // +optional - Username string `json:"username"` - // +optional - Password string `json:"password"` - // +optional - Email string `json:"email"` - // +optional - Auth string `json:"auth"` -} - // getImageHubSecretValue gets imagehub secret value -func getImageHubSecretValue(value []byte) (secretImageHubs, error) { - secretValue := secretImageHubsJSON{} +func getImageHubSecretValue(value []byte) (docker.Auths, error) { + secretValue := docker.AuthConfig{} err := json.Unmarshal(value, &secretValue) if err != nil { return nil, err @@ -187,11 +188,11 @@ func getImageHubSecretValue(value []byte) (secretImageHubs, error) { // createImageHubsSecretValue turn PrivateImageHubEntries into a json string correctly formated for k8s and ImagePullSecrets func createImageHubsSecretValue(imagehubs v1.PrivateImageHubEntries) ([]byte, error) { - imageHubs := secretImageHubs(map[string]secretImageHub{}) + imageHubs := docker.Auths{} for server, config := range imagehubs { pwd := "" - imageHub := secretImageHub{ + imageHub := docker.Credential{ Username: config.Username, Email: config.Email, Password: pwd, @@ -202,13 +203,13 @@ func createImageHubsSecretValue(imagehubs v1.PrivateImageHubEntries) ([]byte, er } // getImageHubsSecretValue turn SecretImageHubs into a correctly formated secret for k8s ImagePullSecrets -func getImageHubsSecretValue(imageHubs secretImageHubs) ([]byte, error) { +func getImageHubsSecretValue(imageHubs docker.Auths) ([]byte, error) { for server, config := range imageHubs { config.Auth = encodeAuthField(config.Username, config.Password) imageHubs[server] = config } - imageHubJSON := secretImageHubsJSON{ + imageHubJSON := docker.AuthConfig{ Auths: imageHubs, } return json.Marshal(imageHubJSON) diff --git a/pkg/apis/applicationconfig/role.go b/pkg/apis/applicationconfig/role.go index 97f2784f9..e925f8948 100644 --- a/pkg/apis/applicationconfig/role.go +++ b/pkg/apis/applicationconfig/role.go @@ -13,7 +13,7 @@ import ( ) func (app *ApplicationConfig) grantAccessToBuildSecrets(namespace string) error { - err := app.grantPipelineAccessToBuildSecrets(namespace) + err := app.grantPipelineAccessToSecret(namespace, defaults.BuildSecretsName) if err != nil { return err } @@ -54,14 +54,14 @@ func (app *ApplicationConfig) grantAppAdminAccessToBuildSecrets(namespace string return app.kubeutil.ApplyRoleBinding(namespace, rolebinding) } -func (app *ApplicationConfig) grantPipelineAccessToBuildSecrets(namespace string) error { - role := rolePipelineBuildSecrets(app.GetRadixRegistration(), defaults.BuildSecretsName) +func (app *ApplicationConfig) grantPipelineAccessToSecret(namespace, secretName string) error { + role := rolePipelineSecret(app.GetRadixRegistration(), secretName) err := app.kubeutil.ApplyRole(namespace, role) if err != nil { return err } - rolebinding := rolebindingPipelineToBuildSecrets(app.GetRadixRegistration(), role) + rolebinding := rolebindingPipelineToRole(app.GetRadixRegistration(), role) return app.kubeutil.ApplyRoleBinding(namespace, rolebinding) } @@ -95,7 +95,7 @@ func (app *ApplicationConfig) garbageCollectAccessToBuildSecretsForRole(namespac func garbageCollectAccessToBuildSecrets(app *ApplicationConfig) error { appNamespace := utils.GetAppNamespace(app.config.Name) for _, roleName := range []string{ - getPipelineRoleNameToBuildSecrets(defaults.BuildSecretsName), + getPipelineRoleNameToSecret(defaults.BuildSecretsName), getAppReaderRoleNameToBuildSecrets(defaults.BuildSecretsName), getAppAdminRoleNameToBuildSecrets(defaults.BuildSecretsName), } { @@ -116,8 +116,8 @@ func roleAppReaderBuildSecrets(registration *radixv1.RadixRegistration, buildSec return kube.CreateReadSecretRole(registration.Name, getAppReaderRoleNameToBuildSecrets(buildSecretName), []string{buildSecretName}, nil) } -func rolePipelineBuildSecrets(registration *radixv1.RadixRegistration, buildSecretName string) *auth.Role { - return kube.CreateManageSecretRole(registration.Name, getPipelineRoleNameToBuildSecrets(buildSecretName), []string{buildSecretName}, nil) +func rolePipelineSecret(registration *radixv1.RadixRegistration, secretName string) *auth.Role { + return kube.CreateReadSecretRole(registration.Name, getPipelineRoleNameToSecret(secretName), []string{secretName}, nil) } func getAppAdminRoleNameToBuildSecrets(buildSecretName string) string { @@ -128,6 +128,6 @@ func getAppReaderRoleNameToBuildSecrets(buildSecretName string) string { return fmt.Sprintf("%s-%s", defaults.AppReaderRoleName, buildSecretName) } -func getPipelineRoleNameToBuildSecrets(buildSecretName string) string { - return fmt.Sprintf("%s-%s", "pipeline", buildSecretName) +func getPipelineRoleNameToSecret(secretName string) string { + return fmt.Sprintf("%s-%s", "pipeline", secretName) } diff --git a/pkg/apis/applicationconfig/rolebinding.go b/pkg/apis/applicationconfig/rolebinding.go index 5a43ba38f..1042166ab 100644 --- a/pkg/apis/applicationconfig/rolebinding.go +++ b/pkg/apis/applicationconfig/rolebinding.go @@ -24,7 +24,7 @@ func rolebindingAppAdminToBuildSecrets(registration *radixv1.RadixRegistration, return kube.GetRolebindingToRoleWithLabelsForSubjects(roleName, subjects, role.Labels) } -func rolebindingPipelineToBuildSecrets(registration *radixv1.RadixRegistration, role *auth.Role) *auth.RoleBinding { +func rolebindingPipelineToRole(registration *radixv1.RadixRegistration, role *auth.Role) *auth.RoleBinding { roleName := role.ObjectMeta.Name return kube.GetRolebindingToRoleForServiceAccountWithLabels(roleName, defaults.PipelineServiceAccountName, role.Namespace, role.Labels)