From 1a8ed1bf8bb2dbf35c307bc4d0ed16808af1c2a3 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Wed, 20 Mar 2024 16:35:21 +0100 Subject: [PATCH 01/13] Added waiting for new radix deployment to gets active to finish the pipeline job --- charts/radix-operator/Chart.yaml | 2 +- pipeline-runner/pipelines/app.go | 3 +- pipeline-runner/steps/build_test.go | 16 ++++----- pipeline-runner/steps/deploy.go | 20 +++++++---- pipeline-runner/steps/deploy_test.go | 15 +++++++-- pkg/apis/kube/deployment.go | 50 ++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 19 deletions(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index da1a20063..f57892543 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator version: 1.30.4 -appVersion: 1.50.3 +appVersion: 1.50.4 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/pipeline-runner/pipelines/app.go b/pipeline-runner/pipelines/app.go index f89605dff..5d22a169f 100644 --- a/pipeline-runner/pipelines/app.go +++ b/pipeline-runner/pipelines/app.go @@ -2,6 +2,7 @@ package pipelines import ( "context" + "time" "github.com/equinor/radix-operator/pipeline-runner/model" "github.com/equinor/radix-operator/pipeline-runner/steps" @@ -109,7 +110,7 @@ func (cli *PipelineRunner) initStepImplementations(registration *v1.RadixRegistr stepImplementations = append(stepImplementations, steps.NewApplyConfigStep()) stepImplementations = append(stepImplementations, steps.NewBuildStep(nil)) stepImplementations = append(stepImplementations, steps.NewRunPipelinesStep(nil)) - stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient))) + stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient), kube.NewRadixDeploymentWatcherImpl(cli.radixclient, time.Minute*5))) stepImplementations = append(stepImplementations, steps.NewPromoteStep()) for _, stepImplementation := range stepImplementations { diff --git a/pipeline-runner/steps/build_test.go b/pipeline-runner/steps/build_test.go index 7c71f504a..6b79022ff 100644 --- a/pipeline-runner/steps/build_test.go +++ b/pipeline-runner/steps/build_test.go @@ -144,7 +144,7 @@ func (s *buildTestSuite) Test_BuildDeploy_JobSpecAndDeploymentConsistent() { jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) @@ -273,7 +273,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_MultipleComponents() { jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) @@ -416,7 +416,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_MultipleComponents_IgnoreDisabled() { jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) @@ -568,7 +568,7 @@ func (s *buildTestSuite) Test_BuildChangedComponents() { jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) @@ -1041,7 +1041,7 @@ func (s *buildTestSuite) Test_DetectComponentsToBuild() { } buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) // Run pipeline steps @@ -1111,7 +1111,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_ImageTagNames() { applyStep := steps.NewApplyConfigStep() applyStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) @@ -1330,7 +1330,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_WithBuildSecrets() { applyStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) s.Require().NoError(buildStep.Run(&pipeline)) @@ -1694,7 +1694,7 @@ func (s *buildTestSuite) Test_BuildJobSpec_EnvConfigSrcAndImage() { jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1) buildStep := steps.NewBuildStep(jobWaiter) buildStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) - deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}) + deployStep := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) deployStep.Init(s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr) s.Require().NoError(applyStep.Run(&pipeline)) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index a222ce908..4be3df711 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -17,16 +17,18 @@ import ( // DeployStepImplementation Step to deploy RD into environment type DeployStepImplementation struct { - stepType pipeline.StepType - namespaceWatcher kube.NamespaceWatcher + stepType pipeline.StepType + namespaceWatcher kube.NamespaceWatcher + radixDeploymentWatcher kube.RadixDeploymentWatcher model.DefaultStepImplementation } // NewDeployStep Constructor -func NewDeployStep(namespaceWatcher kube.NamespaceWatcher) model.Step { +func NewDeployStep(namespaceWatcher kube.NamespaceWatcher, radixDeploymentWatcher kube.RadixDeploymentWatcher) model.Step { return &DeployStepImplementation{ - stepType: pipeline.DeployStep, - namespaceWatcher: namespaceWatcher, + stepType: pipeline.DeployStep, + namespaceWatcher: namespaceWatcher, + radixDeploymentWatcher: radixDeploymentWatcher, } } @@ -111,7 +113,8 @@ func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineIn return fmt.Errorf("failed to create radix deployments objects for app %s. %v", appName, err) } - err = cli.namespaceWatcher.WaitFor(utils.GetEnvironmentNamespace(cli.GetAppName(), env)) + namespace := utils.GetEnvironmentNamespace(cli.GetAppName(), env) + err = cli.namespaceWatcher.WaitFor(namespace) if err != nil { return fmt.Errorf("failed to get environment namespace, %s, for app %s. %v", env, appName, err) } @@ -121,6 +124,11 @@ func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineIn if err != nil { return fmt.Errorf("failed to apply radix deployment for app %s to environment %s. %v", appName, env, err) } + log.Infof("Waiting for radix deployment %s on env %s gets active", radixDeployment.GetName(), radixDeployment.GetNamespace()) + if err = cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeployment.GetName()); err != nil { + return err + } + log.Infof("The radix deployment %s on env %s is active", radixDeployment.GetName(), radixDeployment.GetNamespace()) return nil } diff --git a/pipeline-runner/steps/deploy_test.go b/pipeline-runner/steps/deploy_test.go index 97f40462d..3813371a7 100644 --- a/pipeline-runner/steps/deploy_test.go +++ b/pipeline-runner/steps/deploy_test.go @@ -49,11 +49,20 @@ func setupTest(t *testing.T) (*kubernetes.Clientset, *kube.Kube, *radix.Clientse type FakeNamespaceWatcher struct { } +// FakeRadixDeploymentWatcher Unit tests doesn't handle muliti-threading well +type FakeRadixDeploymentWatcher struct { +} + // WaitFor Waits for namespace to appear func (watcher FakeNamespaceWatcher) WaitFor(namespace string) error { return nil } +// WaitFor Waits for radix deployment gets active +func (watcher FakeRadixDeploymentWatcher) WaitForActive(namespace, deploymentName string) error { + return nil +} + func TestDeploy_BranchIsNotMapped_ShouldSkip(t *testing.T) { kubeclient, kubeUtil, radixclient, _ := setupTest(t) @@ -74,7 +83,7 @@ func TestDeploy_BranchIsNotMapped_ShouldSkip(t *testing.T) { WithName(anyComponentName)). BuildRA() - cli := steps.NewDeployStep(FakeNamespaceWatcher{}) + cli := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) cli.Init(kubeclient, radixclient, kubeUtil, &monitoring.Clientset{}, rr) targetEnvs := application.GetTargetEnvironments(anyNoMappedBranch, ra) @@ -178,7 +187,7 @@ func TestDeploy_PromotionSetup_ShouldCreateNamespacesForAllBranchesIfNotExists(t BuildRA() // Prometheus doesn´t contain any fake - cli := steps.NewDeployStep(FakeNamespaceWatcher{}) + cli := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) cli.Init(kubeclient, radixclient, kubeUtil, &monitoring.Clientset{}, rr) dnsConfig := dnsalias.DNSConfig{ @@ -298,7 +307,7 @@ func TestDeploy_SetCommitID_whenSet(t *testing.T) { BuildRA() // Prometheus doesn´t contain any fake - cli := steps.NewDeployStep(FakeNamespaceWatcher{}) + cli := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) cli.Init(kubeclient, radixclient, kubeUtil, &monitoring.Clientset{}, rr) applicationConfig := application.NewApplicationConfig(kubeclient, kubeUtil, radixclient, rr, ra, nil) diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index 70a347844..3bbfb4bf2 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -4,14 +4,18 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/equinor/radix-operator/pkg/apis/utils/slice" + radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" + k8errs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/wait" ) // ApplyDeployment Create or update deployment in provided namespace @@ -133,3 +137,49 @@ func (kubeutil *Kube) GetDeployment(namespace, name string) (*appsv1.Deployment, return deployment, nil } + +// RadixDeploymentWatcher Watcher to wait for namespace to be created +type RadixDeploymentWatcher interface { + WaitForActive(namespace, deploymentName string) error +} + +// RadixDeploymentWatcherImpl Implementation of watcher +type RadixDeploymentWatcherImpl struct { + radixClient radixclient.Interface + waitTimeout time.Duration +} + +// NewRadixDeploymentWatcherImpl Constructor +func NewRadixDeploymentWatcherImpl(radixClient radixclient.Interface, waitTimeout time.Duration) RadixDeploymentWatcherImpl { + return RadixDeploymentWatcherImpl{ + radixClient, + waitTimeout, + } +} + +// WaitForActive Waits for the radix deployment gets active +func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentName string) error { + log.Infof("Waiting for RadixDeployment %s gets active in the namespace %s", deploymentName, namespace) + if err := watcher.waitFor(func(context.Context) (bool, error) { + rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + if err != nil { + if k8errs.IsNotFound(err) || k8errs.IsForbidden(err) { + return false, nil + } + return false, err + } + return rd != nil && !rd.Status.ActiveFrom.IsZero(), nil + }); err != nil { + return err + } + + log.Infof("RadixDeployment %s in the namespace %s exists and is active", deploymentName, namespace) + return nil + +} + +func (watcher RadixDeploymentWatcherImpl) waitFor(condition wait.ConditionWithContextFunc) error { + timoutContext, cancel := context.WithTimeout(context.Background(), watcher.waitTimeout) + defer cancel() + return wait.PollUntilContextCancel(timoutContext, time.Second, true, condition) +} From 27315a76f05750fa18e270f93df613c1f118889a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 21 Mar 2024 13:45:11 +0100 Subject: [PATCH 02/13] Cleaned up --- charts/radix-operator/Chart.yaml | 2 +- pipeline-runner/steps/deploy.go | 31 ++++++++++++++-------------- pipeline-runner/steps/deploy_test.go | 12 +++++------ pkg/apis/kube/deployment.go | 8 +++---- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index 67942d34f..751609758 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: radix-operator version: 1.30.5 -appVersion: 1.50.5 +appVersion: 1.50.6 kubeVersion: ">=1.24.0" description: Radix Operator keywords: diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 0852bb0dd..4ea0c1865 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -71,7 +71,7 @@ func (cli *DeployStepImplementation) deploy(pipelineInfo *model.PipelineInfo) er return nil } -func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineInfo *model.PipelineInfo) error { +func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipelineInfo *model.PipelineInfo) error { defaultEnvVars, err := getDefaultEnvVars(pipelineInfo) if err != nil { return fmt.Errorf("failed to retrieve default env vars for RadixDeployment in app %s. %v", appName, err) @@ -91,7 +91,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineIn return err } - currentRd, err := internal.GetCurrentRadixDeployment(cli.GetKubeutil(), utils.GetEnvironmentNamespace(appName, env)) + currentRd, err := internal.GetCurrentRadixDeployment(cli.GetKubeutil(), utils.GetEnvironmentNamespace(appName, envName)) if err != nil { return err } @@ -101,8 +101,8 @@ func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineIn pipelineInfo.PipelineArguments.JobName, pipelineInfo.PipelineArguments.ImageTag, pipelineInfo.PipelineArguments.Branch, - pipelineInfo.DeployEnvironmentComponentImages[env], - env, + pipelineInfo.DeployEnvironmentComponentImages[envName], + envName, defaultEnvVars, radixApplicationHash, buildSecretHash, @@ -110,25 +110,24 @@ func (cli *DeployStepImplementation) deployToEnv(appName, env string, pipelineIn pipelineInfo.PipelineArguments.ComponentsToDeploy) if err != nil { - return fmt.Errorf("failed to create radix deployments objects for app %s. %v", appName, err) + return fmt.Errorf("failed to create radix deployments objects for app %s. %w", appName, err) } - namespace := utils.GetEnvironmentNamespace(cli.GetAppName(), env) - err = cli.namespaceWatcher.WaitFor(namespace) - if err != nil { - return fmt.Errorf("failed to get environment namespace, %s, for app %s. %v", env, appName, err) + namespace := utils.GetEnvironmentNamespace(cli.GetAppName(), envName) + if err = cli.namespaceWatcher.WaitFor(namespace); err != nil { + return fmt.Errorf("failed to get environment namespace %s, for app %s. %w", namespace, appName, err) } - log.Info().Msgf("Apply radix deployment %s on env %s", radixDeployment.GetName(), radixDeployment.GetNamespace()) - _, err = cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Create(context.TODO(), radixDeployment, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to apply radix deployment for app %s to environment %s. %v", appName, env, err) + radixDeploymentName := radixDeployment.GetName() + log.Info().Msgf("Apply radix deployment %s on environment %s", radixDeploymentName, envName) + if _, err = cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Create(context.Background(), radixDeployment, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to apply radix deployment for app %s to environment %s. %w", appName, envName, err) } - log.Infof("Waiting for radix deployment %s on env %s gets active", radixDeployment.GetName(), radixDeployment.GetNamespace()) - if err = cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeployment.GetName()); err != nil { + log.Info().Msgf("Waiting for radix deployment %s on environment %s gets active", radixDeploymentName, envName) + if err = cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { return err } - log.Infof("The radix deployment %s on env %s is active", radixDeployment.GetName(), radixDeployment.GetNamespace()) + log.Info().Msgf("The radix deployment %s on environment %s is active", radixDeploymentName, envName) return nil } diff --git a/pipeline-runner/steps/deploy_test.go b/pipeline-runner/steps/deploy_test.go index 3813371a7..908ac9f17 100644 --- a/pipeline-runner/steps/deploy_test.go +++ b/pipeline-runner/steps/deploy_test.go @@ -45,21 +45,21 @@ func setupTest(t *testing.T) (*kubernetes.Clientset, *kube.Kube, *radix.Clientse return kubeclient, kubeUtil, radixclient, testUtils } -// FakeNamespaceWatcher Unit tests doesn't handle muliti-threading well +// FakeNamespaceWatcher Unit tests doesn't handle multi-threading well type FakeNamespaceWatcher struct { } -// FakeRadixDeploymentWatcher Unit tests doesn't handle muliti-threading well +// FakeRadixDeploymentWatcher Unit tests doesn't handle multi-threading well type FakeRadixDeploymentWatcher struct { } // WaitFor Waits for namespace to appear -func (watcher FakeNamespaceWatcher) WaitFor(namespace string) error { +func (watcher FakeNamespaceWatcher) WaitFor(_ string) error { return nil } // WaitFor Waits for radix deployment gets active -func (watcher FakeRadixDeploymentWatcher) WaitForActive(namespace, deploymentName string) error { +func (watcher FakeRadixDeploymentWatcher) WaitForActive(_, _ string) error { return nil } @@ -186,7 +186,7 @@ func TestDeploy_PromotionSetup_ShouldCreateNamespacesForAllBranchesIfNotExists(t WithEnvironmentVariable("DB_PORT", "9876"))). BuildRA() - // Prometheus doesn´t contain any fake + // Prometheus don´t contain any fake cli := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) cli.Init(kubeclient, radixclient, kubeUtil, &monitoring.Clientset{}, rr) @@ -306,7 +306,7 @@ func TestDeploy_SetCommitID_whenSet(t *testing.T) { WithComponents(utils.AnApplicationComponent().WithName("app")). BuildRA() - // Prometheus doesn´t contain any fake + // Prometheus don´t contain any fake cli := steps.NewDeployStep(FakeNamespaceWatcher{}, FakeRadixDeploymentWatcher{}) cli.Init(kubeclient, radixclient, kubeUtil, &monitoring.Clientset{}, rr) diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index 77b797487..fede4552c 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -7,8 +7,8 @@ import ( "time" "github.com/equinor/radix-operator/pkg/apis/utils/slice" - "github.com/rs/zerolog/log" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" k8errs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -159,9 +159,9 @@ func NewRadixDeploymentWatcherImpl(radixClient radixclient.Interface, waitTimeou // WaitForActive Waits for the radix deployment gets active func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentName string) error { - log.Infof("Waiting for RadixDeployment %s gets active in the namespace %s", deploymentName, namespace) + log.Info().Msgf("Waiting for Radix deployment %s in namespace %s gets active", deploymentName, namespace) if err := watcher.waitFor(func(context.Context) (bool, error) { - rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) if err != nil { if k8errs.IsNotFound(err) || k8errs.IsForbidden(err) { return false, nil @@ -173,7 +173,7 @@ func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentNam return err } - log.Infof("RadixDeployment %s in the namespace %s exists and is active", deploymentName, namespace) + log.Info().Msgf("Radix deployment %s in namespace %s exists and is active", deploymentName, namespace) return nil } From 23837d4b621fa5106ed9e17e0aa0080c841ec6ee Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 21 Mar 2024 16:59:58 +0100 Subject: [PATCH 03/13] Deleting RD, added delete RD permit --- .../templates/radix-pipeline-rbac.yaml | 1 + pipeline-runner/pipelines/app.go | 2 +- pipeline-runner/steps/deploy.go | 12 ++++++++---- pkg/apis/kube/deployment.go | 4 ++-- pkg/apis/kube/namespaces.go | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/charts/radix-operator/templates/radix-pipeline-rbac.yaml b/charts/radix-operator/templates/radix-pipeline-rbac.yaml index e56b7a848..6fedecb24 100644 --- a/charts/radix-operator/templates/radix-pipeline-rbac.yaml +++ b/charts/radix-operator/templates/radix-pipeline-rbac.yaml @@ -73,6 +73,7 @@ rules: - get - list - create + - delete - apiGroups: - "" resources: diff --git a/pipeline-runner/pipelines/app.go b/pipeline-runner/pipelines/app.go index d19b0eb6b..03bff1045 100644 --- a/pipeline-runner/pipelines/app.go +++ b/pipeline-runner/pipelines/app.go @@ -110,7 +110,7 @@ func (cli *PipelineRunner) initStepImplementations(registration *v1.RadixRegistr stepImplementations = append(stepImplementations, steps.NewApplyConfigStep()) stepImplementations = append(stepImplementations, steps.NewBuildStep(nil)) stepImplementations = append(stepImplementations, steps.NewRunPipelinesStep(nil)) - stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient), kube.NewRadixDeploymentWatcherImpl(cli.radixclient, time.Minute*5))) + stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient), kube.NewRadixDeploymentWatcherImpl(cli.radixclient, time.Minute*1))) stepImplementations = append(stepImplementations, steps.NewPromoteStep()) for _, stepImplementation := range stepImplementations { diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 4ea0c1865..b80d9a594 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -110,7 +110,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli pipelineInfo.PipelineArguments.ComponentsToDeploy) if err != nil { - return fmt.Errorf("failed to create radix deployments objects for app %s. %w", appName, err) + return fmt.Errorf("failed to create Radix deployments for app %s. %w", appName, err) } namespace := utils.GetEnvironmentNamespace(cli.GetAppName(), envName) @@ -121,13 +121,17 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli radixDeploymentName := radixDeployment.GetName() log.Info().Msgf("Apply radix deployment %s on environment %s", radixDeploymentName, envName) if _, err = cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Create(context.Background(), radixDeployment, metav1.CreateOptions{}); err != nil { - return fmt.Errorf("failed to apply radix deployment for app %s to environment %s. %w", appName, envName, err) + return fmt.Errorf("failed to apply Radix deployment for app %s to environment %s. %w", appName, envName, err) } - log.Info().Msgf("Waiting for radix deployment %s on environment %s gets active", radixDeploymentName, envName) + if err = cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { + log.Error().Err(err).Msgf("Failed while waiting for the Radix deployment %s on environment %s gets active. Delete this Radix deployment.", radixDeploymentName, envName) + if err := cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Delete(context.Background(), radixDeploymentName, metav1.DeleteOptions{}); err != nil { + // && !k8errs.IsNotFound(err) && !k8errs.IsForbidden(err) { + log.Error().Err(err).Msgf("Failed to delete the Radix deployment.") + } return err } - log.Info().Msgf("The radix deployment %s on environment %s is active", radixDeploymentName, envName) return nil } diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index fede4552c..42427b8d0 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -159,7 +159,7 @@ func NewRadixDeploymentWatcherImpl(radixClient radixclient.Interface, waitTimeou // WaitForActive Waits for the radix deployment gets active func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentName string) error { - log.Info().Msgf("Waiting for Radix deployment %s in namespace %s gets active", deploymentName, namespace) + log.Info().Msgf("Waiting while Radix deployment %s in namespace %s gets active", deploymentName, namespace) if err := watcher.waitFor(func(context.Context) (bool, error) { rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) if err != nil { @@ -173,7 +173,7 @@ func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentNam return err } - log.Info().Msgf("Radix deployment %s in namespace %s exists and is active", deploymentName, namespace) + log.Info().Msgf("Radix deployment %s in namespace %s is active", deploymentName, namespace) return nil } diff --git a/pkg/apis/kube/namespaces.go b/pkg/apis/kube/namespaces.go index f63a19967..24c2136fb 100644 --- a/pkg/apis/kube/namespaces.go +++ b/pkg/apis/kube/namespaces.go @@ -114,7 +114,7 @@ func (watcher NamespaceWatcherImpl) WaitFor(namespace string) error { return err } - log.Info().Msgf("namespace %s exists and is active", namespace) + log.Info().Msgf("Namespace %s exists and is active", namespace) return nil } From bc447e4116194ceada2b302f1896a2192f0bb555 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Thu, 21 Mar 2024 17:02:13 +0100 Subject: [PATCH 04/13] Version --- charts/radix-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/radix-operator/Chart.yaml b/charts/radix-operator/Chart.yaml index 751609758..aca2b58bb 100644 --- a/charts/radix-operator/Chart.yaml +++ b/charts/radix-operator/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: radix-operator -version: 1.30.5 +version: 1.30.6 appVersion: 1.50.6 kubeVersion: ">=1.24.0" description: Radix Operator From e61a688dfc5b7474f7f24460ca90aa323f572cc7 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:56:12 +0100 Subject: [PATCH 05/13] Update pkg/apis/kube/deployment.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pkg/apis/kube/deployment.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index 42427b8d0..5777d7cdd 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -163,9 +163,6 @@ func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentNam if err := watcher.waitFor(func(context.Context) (bool, error) { rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) if err != nil { - if k8errs.IsNotFound(err) || k8errs.IsForbidden(err) { - return false, nil - } return false, err } return rd != nil && !rd.Status.ActiveFrom.IsZero(), nil From 8fee2a9d2552344147f16b1080c0017d00d021e3 Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:56:51 +0100 Subject: [PATCH 06/13] Update pkg/apis/kube/deployment.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pkg/apis/kube/deployment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index 5777d7cdd..6b5c509c6 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -159,7 +159,7 @@ func NewRadixDeploymentWatcherImpl(radixClient radixclient.Interface, waitTimeou // WaitForActive Waits for the radix deployment gets active func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentName string) error { - log.Info().Msgf("Waiting while Radix deployment %s in namespace %s gets active", deploymentName, namespace) + log.Info().Msgf("Waiting for Radix deployment %s to activate", deploymentName) if err := watcher.waitFor(func(context.Context) (bool, error) { rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) if err != nil { From a93f88e57f0e35d70b76d96599e68214544dcb2a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:57:08 +0100 Subject: [PATCH 07/13] Update pipeline-runner/steps/deploy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pipeline-runner/steps/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index b80d9a594..6f08c81b6 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -119,7 +119,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli } radixDeploymentName := radixDeployment.GetName() - log.Info().Msgf("Apply radix deployment %s on environment %s", radixDeploymentName, envName) + log.Info().Msgf("Apply Radix deployment %s to environment %s", radixDeploymentName, envName) if _, err = cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Create(context.Background(), radixDeployment, metav1.CreateOptions{}); err != nil { return fmt.Errorf("failed to apply Radix deployment for app %s to environment %s. %w", appName, envName, err) } From 33a99c009f52944bb5276fface24c9cfdbc722ad Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:57:18 +0100 Subject: [PATCH 08/13] Update pipeline-runner/steps/deploy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pipeline-runner/steps/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 6f08c81b6..c0694a472 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -110,7 +110,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli pipelineInfo.PipelineArguments.ComponentsToDeploy) if err != nil { - return fmt.Errorf("failed to create Radix deployments for app %s. %w", appName, err) + return fmt.Errorf("failed to create Radix deployment in environment %s. %w", envName, err) } namespace := utils.GetEnvironmentNamespace(cli.GetAppName(), envName) From 0edf2560f9126f9a3ccc99e52a1630f65b32499a Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:57:26 +0100 Subject: [PATCH 09/13] Update pipeline-runner/steps/deploy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pipeline-runner/steps/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index c0694a472..6eebb85fa 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -124,7 +124,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli return fmt.Errorf("failed to apply Radix deployment for app %s to environment %s. %w", appName, envName, err) } - if err = cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { + if err := cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { log.Error().Err(err).Msgf("Failed while waiting for the Radix deployment %s on environment %s gets active. Delete this Radix deployment.", radixDeploymentName, envName) if err := cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Delete(context.Background(), radixDeploymentName, metav1.DeleteOptions{}); err != nil { // && !k8errs.IsNotFound(err) && !k8errs.IsForbidden(err) { From 0d8fc7b348874c01d3fb6beac37f618df4a21e9c Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:57:47 +0100 Subject: [PATCH 10/13] Update pipeline-runner/steps/deploy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pipeline-runner/steps/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 6eebb85fa..80eb7dfbd 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -128,7 +128,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli log.Error().Err(err).Msgf("Failed while waiting for the Radix deployment %s on environment %s gets active. Delete this Radix deployment.", radixDeploymentName, envName) if err := cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Delete(context.Background(), radixDeploymentName, metav1.DeleteOptions{}); err != nil { // && !k8errs.IsNotFound(err) && !k8errs.IsForbidden(err) { - log.Error().Err(err).Msgf("Failed to delete the Radix deployment.") + log.Error().Err(err).Msgf("Failed to delete Radix deployment") } return err } From d3e9118fadb9b7f0d21f34937c024f32defc2a7e Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 09:57:59 +0100 Subject: [PATCH 11/13] Update pipeline-runner/steps/deploy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nils Gustav Stråbø <65334626+nilsgstrabo@users.noreply.github.com> --- pipeline-runner/steps/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 80eb7dfbd..444843186 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -125,7 +125,7 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli } if err := cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { - log.Error().Err(err).Msgf("Failed while waiting for the Radix deployment %s on environment %s gets active. Delete this Radix deployment.", radixDeploymentName, envName) + log.Error().Err(err).Msgf("Failed to activate Radix deployment %s in environment %s. Deleting deployment", radixDeploymentName, envName) if err := cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Delete(context.Background(), radixDeploymentName, metav1.DeleteOptions{}); err != nil { // && !k8errs.IsNotFound(err) && !k8errs.IsForbidden(err) { log.Error().Err(err).Msgf("Failed to delete Radix deployment") From 7fe788f1ffd847ab401ac1ff29604d101ae35e0b Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 11:23:49 +0100 Subject: [PATCH 12/13] Moved RadixDeploymentWatcher to radix-pipeline. Added unit-test --- Makefile | 1 + .../watcher/radix_deployment_watcher.go | 54 +++++++++++++ .../watcher/radix_deployment_watcher_mock.go | 48 ++++++++++++ pipeline-runner/pipelines/app.go | 3 +- pipeline-runner/steps/deploy.go | 6 +- pipeline-runner/steps/deploy_test.go | 75 +++++++++++++++++++ pkg/apis/kube/deployment.go | 47 ------------ 7 files changed, 183 insertions(+), 51 deletions(-) create mode 100644 pipeline-runner/internal/watcher/radix_deployment_watcher.go create mode 100644 pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go diff --git a/Makefile b/Makefile index a6d36683b..5fc8bac2c 100644 --- a/Makefile +++ b/Makefile @@ -100,6 +100,7 @@ mocks: mockgen -source ./radix-operator/dnsalias/internal/syncerfactory.go -destination ./radix-operator/dnsalias/internal/syncerfactory_mock.go -package internal mockgen -source ./radix-operator/common/handler.go -destination ./radix-operator/common/handler_mock.go -package common mockgen -source ./pipeline-runner/internal/wait/job.go -destination ./pipeline-runner/internal/wait/job_mock.go -package wait + mockgen -source ./pipeline-runner/internal/watcher/radix_deployment_watcher.go -destination ./pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go -package watcher .PHONY: build-pipeline build-pipeline: diff --git a/pipeline-runner/internal/watcher/radix_deployment_watcher.go b/pipeline-runner/internal/watcher/radix_deployment_watcher.go new file mode 100644 index 000000000..1f4f7280b --- /dev/null +++ b/pipeline-runner/internal/watcher/radix_deployment_watcher.go @@ -0,0 +1,54 @@ +package watcher + +import ( + "context" + "time" + + radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + "github.com/rs/zerolog/log" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// RadixDeploymentWatcher Watcher to wait for namespace to be created +type RadixDeploymentWatcher interface { + WaitForActive(namespace, deploymentName string) error +} + +// radixDeploymentWatcher Implementation of watcher +type radixDeploymentWatcher struct { + radixClient radixclient.Interface + waitTimeout time.Duration +} + +// NewRadixDeploymentWatcher Constructor +func NewRadixDeploymentWatcher(radixClient radixclient.Interface, waitTimeout time.Duration) RadixDeploymentWatcher { + return &radixDeploymentWatcher{ + radixClient, + waitTimeout, + } +} + +// WaitForActive Waits for the radix deployment gets active +func (watcher radixDeploymentWatcher) WaitForActive(namespace, deploymentName string) error { + log.Info().Msgf("Waiting for Radix deployment %s to activate", deploymentName) + if err := watcher.waitFor(func(context.Context) (bool, error) { + rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return rd != nil && !rd.Status.ActiveFrom.IsZero(), nil + }); err != nil { + return err + } + + log.Info().Msgf("Radix deployment %s in namespace %s is active", deploymentName, namespace) + return nil + +} + +func (watcher radixDeploymentWatcher) waitFor(condition wait.ConditionWithContextFunc) error { + timoutContext, cancel := context.WithTimeout(context.Background(), watcher.waitTimeout) + defer cancel() + return wait.PollUntilContextCancel(timoutContext, time.Second, true, condition) +} diff --git a/pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go b/pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go new file mode 100644 index 000000000..b6833aa2b --- /dev/null +++ b/pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pipeline-runner/internal/watcher/radix_deployment_watcher.go + +// Package watcher is a generated GoMock package. +package watcher + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockRadixDeploymentWatcher is a mock of RadixDeploymentWatcher interface. +type MockRadixDeploymentWatcher struct { + ctrl *gomock.Controller + recorder *MockRadixDeploymentWatcherMockRecorder +} + +// MockRadixDeploymentWatcherMockRecorder is the mock recorder for MockRadixDeploymentWatcher. +type MockRadixDeploymentWatcherMockRecorder struct { + mock *MockRadixDeploymentWatcher +} + +// NewMockRadixDeploymentWatcher creates a new mock instance. +func NewMockRadixDeploymentWatcher(ctrl *gomock.Controller) *MockRadixDeploymentWatcher { + mock := &MockRadixDeploymentWatcher{ctrl: ctrl} + mock.recorder = &MockRadixDeploymentWatcherMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRadixDeploymentWatcher) EXPECT() *MockRadixDeploymentWatcherMockRecorder { + return m.recorder +} + +// WaitForActive mocks base method. +func (m *MockRadixDeploymentWatcher) WaitForActive(namespace, deploymentName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitForActive", namespace, deploymentName) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitForActive indicates an expected call of WaitForActive. +func (mr *MockRadixDeploymentWatcherMockRecorder) WaitForActive(namespace, deploymentName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForActive", reflect.TypeOf((*MockRadixDeploymentWatcher)(nil).WaitForActive), namespace, deploymentName) +} diff --git a/pipeline-runner/pipelines/app.go b/pipeline-runner/pipelines/app.go index 03bff1045..c439f7fce 100644 --- a/pipeline-runner/pipelines/app.go +++ b/pipeline-runner/pipelines/app.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/equinor/radix-operator/pipeline-runner/internal/watcher" "github.com/equinor/radix-operator/pipeline-runner/model" "github.com/equinor/radix-operator/pipeline-runner/steps" jobs "github.com/equinor/radix-operator/pkg/apis/job" @@ -110,7 +111,7 @@ func (cli *PipelineRunner) initStepImplementations(registration *v1.RadixRegistr stepImplementations = append(stepImplementations, steps.NewApplyConfigStep()) stepImplementations = append(stepImplementations, steps.NewBuildStep(nil)) stepImplementations = append(stepImplementations, steps.NewRunPipelinesStep(nil)) - stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient), kube.NewRadixDeploymentWatcherImpl(cli.radixclient, time.Minute*1))) + stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient), watcher.NewRadixDeploymentWatcher(cli.radixclient, time.Minute*5))) stepImplementations = append(stepImplementations, steps.NewPromoteStep()) for _, stepImplementation := range stepImplementations { diff --git a/pipeline-runner/steps/deploy.go b/pipeline-runner/steps/deploy.go index 444843186..7bb336116 100644 --- a/pipeline-runner/steps/deploy.go +++ b/pipeline-runner/steps/deploy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + pipelineRunnerInternal "github.com/equinor/radix-operator/pipeline-runner/internal/watcher" "github.com/equinor/radix-operator/pipeline-runner/model" "github.com/equinor/radix-operator/pipeline-runner/steps/internal" "github.com/equinor/radix-operator/pkg/apis/defaults" @@ -19,12 +20,12 @@ import ( type DeployStepImplementation struct { stepType pipeline.StepType namespaceWatcher kube.NamespaceWatcher - radixDeploymentWatcher kube.RadixDeploymentWatcher + radixDeploymentWatcher pipelineRunnerInternal.RadixDeploymentWatcher model.DefaultStepImplementation } // NewDeployStep Constructor -func NewDeployStep(namespaceWatcher kube.NamespaceWatcher, radixDeploymentWatcher kube.RadixDeploymentWatcher) model.Step { +func NewDeployStep(namespaceWatcher kube.NamespaceWatcher, radixDeploymentWatcher pipelineRunnerInternal.RadixDeploymentWatcher) model.Step { return &DeployStepImplementation{ stepType: pipeline.DeployStep, namespaceWatcher: namespaceWatcher, @@ -127,7 +128,6 @@ func (cli *DeployStepImplementation) deployToEnv(appName, envName string, pipeli if err := cli.radixDeploymentWatcher.WaitForActive(namespace, radixDeploymentName); err != nil { log.Error().Err(err).Msgf("Failed to activate Radix deployment %s in environment %s. Deleting deployment", radixDeploymentName, envName) if err := cli.GetRadixclient().RadixV1().RadixDeployments(radixDeployment.GetNamespace()).Delete(context.Background(), radixDeploymentName, metav1.DeleteOptions{}); err != nil { - // && !k8errs.IsNotFound(err) && !k8errs.IsForbidden(err) { log.Error().Err(err).Msgf("Failed to delete Radix deployment") } return err diff --git a/pipeline-runner/steps/deploy_test.go b/pipeline-runner/steps/deploy_test.go index 908ac9f17..43719bc88 100644 --- a/pipeline-runner/steps/deploy_test.go +++ b/pipeline-runner/steps/deploy_test.go @@ -2,13 +2,17 @@ package steps_test import ( "context" + "errors" "fmt" + "strings" "testing" + "github.com/equinor/radix-operator/pipeline-runner/internal/watcher" "github.com/equinor/radix-operator/pkg/apis/config/dnsalias" "github.com/equinor/radix-operator/pkg/apis/defaults" commonTest "github.com/equinor/radix-operator/pkg/apis/test" radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" secretproviderfake "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake" @@ -340,3 +344,74 @@ func TestDeploy_SetCommitID_whenSet(t *testing.T) { rd := rds.Items[0] assert.Equal(t, commitID, rd.ObjectMeta.Labels[kube.RadixCommitLabel]) } + +func TestDeploy_WaitActiveDeployment(t *testing.T) { + rr := utils.ARadixRegistration(). + WithName(anyAppName). + BuildRR() + + envName := "dev" + ra := utils.NewRadixApplicationBuilder(). + WithAppName(anyAppName). + WithEnvironment(envName, "master"). + WithComponents(utils.AnApplicationComponent().WithName("app")). + BuildRA() + + type scenario struct { + name string + watcherError error + expectedRadixDeployments int + } + scenarios := []scenario{ + {name: "No fail", watcherError: nil, expectedRadixDeployments: 1}, + {name: "Watch fails", watcherError: errors.New("some error"), expectedRadixDeployments: 0}, + } + for _, ts := range scenarios { + t.Run(ts.name, func(tt *testing.T) { + kubeclient, kubeUtil, radixClient, _ := setupTest(t) + ctrl := gomock.NewController(tt) + radixDeploymentWatcher := watcher.NewMockRadixDeploymentWatcher(ctrl) + cli := steps.NewDeployStep(FakeNamespaceWatcher{}, radixDeploymentWatcher) + cli.Init(kubeclient, radixClient, kubeUtil, &monitoring.Clientset{}, rr) + + applicationConfig := application.NewApplicationConfig(kubeclient, kubeUtil, radixClient, rr, ra, nil) + + pipelineInfo := &model.PipelineInfo{ + PipelineArguments: model.PipelineArguments{ + JobName: anyJobName, + ImageTag: anyImageTag, + Branch: "master", + }, + TargetEnvironments: []string{envName}, + } + + pipelineInfo.SetApplicationConfig(applicationConfig) + namespace := utils.GetEnvironmentNamespace(anyAppName, envName) + radixDeploymentWatcher.EXPECT(). + WaitForActive(namespace, radixDeploymentNameMatcher{envName: envName, imageTag: anyImageTag}). + Return(ts.watcherError) + err := cli.Run(pipelineInfo) + if ts.watcherError == nil { + assert.NoError(tt, err) + } else { + assert.EqualError(tt, err, ts.watcherError.Error()) + } + rdList, err := radixClient.RadixV1().RadixDeployments(namespace).List(context.Background(), metav1.ListOptions{}) + assert.NoError(tt, err) + assert.Len(tt, rdList.Items, ts.expectedRadixDeployments, "Invalid expected RadixDeployment-s count") + }) + } +} + +type radixDeploymentNameMatcher struct { + envName string + imageTag string +} + +func (m radixDeploymentNameMatcher) Matches(name interface{}) bool { + rdName, ok := name.(string) + return ok && strings.HasPrefix(rdName, m.String()) +} +func (m radixDeploymentNameMatcher) String() string { + return fmt.Sprintf("%s-%s", m.envName, m.imageTag) +} diff --git a/pkg/apis/kube/deployment.go b/pkg/apis/kube/deployment.go index 6b5c509c6..3c82cda69 100644 --- a/pkg/apis/kube/deployment.go +++ b/pkg/apis/kube/deployment.go @@ -4,18 +4,14 @@ import ( "context" "encoding/json" "fmt" - "time" "github.com/equinor/radix-operator/pkg/apis/utils/slice" - radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" appsv1 "k8s.io/api/apps/v1" - k8errs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/apimachinery/pkg/util/wait" ) // ApplyDeployment Create or update deployment in provided namespace @@ -137,46 +133,3 @@ func (kubeutil *Kube) GetDeployment(namespace, name string) (*appsv1.Deployment, return deployment, nil } - -// RadixDeploymentWatcher Watcher to wait for namespace to be created -type RadixDeploymentWatcher interface { - WaitForActive(namespace, deploymentName string) error -} - -// RadixDeploymentWatcherImpl Implementation of watcher -type RadixDeploymentWatcherImpl struct { - radixClient radixclient.Interface - waitTimeout time.Duration -} - -// NewRadixDeploymentWatcherImpl Constructor -func NewRadixDeploymentWatcherImpl(radixClient radixclient.Interface, waitTimeout time.Duration) RadixDeploymentWatcherImpl { - return RadixDeploymentWatcherImpl{ - radixClient, - waitTimeout, - } -} - -// WaitForActive Waits for the radix deployment gets active -func (watcher RadixDeploymentWatcherImpl) WaitForActive(namespace, deploymentName string) error { - log.Info().Msgf("Waiting for Radix deployment %s to activate", deploymentName) - if err := watcher.waitFor(func(context.Context) (bool, error) { - rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) - if err != nil { - return false, err - } - return rd != nil && !rd.Status.ActiveFrom.IsZero(), nil - }); err != nil { - return err - } - - log.Info().Msgf("Radix deployment %s in namespace %s is active", deploymentName, namespace) - return nil - -} - -func (watcher RadixDeploymentWatcherImpl) waitFor(condition wait.ConditionWithContextFunc) error { - timoutContext, cancel := context.WithTimeout(context.Background(), watcher.waitTimeout) - defer cancel() - return wait.PollUntilContextCancel(timoutContext, time.Second, true, condition) -} From 8055ef0e9c04aa71423c45c8301b5587effed52c Mon Sep 17 00:00:00 2001 From: Sergey Smolnikov Date: Fri, 22 Mar 2024 13:02:14 +0100 Subject: [PATCH 13/13] Added unit-test --- .../watcher/radix_deployment_watcher_test.go | 68 +++++++++++++++++++ pipeline-runner/steps/deploy_test.go | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 pipeline-runner/internal/watcher/radix_deployment_watcher_test.go diff --git a/pipeline-runner/internal/watcher/radix_deployment_watcher_test.go b/pipeline-runner/internal/watcher/radix_deployment_watcher_test.go new file mode 100644 index 000000000..d75ef0483 --- /dev/null +++ b/pipeline-runner/internal/watcher/radix_deployment_watcher_test.go @@ -0,0 +1,68 @@ +package watcher + +import ( + "context" + "errors" + "testing" + "time" + + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetes "k8s.io/client-go/kubernetes/fake" +) + +func setupTest(t *testing.T) (*radix.Clientset, *kubernetes.Clientset) { + radixClient := radix.NewSimpleClientset() + kubeClient := kubernetes.NewSimpleClientset() + return radixClient, kubeClient +} + +func TestDeploy_WaitActiveDeployment(t *testing.T) { + const ( + namespace = "app-dev" + radixDeploymentName = "any-rd-name" + ) + type scenario struct { + name string + hasRadixDevelopment bool + radixDeploymentStatus radixv1.RadixDeployStatus + watcherError error + } + scenarios := []scenario{ + {name: "Active deployment, no fail", hasRadixDevelopment: true, radixDeploymentStatus: radixv1.RadixDeployStatus{ActiveFrom: metav1.Time{Time: time.Now()}}, watcherError: nil}, + {name: "No active radix deployment, fail", hasRadixDevelopment: true, radixDeploymentStatus: radixv1.RadixDeployStatus{}, watcherError: errors.New("context deadline exceeded")}, + {name: "No radix deployment, fail", hasRadixDevelopment: false, watcherError: errors.New("radixdeployments.radix.equinor.com \"any-rd-name\" not found")}, + } + for _, ts := range scenarios { + t.Run(ts.name, func(tt *testing.T) { + radixClient, kubeClient := setupTest(tt) + require.NoError(t, createNamespace(kubeClient, namespace)) + + if ts.hasRadixDevelopment { + _, err := radixClient.RadixV1().RadixDeployments(namespace).Create(context.Background(), &radixv1.RadixDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: radixDeploymentName}, + Status: ts.radixDeploymentStatus, + }, metav1.CreateOptions{}) + require.NoError(tt, err) + } + + watcher := NewRadixDeploymentWatcher(radixClient, time.Millisecond*10) + err := watcher.WaitForActive(namespace, radixDeploymentName) + + if ts.watcherError == nil { + assert.NoError(tt, err) + } else { + assert.EqualError(tt, err, ts.watcherError.Error()) + } + }) + } +} + +func createNamespace(kubeClient *kubernetes.Clientset, namespace string) error { + _, err := kubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) + return err +} diff --git a/pipeline-runner/steps/deploy_test.go b/pipeline-runner/steps/deploy_test.go index 43719bc88..a41e6d9e5 100644 --- a/pipeline-runner/steps/deploy_test.go +++ b/pipeline-runner/steps/deploy_test.go @@ -368,7 +368,7 @@ func TestDeploy_WaitActiveDeployment(t *testing.T) { } for _, ts := range scenarios { t.Run(ts.name, func(tt *testing.T) { - kubeclient, kubeUtil, radixClient, _ := setupTest(t) + kubeclient, kubeUtil, radixClient, _ := setupTest(tt) ctrl := gomock.NewController(tt) radixDeploymentWatcher := watcher.NewMockRadixDeploymentWatcher(ctrl) cli := steps.NewDeployStep(FakeNamespaceWatcher{}, radixDeploymentWatcher)