Merge pull request #1077 from equinor/master

Restarting aux deployment with job scheduler (#1076)

charts/radix-operator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: radix-operator
-version: 1.30.7
-appVersion: 1.50.7
+version: 1.30.8
+appVersion: 1.50.8
 kubeVersion: ">=1.24.0"
 description: Radix Operator
 keywords:

pkg/apis/deployment/deployment.go

@@ -12,6 +12,7 @@ import (
 	certclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
 	"github.com/equinor/radix-common/utils/slice"
 	"github.com/equinor/radix-operator/pkg/apis/config"
+	"github.com/equinor/radix-operator/pkg/apis/defaults"
 	"github.com/equinor/radix-operator/pkg/apis/ingress"
 	"github.com/equinor/radix-operator/pkg/apis/kube"
 	"github.com/equinor/radix-operator/pkg/apis/metrics"
@@ -21,6 +22,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -598,10 +600,23 @@ func (deploy *Deployment) createOrUpdateJobAuxDeployment(deployComponent v1.Radi
 	// Remove volumes and volume mounts from job scheduler deployment
 	desiredDeployment.Spec.Template.Spec.Volumes = nil
 	desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = nil
-
+	syncRadixRestartEnvironmentVariable(deployComponent, desiredJobAuxDeployment)
 	return currentJobAuxDeployment, desiredJobAuxDeployment, nil
 }
 
+func syncRadixRestartEnvironmentVariable(deployComponent v1.RadixCommonDeployComponent, desiredJobAuxDeployment *appsv1.Deployment) {
+	auxDeploymentEnvVars := desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Env
+	if restartComponentValue, ok := deployComponent.GetEnvironmentVariables()[defaults.RadixRestartEnvironmentVariable]; ok {
+		if index := slice.FindIndex(auxDeploymentEnvVars, func(envVar corev1.EnvVar) bool {
+			return strings.EqualFold(envVar.Name, defaults.RadixRestartEnvironmentVariable)
+		}); index >= 0 {
+			desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Env[index].Value = restartComponentValue
+			return
+		}
+		desiredJobAuxDeployment.Spec.Template.Spec.Containers[0].Env = append(auxDeploymentEnvVars, corev1.EnvVar{Name: defaults.RadixRestartEnvironmentVariable, Value: restartComponentValue})
+	}
+}
+
 func (deploy *Deployment) getCurrentAndDesiredJobAuxDeployment(deployComponent v1.RadixCommonDeployComponent, desiredDeployment *appsv1.Deployment) (*appsv1.Deployment, *appsv1.Deployment, error) {
 	currentJobAuxDeployment, err := deploy.kubeutil.GetDeployment(desiredDeployment.Namespace, getJobAuxObjectName(desiredDeployment.Name))
 	if err != nil {

pkg/apis/deployment/deployment_test.go

@@ -3799,6 +3799,67 @@ func Test_JobSynced_SecretRefs(t *testing.T) {
 	assert.False(t, envVariableByNameExistOnDeployment("SECRET2", jobName, jobDeployments), "SECRET2 exist")
 }
 
+func Test_RestartJobManager_RestartsAuxDeployment(t *testing.T) {
+	appName, environment, jobName := "app", "dev", "job"
+	tu, client, kubeUtil, radixclient, prometheusclient, _, certClient := setupTest(t)
+	defer teardownTest()
+	// Setup
+	os.Setenv(defaults.ActiveClusternameEnvironmentVariable, testClusterName)
+
+	applicationBuilder := utils.NewRadixApplicationBuilder().WithAppName(appName).WithRadixRegistration(utils.NewRegistrationBuilder().WithName(appName))
+	jobComponentBuilder := utils.NewDeployJobComponentBuilder().WithName(jobName)
+	_, err := applyDeploymentWithSync(tu, client, kubeUtil, radixclient, prometheusclient, certClient,
+		utils.NewDeploymentBuilder().
+			WithRadixApplication(
+				applicationBuilder,
+			).
+			WithAppName(appName).
+			WithEnvironment(environment).
+			WithJobComponent(
+				jobComponentBuilder,
+			),
+	)
+	require.NoError(t, err)
+
+	envNamespace := utils.GetEnvironmentNamespace(appName, environment)
+	allDeployments, _ := client.AppsV1().Deployments(envNamespace).List(context.Background(), metav1.ListOptions{})
+	jobDeployments := getDeploymentsForRadixComponents(allDeployments.Items)
+	jobAuxDeployments := getDeploymentsForRadixJobAux(allDeployments.Items)
+	require.Len(t, jobDeployments, 1)
+	require.Len(t, jobAuxDeployments, 1)
+	require.Equal(t, "job", jobDeployments[0].Name)
+	require.Equal(t, "job-aux", jobAuxDeployments[0].Name)
+	restartTimestamp := "some-time-stamp"
+	require.False(t, slice.Any(jobDeployments[0].Spec.Template.Spec.Containers[0].Env, func(envVar corev1.EnvVar) bool {
+		return envVar.Name == defaults.RadixRestartEnvironmentVariable && envVar.Value == restartTimestamp
+	}), "Not found restart env var in the job deployment")
+	require.False(t, slice.Any(jobAuxDeployments[0].Spec.Template.Spec.Containers[0].Env, func(envVar corev1.EnvVar) bool {
+		return envVar.Name == defaults.RadixRestartEnvironmentVariable && envVar.Value == restartTimestamp
+	}), "Not found restart env var in the job aux deployment")
+
+	_, err = applyDeploymentWithSync(tu, client, kubeUtil, radixclient, prometheusclient, certClient,
+		utils.NewDeploymentBuilder().
+			WithRadixApplication(applicationBuilder).
+			WithAppName(appName).
+			WithEnvironment(environment).
+			WithJobComponent(jobComponentBuilder.WithEnvironmentVariable(defaults.RadixRestartEnvironmentVariable, restartTimestamp)),
+	)
+	require.NoError(t, err)
+
+	allDeployments, _ = client.AppsV1().Deployments(envNamespace).List(context.Background(), metav1.ListOptions{})
+	jobDeployments = getDeploymentsForRadixComponents(allDeployments.Items)
+	jobAuxDeployments = getDeploymentsForRadixJobAux(allDeployments.Items)
+	require.Len(t, jobDeployments, 1)
+	require.Len(t, jobAuxDeployments, 1)
+	require.Equal(t, "job", jobDeployments[0].Name)
+	require.True(t, slice.Any(jobDeployments[0].Spec.Template.Spec.Containers[0].Env, func(envVar corev1.EnvVar) bool {
+		return envVar.Name == defaults.RadixRestartEnvironmentVariable && envVar.Value == restartTimestamp
+	}), "Not found restart env var in the job deployment")
+	require.True(t, slice.Any(jobAuxDeployments[0].Spec.Template.Spec.Containers[0].Env, func(envVar corev1.EnvVar) bool {
+		return envVar.Name == defaults.RadixRestartEnvironmentVariable && envVar.Value == restartTimestamp
+	}), "Not found restart env var in the job aux deployment")
+}
+
 func TestRadixBatch_IsGarbageCollected(t *testing.T) {
 	// Setup
 	tu, client, kubeUtil, radixclient, prometheusclient, _, certClient := setupTest(t)