-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix pipeline job deployment race condition (#1073)
* Added waiting for new radix deployment to gets active to finish the pipeline job * Cleaned up * Deleting RD, added delete RD permit * Version * Update pkg/apis/kube/deployment.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pkg/apis/kube/deployment.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pipeline-runner/steps/deploy.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pipeline-runner/steps/deploy.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pipeline-runner/steps/deploy.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pipeline-runner/steps/deploy.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Update pipeline-runner/steps/deploy.go Co-authored-by: Nils Gustav Stråbø <[email protected]> * Moved RadixDeploymentWatcher to radix-pipeline. Added unit-test * Added unit-test --------- Co-authored-by: Nils Gustav Stråbø <[email protected]>
- Loading branch information
1 parent
75b8bd3
commit 2884277
Showing
11 changed files
with
305 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -73,6 +73,7 @@ rules: | |
| - get | ||
| - list | ||
| - create | ||
| - delete | ||
| - apiGroups: | ||
| - "" | ||
| resources: | ||
|
|
||
54 changes: 54 additions & 0 deletions
54
pipeline-runner/internal/watcher/radix_deployment_watcher.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package watcher | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" | ||
| "github.com/rs/zerolog/log" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| ) | ||
|
|
||
| // RadixDeploymentWatcher Watcher to wait for namespace to be created | ||
| type RadixDeploymentWatcher interface { | ||
| WaitForActive(namespace, deploymentName string) error | ||
| } | ||
|
|
||
| // radixDeploymentWatcher Implementation of watcher | ||
| type radixDeploymentWatcher struct { | ||
| radixClient radixclient.Interface | ||
| waitTimeout time.Duration | ||
| } | ||
|
|
||
| // NewRadixDeploymentWatcher Constructor | ||
| func NewRadixDeploymentWatcher(radixClient radixclient.Interface, waitTimeout time.Duration) RadixDeploymentWatcher { | ||
| return &radixDeploymentWatcher{ | ||
| radixClient, | ||
| waitTimeout, | ||
| } | ||
| } | ||
|
|
||
| // WaitForActive Waits for the radix deployment gets active | ||
| func (watcher radixDeploymentWatcher) WaitForActive(namespace, deploymentName string) error { | ||
| log.Info().Msgf("Waiting for Radix deployment %s to activate", deploymentName) | ||
| if err := watcher.waitFor(func(context.Context) (bool, error) { | ||
| rd, err := watcher.radixClient.RadixV1().RadixDeployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{}) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| return rd != nil && !rd.Status.ActiveFrom.IsZero(), nil | ||
| }); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| log.Info().Msgf("Radix deployment %s in namespace %s is active", deploymentName, namespace) | ||
| return nil | ||
|
|
||
| } | ||
|
|
||
| func (watcher radixDeploymentWatcher) waitFor(condition wait.ConditionWithContextFunc) error { | ||
| timoutContext, cancel := context.WithTimeout(context.Background(), watcher.waitTimeout) | ||
| defer cancel() | ||
| return wait.PollUntilContextCancel(timoutContext, time.Second, true, condition) | ||
| } |
48 changes: 48 additions & 0 deletions
48
pipeline-runner/internal/watcher/radix_deployment_watcher_mock.go
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
68 changes: 68 additions & 0 deletions
68
pipeline-runner/internal/watcher/radix_deployment_watcher_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package watcher | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "testing" | ||
| "time" | ||
|
|
||
| radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" | ||
| radix "github.com/equinor/radix-operator/pkg/client/clientset/versioned/fake" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| kubernetes "k8s.io/client-go/kubernetes/fake" | ||
| ) | ||
|
|
||
| func setupTest(t *testing.T) (*radix.Clientset, *kubernetes.Clientset) { | ||
| radixClient := radix.NewSimpleClientset() | ||
| kubeClient := kubernetes.NewSimpleClientset() | ||
| return radixClient, kubeClient | ||
| } | ||
|
|
||
| func TestDeploy_WaitActiveDeployment(t *testing.T) { | ||
| const ( | ||
| namespace = "app-dev" | ||
| radixDeploymentName = "any-rd-name" | ||
| ) | ||
| type scenario struct { | ||
| name string | ||
| hasRadixDevelopment bool | ||
| radixDeploymentStatus radixv1.RadixDeployStatus | ||
| watcherError error | ||
| } | ||
| scenarios := []scenario{ | ||
| {name: "Active deployment, no fail", hasRadixDevelopment: true, radixDeploymentStatus: radixv1.RadixDeployStatus{ActiveFrom: metav1.Time{Time: time.Now()}}, watcherError: nil}, | ||
| {name: "No active radix deployment, fail", hasRadixDevelopment: true, radixDeploymentStatus: radixv1.RadixDeployStatus{}, watcherError: errors.New("context deadline exceeded")}, | ||
| {name: "No radix deployment, fail", hasRadixDevelopment: false, watcherError: errors.New("radixdeployments.radix.equinor.com \"any-rd-name\" not found")}, | ||
| } | ||
| for _, ts := range scenarios { | ||
| t.Run(ts.name, func(tt *testing.T) { | ||
| radixClient, kubeClient := setupTest(tt) | ||
| require.NoError(t, createNamespace(kubeClient, namespace)) | ||
|
|
||
| if ts.hasRadixDevelopment { | ||
| _, err := radixClient.RadixV1().RadixDeployments(namespace).Create(context.Background(), &radixv1.RadixDeployment{ | ||
| ObjectMeta: metav1.ObjectMeta{Name: radixDeploymentName}, | ||
| Status: ts.radixDeploymentStatus, | ||
| }, metav1.CreateOptions{}) | ||
| require.NoError(tt, err) | ||
| } | ||
|
|
||
| watcher := NewRadixDeploymentWatcher(radixClient, time.Millisecond*10) | ||
| err := watcher.WaitForActive(namespace, radixDeploymentName) | ||
|
|
||
| if ts.watcherError == nil { | ||
| assert.NoError(tt, err) | ||
| } else { | ||
| assert.EqualError(tt, err, ts.watcherError.Error()) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func createNamespace(kubeClient *kubernetes.Clientset, namespace string) error { | ||
| _, err := kubeClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}) | ||
| return err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.