🐛 [patch] Fix applying empty target namespace (#9)

* 🐛 Fix applying empty target namespace * 🐛 Fix fulfilled error * 🎨 Format code Co-authored-by: Natcha Wattanaarunkij <[email protected]>
agoda-com · Apr 2, 2020 · 7cb1d97 · 7cb1d97
1 parent 3428d8c
commit 7cb1d97
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 36 deletions.
diff --git a/internal/errors/errors.go b/internal/errors/errors.go
@@ -17,21 +17,21 @@ const (
 	ErrImageVersionNotFound      = Error("image version not found")
 	ErrNoDesiredComponentVersion = Error("no desired component version")
 
-	ErrTeamNotFound               = Error("team not found")
 	ErrTeamNamespaceStillCreating = Error("still creating namespace")
 	ErrTeamNamespaceStillExists   = Error("destroyed namespace still exists")
 
-	ErrActivePromotionTimeout         = Error("active promotion timeout")
-	ErrActiveDemotionTimeout          = Error("demoted active environment timeout")
-	ErrRollbackActivePromotionTimeout = Error("rollback active promotion timeout")
-	ErrEnsureNamespaceDestroyed       = Error("namespace has not been destroyed")
-	ErrEnsureActiveDemoted            = Error("active environment has been being demoted")
-	ErrEnsureActivePromoted           = Error("active environment has been being promoted")
-	ErrEnsureComponentDeployed        = Error("components has been being deployed")
-	ErrEnsureComponentTested          = Error("components has been being tested")
-	ErrDeletingReleases               = Error("deleting releases")
-	ErrForceDeletingComponents        = Error("force deleting components")
-	ErrRollingBackActivePromotion     = Error("rolling back active promotion process")
+	ErrActivePromotionTimeout            = Error("active promotion timeout")
+	ErrActiveDemotionTimeout             = Error("demoted active environment timeout")
+	ErrRollbackActivePromotionTimeout    = Error("rollback active promotion timeout")
+	ErrEnsurePreActiveEnvironmentCreated = Error("pre-active environment has been being created")
+	ErrEnsureNamespaceDestroyed          = Error("namespace has not been destroyed")
+	ErrEnsureActiveDemoted               = Error("active environment has been being demoted")
+	ErrEnsureActivePromoted              = Error("active environment has been being promoted")
+	ErrEnsureComponentDeployed           = Error("components has been being deployed")
+	ErrEnsureComponentTested             = Error("components has been being tested")
+	ErrDeletingReleases                  = Error("deleting releases")
+	ErrForceDeletingComponents           = Error("force deleting components")
+	ErrRollingBackActivePromotion        = Error("rolling back active promotion process")
 
 	ErrUnauthorized      = Error("unauthorized")
 	ErrAuthTokenNotFound = Error("auth token not found")
@@ -61,11 +61,6 @@ func IsImageNotFound(err error) bool {
 	return ErrImageVersionNotFound.Error() == err.Error()
 }
 
-// IsTeamNotFound checks team is exist
-func IsTeamNotFound(err error) bool {
-	return ErrTeamNotFound.Error() == err.Error()
-}
-
 // IsNamespaceStillCreating checks namespace is still creating
 func IsNamespaceStillCreating(err error) bool {
 	return ErrTeamNamespaceStillCreating.Error() == err.Error()
@@ -76,6 +71,11 @@ func IsNamespaceStillExists(err error) bool {
 	return ErrTeamNamespaceStillExists.Error() == err.Error()
 }
 
+// IsEnsuringPreActiveEnvironmentCreated checks ensuring pre-active created
+func IsEnsuringPreActiveEnvironmentCreated(err error) bool {
+	return ErrEnsureActivePromoted.Error() == err.Error()
+}
+
 // IsEnsuringActivePromoted checks ensuring active promoted
 func IsEnsuringActivePromoted(err error) bool {
 	return ErrEnsureActivePromoted.Error() == err.Error()

diff --git a/internal/samsahai/activepromotion/collect_result.go b/internal/samsahai/activepromotion/collect_result.go
@@ -12,7 +12,7 @@ import (
 
 func (c *controller) collectResult(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	q, err := queue.EnsurePreActiveComponents(c.client, teamName, targetNs)
 	if err != nil {
 		return errors.Wrapf(err, "cannot ensure pre-active components, namespace %s", targetNs)

diff --git a/internal/samsahai/activepromotion/controller.go b/internal/samsahai/activepromotion/controller.go
@@ -104,7 +104,7 @@ func (c *controller) setup(ctx context.Context, atpComp *s2hv1beta1.ActivePromot
 }
 
 func (c *controller) teardown(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	_ = queue.DeletePreActiveQueue(c.client, targetNs)
 	_ = queue.DeletePromoteToActiveQueue(c.client, targetNs)
 	_ = queue.DeleteDemoteFromActiveQueue(c.client, targetNs)
@@ -342,7 +342,7 @@ func (c *controller) Reconcile(req reconcile.Request) (reconcile.Result, error)
 
 	case s2hv1beta1.ActivePromotionCreatingPreActive:
 		if err := c.createPreActiveEnvAndDeployStableCompObjects(ctx, atpComp); err != nil {
-			if s2herrors.IsNamespaceStillCreating(err) {
+			if s2herrors.IsEnsuringPreActiveEnvironmentCreated(err) {
 				return reconcile.Result{
 					Requeue:      true,
 					RequeueAfter: 2 * time.Second,

diff --git a/internal/samsahai/activepromotion/deploy.go b/internal/samsahai/activepromotion/deploy.go
@@ -11,7 +11,7 @@ import (
 
 func (c *controller) deployComponentsToTargetNamespace(atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	q, err := c.ensurePreActiveComponentsDeployed(teamName, targetNs)
 	if err != nil {
 		return err
@@ -55,7 +55,7 @@ func (c *controller) ensurePreActiveComponentsDeployed(teamName, targetNs string
 
 func (c *controller) testPreActiveEnvironment(atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	q, err := c.ensurePreActiveComponentsTested(teamName, targetNs)
 	if err != nil {
 		return err

diff --git a/internal/samsahai/activepromotion/environment_cleanup.go b/internal/samsahai/activepromotion/environment_cleanup.go
@@ -69,7 +69,7 @@ func (c *controller) destroyPreviousActiveEnvironmentAt(ctx context.Context, tea
 }
 
 func (c *controller) destroyPreActiveEnvironment(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	teamName := atpComp.Name
 
 	startedCleaningTime := atpComp.Status.GetConditionLatestTime(s2hv1beta1.ActivePromotionCondActivePromoted)

diff --git a/internal/samsahai/activepromotion/environment_creator.go b/internal/samsahai/activepromotion/environment_creator.go
@@ -52,7 +52,7 @@ func (c *controller) createPreActiveEnvAndDeployStableCompObjects(ctx context.Co
 }
 
 func (c *controller) ensureActiveEnvironmentPromoted(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 
 	teamComp, err := c.getTeam(ctx, atpComp.Name)
 	if err != nil {
@@ -65,6 +65,7 @@ func (c *controller) ensureActiveEnvironmentPromoted(ctx context.Context, atpCom
 	}
 
 	if err := c.ensureTeamNamespaceUpdated(teamComp.Status.Namespace.Active, targetNs); err != nil {
+		logger.Warn("cannot ensure team active namespace updated", "error", err.Error())
 		return s2herrors.ErrEnsureActivePromoted
 	}
 
@@ -73,6 +74,9 @@ func (c *controller) ensureActiveEnvironmentPromoted(ctx context.Context, atpCom
 
 func (c *controller) createPreActiveEnvironment(ctx context.Context, teamName, preActiveNs string) error {
 	if err := c.s2hCtrl.CreatePreActiveEnvironment(teamName, preActiveNs); err != nil && !k8serrors.IsAlreadyExists(err) {
+		if s2herrors.IsNamespaceStillCreating(err) {
+			return s2herrors.ErrEnsurePreActiveEnvironmentCreated
+		}
 		return err
 	}
 
@@ -81,12 +85,14 @@ func (c *controller) createPreActiveEnvironment(ctx context.Context, teamName, p
 		return err
 	}
 
-	if err := c.ensureTeamNamespaceUpdated(teamComp.Status.Namespace.PreActive, preActiveNs); err != nil {
-		return err
+	if err := c.ensureNamespaceReady(ctx, preActiveNs); err != nil {
+		logger.Warn("cannot ensure pre-active namespace created", "error", err.Error())
+		return s2herrors.ErrEnsurePreActiveEnvironmentCreated
 	}
 
-	if err := c.ensureNamespaceReady(ctx, preActiveNs); err != nil {
-		return err
+	if err := c.ensureTeamNamespaceUpdated(teamComp.Status.Namespace.PreActive, preActiveNs); err != nil {
+		logger.Warn("cannot ensure team pre-active namespace updated", "error", err.Error())
+		return s2herrors.ErrEnsurePreActiveEnvironmentCreated
 	}
 
 	return nil

diff --git a/internal/samsahai/activepromotion/history.go b/internal/samsahai/activepromotion/history.go
@@ -23,7 +23,7 @@ func (c *controller) createActivePromotionHistory(ctx context.Context, atpComp *
 
 	now := metav1.Now()
 	atpLabels := internal.GetDefaultLabels(atpComp.Name)
-	atpLabels["namespace"] = atpComp.Status.TargetNamespace
+	atpLabels["namespace"] = c.getTargetNamespace(atpComp)
 
 	history := &s2hv1beta1.ActivePromotionHistory{
 		ObjectMeta: metav1.ObjectMeta{

diff --git a/internal/samsahai/activepromotion/postactive.go b/internal/samsahai/activepromotion/postactive.go
@@ -47,7 +47,7 @@ func (c *controller) runPostActive(ctx context.Context, atpComp *s2hv1beta1.Acti
 }
 
 func (c *controller) sendReport(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
-	currentNs := atpComp.Status.TargetNamespace
+	currentNs := c.getTargetNamespace(atpComp)
 	if atpComp.Status.Result != s2hv1beta1.ActivePromotionSuccess {
 		currentNs = atpComp.Status.PreviousActiveNamespace
 		if atpComp.Status.DemotionStatus == s2hv1beta1.ActivePromotionDemotionFailure {
@@ -85,7 +85,7 @@ func (c *controller) setOutdatedDuration(ctx context.Context, atpComp *s2hv1beta
 	}
 
 	configCtrl := c.s2hCtrl.GetConfigController()
-	atpNs := atpComp.Status.TargetNamespace
+	atpNs := c.getTargetNamespace(atpComp)
 	if atpComp.Status.Result != s2hv1beta1.ActivePromotionSuccess {
 		atpNs = atpComp.Status.PreviousActiveNamespace
 	}

diff --git a/internal/samsahai/activepromotion/promote.go b/internal/samsahai/activepromotion/promote.go
@@ -14,7 +14,7 @@ import (
 
 func (c *controller) promoteActiveEnvironment(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	prevNs := atpComp.Status.PreviousActiveNamespace
 
 	if err := queue.DeletePreActiveQueue(c.client, targetNs); err != nil {

diff --git a/internal/samsahai/activepromotion/rollback.go b/internal/samsahai/activepromotion/rollback.go
@@ -41,7 +41,7 @@ func (c *controller) demoteAndDestroyPreActive(ctx context.Context, atpComp *s2h
 
 	if !isTimeout {
 		teamName := atpComp.Name
-		targetNs := atpComp.Status.TargetNamespace
+		targetNs := c.getTargetNamespace(atpComp)
 
 		errCh := make(chan error, 2)
 		go func() {
@@ -87,7 +87,7 @@ func (c *controller) demotePreActiveForRollback(ctx context.Context, teamName, t
 
 func (c *controller) destroyPreActiveEnvironmentForRollback(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	startedCleaningTime := atpComp.Status.GetConditionLatestTime(s2hv1beta1.ActivePromotionCondRollbackStarted)
 	if err := c.ensureDestroyEnvironment(ctx, preActiveEnvirontment, teamName, targetNs, startedCleaningTime); err != nil {
 		return err
@@ -103,7 +103,7 @@ func (c *controller) destroyPreActiveEnvironmentForRollback(ctx context.Context,
 
 func (c *controller) rePromoteCurrentActive(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
 	teamName := atpComp.Name
-	targetNs := atpComp.Status.TargetNamespace
+	targetNs := c.getTargetNamespace(atpComp)
 	// get current namespace from team, current namespace might be destroyed due to demoting timeout
 	teamComp, err := c.getTeam(ctx, teamName)
 	if err != nil {

diff --git a/internal/samsahai/activepromotion/util.go b/internal/samsahai/activepromotion/util.go
@@ -12,6 +12,22 @@ import (
 	"github.com/agoda-com/samsahai/internal/util/stringutils"
 )
 
+func (c *controller) getTargetNamespace(atpComp *s2hv1beta1.ActivePromotion) string {
+	if atpComp.Status.TargetNamespace == "" {
+		logger.Warn("target namespace has not been set, getting namespace from team", "team",
+			atpComp.Name)
+		teamComp, err := c.getTeam(context.TODO(), atpComp.Name)
+		if err != nil {
+			logger.Error(err, "cannot pre-active namespace from team", "team", atpComp.Name)
+			return ""
+		}
+
+		return teamComp.Status.Namespace.PreActive
+	}
+
+	return atpComp.Status.TargetNamespace
+}
+
 func (c *controller) updateActivePromotion(ctx context.Context, atpComp *s2hv1beta1.ActivePromotion) error {
 	if err := c.client.Update(ctx, atpComp); err != nil {
 		return errors.Wrapf(err, "cannot update activepromotion %s", atpComp.Name)

diff --git a/internal/samsahai/controller.go b/internal/samsahai/controller.go
@@ -1176,6 +1176,11 @@ func (c *controller) Reconcile(req reconcile.Request) (reconcile.Result, error)
 		return reconcile.Result{}, err
 	}
 
+	teamComp = &s2hv1beta1.Team{}
+	if err := c.getTeam(teamName, teamComp); err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "cannot get team %s", teamName)
+	}
+
 	if err := c.LoadTeamSecret(teamComp); err != nil {
 		return reconcile.Result{}, err
 	}

diff --git a/internal/staging/controller.go b/internal/staging/controller.go
@@ -352,6 +352,7 @@ func (c *controller) cleanBefore(queue *s2hv1beta1.Queue) error {
 	if err != nil {
 		return err
 	} else if !isCleaned {
+		logger.Warn("waiting for component cleaned", "queue", queue.Name)
 		time.Sleep(2 * time.Second)
 		return nil
 	}
@@ -478,21 +479,25 @@ func WaitForComponentsCleaned(
 			if forceClean {
 				return false, forceCleanupPod(log, c, namespace, selectors)
 			}
+
+			log.Warn("pods still exist", "pods", getPodNames(pods))
 			return false, nil
 		}
 
 		// check services
 		services := &corev1.ServiceList{}
 		err = c.List(context.TODO(), services, listOpt)
 		if err != nil {
-			logger.Error(err, "list services error")
+			log.Error(err, "list services error")
 			return false, err
 		}
 
 		if len(services.Items) > 0 {
 			if forceClean {
 				return false, forceCleanupService(log, c, namespace, selectors)
 			}
+
+			log.Warn("services still exist", "services", getServiceNames(services))
 			return false, nil
 		}
 
@@ -609,3 +614,21 @@ func generateQueueHistoryName(queueName string) string {
 	now := metav1.Now()
 	return fmt.Sprintf("%s-%s", queueName, now.Format("20060102-150405"))
 }
+
+func getPodNames(pods *corev1.PodList) []string {
+	podNames := make([]string, 0)
+	for _, pod := range pods.Items {
+		podNames = append(podNames, pod.Name)
+	}
+
+	return podNames
+}
+
+func getServiceNames(svcs *corev1.ServiceList) []string {
+	svcNames := make([]string, 0)
+	for _, svc := range svcs.Items {
+		svcNames = append(svcNames, svc.Name)
+	}
+
+	return svcNames
+}