Issue 153: Add Rollback support for Pravega Cluster

doc/rollback-cluster.md

@@ -0,0 +1,101 @@
+# Pravega Cluster Rollback
+
+This document details how manual rollback can be triggered after a Pravega cluster upgrade fails.
+Note that a rollback can be triggered only on Upgrade Failure.
+
+## Upgrade Failure
+
+An Upgrade can fail because of following reasons:
+
+1. Incorrect configuration (wrong quota, permissions, limit ranges)
+2. Network issues (ImagePullError)
+3. K8s Cluster Issues.
+4. Application issues (Application runtime misconfiguration or code bugs)
+
+An upgrade failure can manifest through a Pod to staying in `Pending` state forever or continuously restarting or crashing (CrashLoopBackOff).
+A component deployment failure needs to be tracked and mapped to "Upgrade Failure" for Pravega Cluster.
+Here we try to fail-fast by explicitly checking for some common causes for deployment failure like image pull errors or  CrashLoopBackOff State and failing the upgrade if any pod runs into this state during upgrade.
+
+The following Pravega Cluster Status Condition indicates a Failed Upgrade:
+
+```
+ClusterConditionType: Error
+Status: True
+Reason: UpgradeFailed
+Message: <Details of exception/cause of failure>
+```
+After an Upgrade Failure output of `kubectl describe pravegacluster pravega` would look like this:
+
+```
+$> kubectl describe pravegacluster pravega
+. . .
+Spec:
+. . .
+Version:        0.6.0-2252.b6f6512
+. . .
+Status:
+. . .
+Conditions:
+    Last Transition Time:  2019-09-06T09:00:13Z
+    Last Update Time:      2019-09-06T09:00:13Z
+    Status:                False
+    Type:                  Upgrading
+    Last Transition Time:  2019-09-06T08:58:40Z
+    Last Update Time:      2019-09-06T08:58:40Z
+    Status:                False
+    Type:                  PodsReady
+    Last Transition Time:  2019-09-06T09:00:13Z
+    Last Update Time:      2019-09-06T09:00:13Z
+    Message:               failed to sync segmentstore version. pod pravega-pravega-segmentstore-0 update failed because of ImagePullBackOff
+    Reason:                UpgradeFailed
+    Status:                True
+    Type:                  Error
+  . . .
+  Current Version:         0.6.0-2239.6e24df7
+. . .
+Version History:
+    0.6.0-2239.6e24df7
+```
+where `0.6.0-2252.b6f6512` is the version we tried upgrading to and `0.6.0-2239.6e24df7` is the version before upgrade.
+
+## Manual Rollback Trigger
+A Rollback is triggered when a Pravgea Cluster is `UpgradeFailed` Error State and a user manually updates in the PravegaCluster spec the version field to point to cluster version prior to upgrade.
+
+Note:
+1. Rollback to any other cluster version (other than the previousVersion) is not supported at this point.
+2. Changing the cluster spec version to the previous cluster version, when cluster is not in `UpgradeFailed` state, will trigger a rollback, but will be treated like a regular upgrade.
+
+## Rollback Implementation
+When Rollback is started cluster moves into ClusterCondition `RollbackInProgress`.
+Once Rollback completes this condition is set to false.
+The order in which the components are rolled back the reverse as upgrade :
+
+1. Pravega Controller
+2. Pravega Segment Store
+3. BookKeeper
+
+A new field `versionHistory` has been added to Pravega ClusterStatus to maintain history of upgrades.
+
+Rollback involves moving all components in the cluster back to the previous cluster version. As in case of upgrade, operator would rollback one component at a time and one pod at a time to maintain HA.
+
+If Rollback completes successfully, cluster state goes back to `PodsReady` which would mean the cluster is now in a stable state.
+If Rollback Fails, the cluster would move to state `RollbackFailed` indicated by this cluster condition:
+```
+ClusterConditionType: Error
+Status: True
+Reason: RollbackFailed
+Message: <Details of exception/cause of failure>
+```
+
+Manual intervention would be needed for resolving this.
+
+
+
+
+
+
+
+## Pending tasks
+
+
+## Prerequisites

pkg/apis/pravega/v1alpha1/status.go

@@ -11,6 +11,7 @@
 package v1alpha1
 
 import (
+	"log"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -21,12 +22,13 @@ type ClusterConditionType string
 const (
 	ClusterConditionPodsReady ClusterConditionType = "PodsReady"
 	ClusterConditionUpgrading                      = "Upgrading"
+	ClusterConditionRollback                       = "RollbackInProgress"
 	ClusterConditionError                          = "Error"
 
 	// Reasons for cluster upgrading condition
-	UpgradingControllerReason   = "UpgradingController"
-	UpgradingSegmentstoreReason = "UpgradingSegmentstore"
-	UpgradingBookkeeperReason   = "UpgradingBookkeeper"
+	UpdatingControllerReason   = "Updating Controller"
+	UpdatingSegmentstoreReason = "Updating Segmentstore"
+	UpdatingBookkeeperReason   = "Updating Bookkeeper"
 )
 
 // ClusterStatus defines the observed state of PravegaCluster
@@ -41,6 +43,8 @@ type ClusterStatus struct {
 	// If the cluster is not upgrading, TargetVersion is empty.
 	TargetVersion string `json:"targetVersion,omitempty"`
 
+	VersionHistory []string `json:"versionHistory,omitempty"`
+
 	// Replicas is the number of desired replicas in the cluster
 	Replicas int32 `json:"replicas"`
 
@@ -83,7 +87,8 @@ type ClusterCondition struct {
 	LastTransitionTime string `json:"lastTransitionTime,omitempty"`
 }
 
-func (ps *ClusterStatus) InitConditions() {
+func (ps *ClusterStatus) Init() {
+	// Initialise conditions
 	conditionTypes := []ClusterConditionType{
 		ClusterConditionPodsReady,
 		ClusterConditionUpgrading,
@@ -95,6 +100,12 @@ func (ps *ClusterStatus) InitConditions() {
 			ps.setClusterCondition(*c)
 		}
 	}
+
+	// Set current cluster version in version history,
+	// so if the first upgrade fails we can rollback to this version
+	if ps.VersionHistory == nil && ps.CurrentVersion != "" {
+		ps.VersionHistory = []string{ps.CurrentVersion}
+	}
 }
 
 func (ps *ClusterStatus) SetPodsReadyConditionTrue() {
@@ -117,6 +128,16 @@ func (ps *ClusterStatus) SetUpgradingConditionFalse() {
 	ps.setClusterCondition(*c)
 }
 
+/*
+func (ps *ClusterStatus) SetUpdatedReplicasForComponent(componentName string, updatedReplicas int32, totalReplicas int32) {
+	_, upgradeCondition := ps.GetClusterCondition(ClusterConditionUpgrading)
+	if upgradeCondition != nil && upgradeCondition.Status == corev1.ConditionTrue {
+		reason := fmt.Sprintf("Updating component: %s. Updated Replicas: %v, Total Replicas: %v", componentName, updatedReplicas, totalReplicas)
+		c := newClusterCondition(ClusterConditionUpgrading, corev1.ConditionTrue, reason, message)
+		ps.setClusterCondition(*c)
+	}
+}
+*/
 func (ps *ClusterStatus) SetErrorConditionTrue(reason, message string) {
 	c := newClusterCondition(ClusterConditionError, corev1.ConditionTrue, reason, message)
 	ps.setClusterCondition(*c)
@@ -127,6 +148,15 @@ func (ps *ClusterStatus) SetErrorConditionFalse() {
 	ps.setClusterCondition(*c)
 }
 
+func (ps *ClusterStatus) SetRollbackConditionTrue(reason, message string) {
+	c := newClusterCondition(ClusterConditionRollback, corev1.ConditionTrue, "", "")
+	ps.setClusterCondition(*c)
+}
+func (ps *ClusterStatus) SetRollbackConditionFalse() {
+	c := newClusterCondition(ClusterConditionRollback, corev1.ConditionFalse, "", "")
+	ps.setClusterCondition(*c)
+}
+
 func newClusterCondition(condType ClusterConditionType, status corev1.ConditionStatus, reason, message string) *ClusterCondition {
 	return &ClusterCondition{
 		Type:               condType,
@@ -170,3 +200,77 @@ func (ps *ClusterStatus) setClusterCondition(newCondition ClusterCondition) {
 
 	ps.Conditions[position] = *existingCondition
 }
+
+func (ps *ClusterStatus) AddToVersionHistory(version string) {
+	lastIndex := len(ps.VersionHistory) - 1
+	if version != "" && ps.VersionHistory[lastIndex] != version {
+		ps.VersionHistory = append(ps.VersionHistory, version)
+		log.Printf("Updating version history adding version %v", version)
+	}
+}
+
+func (ps *ClusterStatus) GetLastVersion() (previousVersion string) {
+	len := len(ps.VersionHistory)
+	return ps.VersionHistory[len-1]
+}
+
+func (ps *ClusterStatus) IsClusterInUpgradeFailedState() bool {
+	_, errorCondition := ps.GetClusterCondition(ClusterConditionError)
+	if errorCondition == nil {
+		return false
+	}
+	if errorCondition.Status == corev1.ConditionTrue && errorCondition.Reason == "UpgradeFailed" {
+		return true
+	}
+	return false
+}
+
+func (ps *ClusterStatus) IsClusterInUpgradeFailedOrRollbackState() bool {
+	if ps.IsClusterInUpgradeFailedState() || ps.IsClusterInRollbackState() {
+		return true
+	}
+	return false
+}
+
+func (ps *ClusterStatus) IsClusterInRollbackState() bool {
+	_, rollbackCondition := ps.GetClusterCondition(ClusterConditionRollback)
+	if rollbackCondition == nil {
+		return false
+	}
+	if rollbackCondition.Status == corev1.ConditionTrue {
+		return true
+	}
+	return false
+}
+
+func (ps *ClusterStatus) IsClusterInUpgradingState() bool {
+	_, upgradeCondition := ps.GetClusterCondition(ClusterConditionUpgrading)
+	if upgradeCondition == nil {
+		return false
+	}
+	if upgradeCondition.Status == corev1.ConditionTrue {
+		return true
+	}
+	return false
+}
+
+func (ps *ClusterStatus) UpdateProgress(reason, updatedReplicas string) {
+	if ps.IsClusterInUpgradingState() {
+		// Set the upgrade condition reason to be UpgradingBookkeeperReason, message to be 0
+		ps.SetUpgradingConditionTrue(reason, updatedReplicas)
+	} else {
+		ps.SetRollbackConditionTrue(reason, updatedReplicas)
+	}
+}
+
+func (ps *ClusterStatus) GetLastCondition() (lastCondition *ClusterCondition) {
+	if ps.IsClusterInUpgradingState() {
+		_, lastCondition := ps.GetClusterCondition(ClusterConditionUpgrading)
+		return lastCondition
+	} else if ps.IsClusterInRollbackState() {
+		_, lastCondition := ps.GetClusterCondition(ClusterConditionRollback)
+		return lastCondition
+	}
+	// nothing to do if we are neither upgrading nor rolling back,
+	return nil
+}

pkg/controller/pravegacluster/pravegacluster_controller.go

@@ -138,11 +138,18 @@ func (r *ReconcilePravegaCluster) run(p *pravegav1alpha1.PravegaCluster) (err er
 		return fmt.Errorf("failed to sync cluster size: %v", err)
 	}
 
+	// Upgrade
 	err = r.syncClusterVersion(p)
 	if err != nil {
 		return fmt.Errorf("failed to sync cluster version: %v", err)
 	}
 
+	// Rollback
+	err = r.rollbackFailedUpgrade(p)
+	if err != nil {
+		return fmt.Errorf("Rollback attempt failed: %v", err)
+	}
+
 	err = r.reconcileClusterStatus(p)
 	if err != nil {
 		return fmt.Errorf("failed to reconcile cluster status: %v", err)
@@ -151,6 +158,7 @@ func (r *ReconcilePravegaCluster) run(p *pravegav1alpha1.PravegaCluster) (err er
 }
 
 func (r *ReconcilePravegaCluster) deployCluster(p *pravegav1alpha1.PravegaCluster) (err error) {
+
 	err = r.deployBookie(p)
 	if err != nil {
 		log.Printf("failed to deploy bookie: %v", err)
@@ -168,10 +176,12 @@ func (r *ReconcilePravegaCluster) deployCluster(p *pravegav1alpha1.PravegaCluste
 		log.Printf("failed to deploy segment store: %v", err)
 		return err
 	}
+
 	return nil
 }
 
 func (r *ReconcilePravegaCluster) deployController(p *pravegav1alpha1.PravegaCluster) (err error) {
+
 	pdb := pravega.MakeControllerPodDisruptionBudget(p)
 	controllerutil.SetControllerReference(p, pdb, r.scheme)
 	err = r.client.Create(context.TODO(), pdb)
@@ -251,6 +261,7 @@ func (r *ReconcilePravegaCluster) deploySegmentStore(p *pravegav1alpha1.PravegaC
 }
 
 func (r *ReconcilePravegaCluster) deployBookie(p *pravegav1alpha1.PravegaCluster) (err error) {
+
 	headlessService := pravega.MakeBookieHeadlessService(p)
 	controllerutil.SetControllerReference(p, headlessService, r.scheme)
 	err = r.client.Create(context.TODO(), headlessService)
@@ -439,7 +450,7 @@ func (r *ReconcilePravegaCluster) syncStatefulSetPvc(sts *appsv1.StatefulSet) er
 
 func (r *ReconcilePravegaCluster) reconcileClusterStatus(p *pravegav1alpha1.PravegaCluster) error {
 
-	p.Status.InitConditions()
+	p.Status.Init()
 
 	expectedSize := util.GetClusterExpectedSize(p)
 	listOps := &client.ListOptions{
@@ -483,3 +494,21 @@ func (r *ReconcilePravegaCluster) reconcileClusterStatus(p *pravegav1alpha1.Prav
 	}
 	return nil
 }
+
+func (r *ReconcilePravegaCluster) rollbackFailedUpgrade(p *pravegav1alpha1.PravegaCluster) error {
+	if r.isRollbackTriggered(p) {
+		// start rollback to previous version
+		previousVersion := p.Status.GetLastVersion()
+		log.Printf("Rolling back to last cluster version  %v", previousVersion)
+		//Rollback cluster to previous version
+		return r.rollbackClusterVersion(p, previousVersion)
+	}
+	return nil
+}
+
+func (r *ReconcilePravegaCluster) isRollbackTriggered(p *pravegav1alpha1.PravegaCluster) bool {
+	if p.Status.IsClusterInUpgradeFailedState() && p.Spec.Version == p.Status.GetLastVersion() {
+		return true
+	}
+	return false
+}