Adding in Full mode (#56)

cmd/full.go

@@ -0,0 +1,109 @@
+// Copyright 2017 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/palantir/bouncer/bouncer"
+	"github.com/palantir/bouncer/full"
+)
+
+var fullCmd = &cobra.Command{
+	Use:   "full",
+	Short: "Run bouncer in full",
+	Long:  `Run bouncer in full mode, where we destroy all nodes across all AGS's one node at a time.  Then restore the ASG set one node at a time, but in reverse order.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		log.SetLevel(logLevelFromViper())
+
+		log.Debug("full called")
+		if log.GetLevel() == log.DebugLevel {
+			cmd.DebugFlags()
+			viper.Debug()
+		}
+
+		asgsString := viper.GetString("full.asgs")
+		if asgsString == "" {
+			log.Fatal("You must specify ASGs to cycle nodes from (in a comma-delimited list)")
+		}
+
+		commandString := viper.GetString("full.command")
+		noop := viper.GetBool("full.noop")
+		force := viper.GetBool("full.force")
+		termHook := viper.GetString("terminate-hook")
+		pendHook := viper.GetString("pending-hook")
+		timeout := timeoutFromViper()
+
+		log.Debugf("Binding vars, got %+v %+v %+v %+v", asgsString, noop, version, commandString)
+
+		log.Info("Beginning bouncer full run")
+
+		var defCap int64
+		defCap = 1
+		opts := bouncer.RunnerOpts{
+			Noop:            noop,
+			Force:           force,
+			AsgString:       asgsString,
+			CommandString:   commandString,
+			DefaultCapacity: &defCap,
+			TerminateHook:   termHook,
+			PendingHook:     pendHook,
+			ItemTimeout:     timeout,
+		}
+
+		r, err := full.NewRunner(&opts)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "error initializing runner"))
+		}
+
+		r.MustValidatePrereqs()
+
+		err = r.Run()
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "error in run"))
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(fullCmd)
+
+	fullCmd.Flags().BoolP("noop", "n", false, "Run this in noop mode, and only print what you would do")
+	err := viper.BindPFlag("full.noop", fullCmd.Flags().Lookup("noop"))
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Binding PFlag 'noop' to viper var 'full.noop' failed: %s"))
+	}
+
+	fullCmd.Flags().StringP("asgs", "a", "", "ASGs to check for nodes to cycle in")
+	err = viper.BindPFlag("full.asgs", fullCmd.Flags().Lookup("asgs"))
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Binding PFlag 'asgs' to viper var 'full.asgs' failed: %s"))
+	}
+
+	fullCmd.Flags().StringP("preterminatecall", "p", "", "External command to run before host is removed from its ELB & terminate process begins")
+	err = viper.BindPFlag("full.command", fullCmd.Flags().Lookup("preterminatecall"))
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Binding PFlag 'command' to viper var 'full.command' failed: %s"))
+	}
+
+	fullCmd.Flags().BoolP("force", "f", false, "Force all nodes to be recycled, even if they're running the latest launch config")
+	err = viper.BindPFlag("full.force", fullCmd.Flags().Lookup("force"))
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "Binding PFlag 'force' to viper var 'full.force' failed: %s"))
+	}
+}

full/runner.go

@@ -0,0 +1,155 @@
+// Copyright 2017 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package full
+
+import (
+	"os"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/pkg/errors"
+
+	"github.com/palantir/bouncer/bouncer"
+)
+
+// Runner holds data for a particular full run
+// Note that in the full case, asgs will always be of length 1
+type Runner struct {
+	bouncer.BaseRunner
+}
+
+// NewRunner instantiates a new full runner
+func NewRunner(opts *bouncer.RunnerOpts) (*Runner, error) {
+	br, err := bouncer.NewBaseRunner(opts)
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting base runner")
+	}
+
+	r := Runner{
+		*br,
+	}
+	return &r, nil
+}
+
+// MustValidatePrereqs checks that the batch runner is safe to proceed
+func (r *Runner) MustValidatePrereqs() {
+	asgSet, err := r.NewASGSet()
+	if err != nil {
+		log.Fatal(errors.Wrap(err, "error building ASGSet"))
+	}
+
+	divergedASGs := asgSet.GetDivergedASGs()
+	if len(divergedASGs) != 0 {
+		for _, badASG := range divergedASGs {
+			log.WithFields(log.Fields{
+				"ASG":                     *badASG.ASG.AutoScalingGroupName,
+				"desired_capacity actual": *badASG.ASG.DesiredCapacity,
+				"desired_capacity given":  badASG.DesiredASG.DesiredCapacity,
+			}).Error("ASG desired capacity doesn't match expected starting value")
+		}
+		os.Exit(1)
+	}
+
+	for _, asg := range asgSet.ASGs {
+		if *asg.ASG.DesiredCapacity == 0 {
+			log.WithFields(log.Fields{
+				"ASG": *asg.ASG.AutoScalingGroupName,
+			}).Warn("ASG desired capacity is 0 - nothing to do here")
+			os.Exit(0)
+		}
+
+		if *asg.ASG.MinSize != 0 {
+			log.WithFields(log.Fields{
+				"ASG":      *asg.ASG.AutoScalingGroupName,
+				"min_size": *asg.ASG.MinSize,
+			}).Error("ASG min size must equal 0")
+			os.Exit(1)
+		}
+	}
+}
+
+func reverseASGSetOrder(asg []*bouncer.ASG) []*bouncer.ASG {
+	// copy to new slice
+	new := append(asg[:0:0], asg...)
+
+	// reverse order of new slice
+	for i := len(new)/2 - 1; i >= 0; i-- {
+		rev := len(new) - 1 - i
+		new[i], new[rev] = new[rev], new[i]
+	}
+
+	return new
+}
+
+func asgSetWrapper(asg *bouncer.ASG) *bouncer.ASGSet {
+	return &bouncer.ASGSet{
+		ASGs: []*bouncer.ASG{asg},
+	}
+}
+
+// Run has the meat of the batch job
+func (r *Runner) Run() error {
+	var newDesiredCapacity int64
+
+start:
+	for {
+		if r.TimedOut() {
+			return errors.Errorf("timeout exceeded, something is probably wrong with rollout")
+		}
+
+		// Rebuild the state of the world every iteration of the loop because instance and ASG statuses are changing
+		log.Debug("Beginning new full run check")
+		asgSet, err := r.NewASGSet()
+		if err != nil {
+			return errors.Wrap(err, "error building ASGSet")
+		}
+
+		// See if we're still waiting on a change we made previously to finish or settle
+		if asgSet.IsTerminating() || asgSet.IsNewUnhealthy() || asgSet.IsImmutableAutoscalingEvent() || asgSet.IsCountMismatch() {
+			r.Sleep()
+			continue
+		}
+
+		// drain one ASG at a time one instance at a time until no ASGs have any old instances
+		for _, asg := range asgSet.ASGs {
+			set := asgSetWrapper(asg)
+
+			if set.IsOldInstance() {
+				err := r.KillInstance(set.GetBestOldInstance())
+				if err != nil {
+					return errors.Wrap(err, "failed to kill instance")
+				}
+				r.Sleep()
+				continue start
+			}
+		}
+
+		// restore ASG's in reversed order
+		for _, asg := range reverseASGSetOrder(asgSet.ASGs) {
+			// restore one instance at a time until back to desired cap
+			if *asg.ASG.DesiredCapacity < asg.DesiredASG.DesiredCapacity {
+				newDesiredCapacity = *asg.ASG.DesiredCapacity + 1
+
+				err = r.SetDesiredCapacity(asg, &newDesiredCapacity)
+				if err != nil {
+					return errors.Wrap(err, "error setting desired capacity")
+				}
+				r.Sleep()
+				continue start
+			}
+		}
+
+		return nil
+	}
+}

full/runner_test.go

@@ -0,0 +1,57 @@
+// Copyright 2017 Palantir Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package full
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/palantir/bouncer/bouncer"
+)
+
+func asgSliceTestConstructor(len int) []*bouncer.ASG {
+	var asgs []*bouncer.ASG
+
+	for i := 0; i < len; i++ {
+		name := fmt.Sprintf("asg-%v", i)
+		asgs = append(asgs, &bouncer.ASG{
+			ASG: &autoscaling.Group{
+				AutoScalingGroupName: &name,
+			},
+		})
+	}
+
+	return asgs
+}
+
+func TestReverseASGSlice(t *testing.T) {
+	asgs := asgSliceTestConstructor(5)
+	rev := reverseASGSetOrder(asgs)
+
+	assert.NotEqual(t, asgs, rev, "The original asg slice should NOT equal the reversed")
+	assert.Equal(t, asgs, asgSliceTestConstructor(5), "The original asg slice should still equal a fresh slice with the same number of instances, even after being reversed")
+	assert.Equal(t, asgs, reverseASGSetOrder(rev), "Reversing the already reversed asg set, should equal the original")
+
+	// tests on varying slice length
+	assert.Equal(t, asgSliceTestConstructor(1), reverseASGSetOrder(asgSliceTestConstructor(1)), "An asg slice of 1 does not change when reversed")
+	assert.Equal(t, asgSliceTestConstructor(3)[0], reverseASGSetOrder(asgSliceTestConstructor(3))[2], "The first asg in a set of 3 should equal the last asg in the reverse")
+	assert.Equal(t, asgSliceTestConstructor(4)[0], reverseASGSetOrder(asgSliceTestConstructor(4))[3], "The first asg in a set of 4 should equal the last asg in the reverse")
+	assert.Equal(t, asgSliceTestConstructor(7)[6], reverseASGSetOrder(asgSliceTestConstructor(7))[0], "The last asg in a set of 7 should equal the first entry in the reverse")
+	assert.Equal(t, asgSliceTestConstructor(8)[7], reverseASGSetOrder(asgSliceTestConstructor(8))[0], "The last asg in a set of 8 should equal the first entry in the reverse")
+	assert.Equal(t, asgSliceTestConstructor(15)[7], reverseASGSetOrder(asgSliceTestConstructor(15))[7], "The middle asg in a set of 15 should equal the middle asg in the reverse as it is an odd numbered slice")
+}