Skip to content

Commit

Permalink
safely delete stack (#71)
Browse files Browse the repository at this point in the history 
* implement deleteScheduled tag to postpone deletion of ALBs 1 hour after trigger

* use internal log package instead of logrus to fix compile time error

* fix test to have tags

* parameterize stackTTL the time to delete the CF stack

* Unit Tests for CF functions

* fix test case to fail on not Equal

* add test case to delete stack

* drop error checking that provides no value

* reset stack parameters to fix validation runtime errors from aws api
  • Loading branch information
@szuecs
szuecs authored Jun 20, 2017
1 parent ce6ce5d commit 200d71a
Show file tree
Hide file tree
Showing 5 changed files with 277 additions and 3 deletions.
16 changes: 15 additions & 1 deletion aws/adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type Adapter struct {
healthCheckPort uint
healthCheckInterval time.Duration
creationTimeout time.Duration
stackTTL time.Duration
}

type manifest struct {
Expand All @@ -60,6 +61,7 @@ const (
DefaultHealthCheckInterval = 10 * time.Second
DefaultCertificateUpdateInterval = 30 * time.Minute
DefaultCreationTimeout = 5 * time.Minute
DefaultStackTTL = 5 * time.Minute

clusterIDTag = "ClusterID"
nameTag = "Name"
Expand Down Expand Up @@ -119,6 +121,7 @@ func NewAdapter() (adapter *Adapter, err error) {
healthCheckPort: DefaultHealthCheckPort,
healthCheckInterval: DefaultHealthCheckInterval,
creationTimeout: DefaultCreationTimeout,
stackTTL: DefaultStackTTL,
}

adapter.manifest, err = buildManifest(adapter)
Expand Down Expand Up @@ -232,7 +235,7 @@ func (a *Adapter) FindManagedStacks() ([]*Stack, error) {
// Failure to create the stack causes it to be deleted automatically.
func (a *Adapter) CreateStack(certificateARN string) (string, error) {
spec := &createStackSpec{
name: normalizeStackName(a.ClusterID(), certificateARN),
name: a.stackName(certificateARN),
scheme: elbv2.LoadBalancerSchemeEnumInternetFacing,
certificateARN: certificateARN,
securityGroupID: a.SecurityGroupID(),
Expand All @@ -250,11 +253,22 @@ func (a *Adapter) CreateStack(certificateARN string) (string, error) {
return createStack(a.cloudformation, spec)
}

func (a *Adapter) stackName(certificateARN string) string {
return normalizeStackName(a.ClusterID(), certificateARN)
}

// GetStack returns the CloudFormation stack details with the name or ID from the argument
func (a *Adapter) GetStack(stackID string) (*Stack, error) {
return getStack(a.cloudformation, stackID)
}

// MarkToDeleteStack adds a "deleteScheduled" Tag to the CloudFormation stack with the given name
func (a *Adapter) MarkToDeleteStack(stack *Stack) (time.Time, error) {
t0 := time.Now().Add(a.stackTTL)

return t0, markToDeleteStack(a.cloudformation, a.stackName(stack.CertificateARN()), t0.Format(time.RFC3339))
}

// DeleteStack deletes the CloudFormation stack with the given name
func (a *Adapter) DeleteStack(stack *Stack) error {
if err := detachTargetGroupFromAutoScalingGroup(a.autoscaling, stack.TargetGroupARN(), a.AutoScalingGroupName()); err != nil {
Expand Down
77 changes: 76 additions & 1 deletion aws/cf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws

import (
"fmt"
"log"
"strings"

"time"
Expand All @@ -12,12 +13,17 @@ import (
"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
)

const (
deleteScheduled = "deleteScheduled"
)

// Stack is a simple wrapper around a CloudFormation Stack.
type Stack struct {
name string
dnsName string
targetGroupARN string
certificateARN string
tags map[string]string
}

func (s *Stack) Name() string {
Expand All @@ -36,6 +42,47 @@ func (s *Stack) TargetGroupARN() string {
return s.targetGroupARN
}

// IsDeleteInProgress returns true if the stack has already a tag
// deleteScheduled.
func (s *Stack) IsDeleteInProgress() bool {
if s == nil {
return false
}
_, ok := s.tags[deleteScheduled]
return ok
}

// ShouldDelete returns true if stack is marked to delete and the
// deleteScheduled tag is after time.Now(). In all other cases it
// returns false.
func (s *Stack) ShouldDelete() bool {
if s == nil {
return false
}
t0 := s.deleteTime()
if t0 == nil {
return false
}
now := time.Now()
return now.After(*t0)
}

func (s *Stack) deleteTime() *time.Time {
if s == nil {
return nil
}
ts, ok := s.tags[deleteScheduled]
if !ok {
return nil
}
t, err := time.Parse(time.RFC3339, ts)
if err != nil {
log.Printf("Failed to parse time: %v", err)
return nil
}
return &t
}

type stackOutput map[string]string

func newStackOutput(outputs []*cloudformation.Output) stackOutput {
Expand Down Expand Up @@ -148,7 +195,34 @@ func deleteStack(svc cloudformationiface.CloudFormationAPI, stackName string) er
return err
}

// maybe use https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateChangeSet.html instead
func markToDeleteStack(svc cloudformationiface.CloudFormationAPI, stackName, ts string) error {
stack, err := getCFStackByName(svc, stackName)
if err != nil {
return err
}
tags := append(stack.Tags, cfTag(deleteScheduled, ts))

params := &cloudformation.UpdateStackInput{
StackName: aws.String(stackName),
Tags: tags,
Parameters: stack.Parameters,
UsePreviousTemplate: aws.Bool(true),
}

_, err = svc.UpdateStack(params)
return err
}

func getStack(svc cloudformationiface.CloudFormationAPI, stackName string) (*Stack, error) {
stack, err := getCFStackByName(svc, stackName)
if err != nil {
return nil, ErrLoadBalancerStackNotReady
}
return mapToManagedStack(stack), nil
}

func getCFStackByName(svc cloudformationiface.CloudFormationAPI, stackName string) (*cloudformation.Stack, error) {
params := &cloudformation.DescribeStacksInput{StackName: aws.String(stackName)}

resp, err := svc.DescribeStacks(params)
Expand All @@ -171,7 +245,7 @@ func getStack(svc cloudformationiface.CloudFormationAPI, stackName string) (*Sta
return nil, ErrLoadBalancerStackNotReady
}

return mapToManagedStack(stack), nil
return stack, nil
}

func mapToManagedStack(stack *cloudformation.Stack) *Stack {
Expand All @@ -181,6 +255,7 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack {
dnsName: o.dnsName(),
targetGroupARN: o.targetGroupARN(),
certificateARN: t[certificateARNTag],
tags: convertCloudFormationTags(stack.Tags),
}
}

Expand Down
157 changes: 157 additions & 0 deletions aws/cf_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aws
import (
"reflect"
"testing"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudformation"
Expand Down Expand Up @@ -54,6 +55,37 @@ func TestCreatingStack(t *testing.T) {
}
}

func TestDeleteStack(t *testing.T) {
for _, ti := range []struct {
msg string
givenSpec createStackSpec
givenOutputs cfMockOutputs
wantErr bool
}{
{
"delete-existing-stack",
createStackSpec{name: "existing-stack-id"},
cfMockOutputs{deleteStack: R(mockDeleteStackOutput("existing-stack-id"), nil)},
false,
},
{
"delete-non-existing-stack",
createStackSpec{name: "non-existing-stack-id"},
cfMockOutputs{deleteStack: R(mockDeleteStackOutput("existing-stack-id"), nil)},
false,
},
} {
t.Run(ti.msg, func(t *testing.T) {
c := &mockCloudFormationClient{outputs: ti.givenOutputs}
err := deleteStack(c, ti.givenSpec.name)
haveErr := err != nil
if haveErr != ti.wantErr {
t.Errorf("unexpected result from %s. wanted error %v, got err: %+v", ti.msg, ti.wantErr, err)
}
})
}
}

func TestStackReadiness(t *testing.T) {
for _, ti := range []struct {
given string
Expand Down Expand Up @@ -211,6 +243,11 @@ func TestFindingManagedStacks(t *testing.T) {
dnsName: "example.com",
certificateARN: "cert-arn",
targetGroupARN: "tg-arn",
tags: map[string]string{
kubernetesCreatorTag: kubernetesCreatorValue,
clusterIDTag: "test-cluster",
certificateARNTag: "cert-arn",
},
},
},
false,
Expand Down Expand Up @@ -296,3 +333,123 @@ func TestFindingManagedStacks(t *testing.T) {
})
}
}

func TestIsDeleteInProgress(t *testing.T) {
for _, ti := range []struct {
msg string
given *Stack
want bool
}{
{
"DeleteInProgress",
&Stack{tags: map[string]string{deleteScheduled: time.Now().Add(1 * time.Minute).Format(time.RFC3339)}},
true,
},
{
"EmptyStack",
&Stack{},
false,
},
{
"StackNil",
nil,
false,
},
} {
t.Run(ti.msg, func(t *testing.T) {
got := ti.given.IsDeleteInProgress()
if ti.want != got {
t.Errorf("unexpected result. wanted %+v, got %+v", ti.want, got)
}
})
}

}

func TestShouldDelete(t *testing.T) {
for _, ti := range []struct {
msg string
given *Stack
want bool
}{
{
"DeleteInProgress",
&Stack{tags: map[string]string{deleteScheduled: time.Now().Add(1 * time.Minute).Format(time.RFC3339)}},
false,
},
{
"DeleteInProgressSecond",
&Stack{tags: map[string]string{deleteScheduled: time.Now().Add(1 * time.Second).Format(time.RFC3339)}},
false,
},
{
"ShouldDelete",
&Stack{tags: map[string]string{deleteScheduled: time.Now().Add(-1 * time.Second).Format(time.RFC3339)}},
true,
},
{
"ShouldDeleteMinute",
&Stack{tags: map[string]string{deleteScheduled: time.Now().Add(-1 * time.Minute).Format(time.RFC3339)}},
true,
}, {
"EmptyStack",
&Stack{},
false,
},
{
"StackNil",
nil,
false,
},
} {
t.Run(ti.msg, func(t *testing.T) {
got := ti.given.ShouldDelete()
if ti.want != got {
t.Errorf("unexpected result for %s. wanted %+v, got %+v", ti.msg, ti.want, got)
}
})
}

}

func TestDeleteTime(t *testing.T) {
now := time.Now()
for _, ti := range []struct {
msg string
given *Stack
want *time.Time
}{
{
"GetCorrectTime",
&Stack{tags: map[string]string{deleteScheduled: now.Format(time.RFC3339Nano)}},
&now,
},
{
"IncorrectTime",
&Stack{tags: map[string]string{deleteScheduled: "foo"}},
nil,
},
{
"EmptyStack",
&Stack{},
nil,
},
{
"StackNil",
nil,
nil,
},
} {
t.Run(ti.msg, func(t *testing.T) {
got := ti.given.deleteTime()
if ti.want != nil {
if !ti.want.Equal(*got) {
t.Errorf("unexpected result for non nil %s. wanted %+v, got %+v", ti.msg, ti.want, got)
}
} else if ti.want != got {
t.Errorf("unexpected result for %s. wanted %+v, got %+v", ti.msg, ti.want, got)
}
})
}

}
12 changes: 12 additions & 0 deletions aws/cfmock_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type cfMockOutputs struct {
describeStackPages *apiResponse
describeStacks *apiResponse
createStack *apiResponse
deleteStack *apiResponse
}

type mockCloudFormationClient struct {
Expand Down Expand Up @@ -54,3 +55,14 @@ func mockCSOutput(stackId string) *cloudformation.CreateStackOutput {
StackId: aws.String(stackId),
}
}

func (m *mockCloudFormationClient) DeleteStack(params *cloudformation.DeleteStackInput) (*cloudformation.DeleteStackOutput, error) {
if out, ok := m.outputs.deleteStack.response.(*cloudformation.DeleteStackOutput); ok {
return out, m.outputs.deleteStack.err
}
return nil, m.outputs.deleteStack.err
}

func mockDeleteStackOutput(stackId string) *cloudformation.DeleteStackOutput {
return &cloudformation.DeleteStackOutput{}
}
Loading

0 comments on commit 200d71a

Please sign in to comment.