From cad269241fc2c0a90d1949f63aa4f7ffcfdc0ebe Mon Sep 17 00:00:00 2001 From: JP Bochi Date: Fri, 25 Jan 2019 16:27:19 +0100 Subject: [PATCH 1/2] writes both verbose logs file and succinct plan files to /tmp/artifacts/ --- tests/bats-utils | 1 + tests/plan.bats | 96 +++++++++++++++++++++++++++++++++++++++++++- toolbox/bin/apply.sh | 10 ++--- toolbox/bin/plan.sh | 29 +++++++------ 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/tests/bats-utils b/tests/bats-utils index 09ade96..40b6346 100644 --- a/tests/bats-utils +++ b/tests/bats-utils @@ -22,6 +22,7 @@ function setup_gitrepo() { cd ../cloned_infra git config --local user.email "test@example.com" git config --local user.name "test" + rm -rf /tmp/artifacts } function unset_vars() { diff --git a/tests/plan.bats b/tests/plan.bats index 82dad8c..22bc8b5 100644 --- a/tests/plan.bats +++ b/tests/plan.bats @@ -39,6 +39,11 @@ function teardown() { pwd > "$output" cat ./.terraform/init >> "$output" echo $@ >> "$output" + echo "plan yada yada" + echo "-----------" + echo " + new_resource (at $(pwd))" + echo "-----------" + echo "more yada yada" fi done fi @@ -50,10 +55,97 @@ function teardown() { diff ./workspace/terraform.base_network.plan <(echo \ "${TEST_LOCAL_REPO}/layers/base_network -init -backend=true -backend-config=bucket=le-bucket -backend-config=region=le-region -backend-config=encrypt=true -input=false -no-color +init -no-color -input=false -backend=true -backend-config=bucket=le-bucket -backend-config=region=le-region -backend-config=encrypt=true plan -no-color -input=false -out=${TEST_LOCAL_REPO}/workspace/terraform.base_network.plan") diff ./workspace/terraform.route53_internal_zone.plan <(echo \ "${TEST_LOCAL_REPO}/layers/route53_internal_zone -init -backend=true -backend-config=bucket=le-bucket -backend-config=region=le-region -backend-config=encrypt=true -input=false -no-color +init -no-color -input=false -backend=true -backend-config=bucket=le-bucket -backend-config=region=le-region -backend-config=encrypt=true plan -no-color -input=false -out=${TEST_LOCAL_REPO}/workspace/terraform.route53_internal_zone.plan") + diff /tmp/artifacts/terraform_all_plans.log <(echo \ +"> Planning layer: base_network +----------- + + new_resource (at ${TEST_LOCAL_REPO}/layers/base_network) +----------- +> Planning layer: route53_internal_zone +----------- + + new_resource (at ${TEST_LOCAL_REPO}/layers/route53_internal_zone) +-----------") + diff /tmp/artifacts/terraform_plan.base_network.log <(echo \ +"> Planning layer: base_network +----------- + + new_resource (at ${TEST_LOCAL_REPO}/layers/base_network) +-----------") + diff /tmp/artifacts/terraform_plan.route53_internal_zone.log <(echo \ +"> Planning layer: route53_internal_zone +----------- + + new_resource (at ${TEST_LOCAL_REPO}/layers/route53_internal_zone) +-----------") +} + +@test "plans includes 'no changes'" { + mkdir -p ./workspace + printf 'base_network\n' > ./workspace/changed_layers + TEST_LOCAL_REPO=$(pwd) + + echo ' + echo "$@" + if [ "$1" = "init" ]; then + mkdir -p ./.terraform + echo "$@" > ./.terraform/init + elif [ "$1" = "plan" ]; then + for arg in $@; do + if (echo $arg | grep -q "^-out="); then + output=$(echo $arg | sed "s/^-out=//") + pwd > "$output" + cat ./.terraform/init >> "$output" + echo $@ >> "$output" + echo "Refreshing Terraform state in-memory prior to plan..." + echo "--------------------" + echo "" + echo "No changes. Infrastructure is up-to-date." + echo "" + echo "This means that Terraform yada yada yada..." + fi + done + fi + ' > $bin_terraform + + TF_STATE_BUCKET='le-bucket' + TF_STATE_REGION='le-region' + plan.sh + + diff /tmp/artifacts/terraform_all_plans.log <(echo \ +"> Planning layer: base_network +-------------------- + +No changes. Infrastructure is up-to-date.") + diff /tmp/artifacts/terraform_plan.base_network.log <(echo \ +"> Planning layer: base_network +-------------------- + +No changes. Infrastructure is up-to-date.") +} + +@test "fails if plan fails" { + mkdir -p ./workspace + printf 'base_network\n' > ./workspace/changed_layers + TEST_LOCAL_REPO=$(pwd) + + echo ' + echo "$@" + if [ "$1" = "init" ]; then + mkdir -p ./.terraform + echo "$@" > ./.terraform/init + elif [ "$1" = "plan" ]; then + exit 1 + fi + ' > $bin_terraform + + TF_STATE_BUCKET='le-bucket' + TF_STATE_REGION='le-region' + run plan.sh + + echo ">> output:" + echo "$output" + [ "$status" != 0 ] } diff --git a/toolbox/bin/apply.sh b/toolbox/bin/apply.sh index 73f42cd..e967858 100755 --- a/toolbox/bin/apply.sh +++ b/toolbox/bin/apply.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -eu +set -eu -o pipefail source $(dirname $(realpath $0))/variables.sh @@ -22,14 +22,14 @@ for LAYER in $CHANGED_LAYERS; do ls -la "$WORKSPACE_DIR/terraform.$LAYER.plan" # uncache .terraform for the apply - (cd "$LAYERS_DIR/$LAYER" && tar xzf "$WORKSPACE_DIR/.terraform.$LAYER.tar.gz") + cd "$LAYERS_DIR/$LAYER" + tar xzf "$WORKSPACE_DIR/.terraform.$LAYER.tar.gz" - set -x - (cd "$LAYERS_DIR/$LAYER" && terraform apply -input=false -no-color "$WORKSPACE_DIR/terraform.$LAYER.plan") - set +x + (set -x && terraform apply -input=false -no-color "$WORKSPACE_DIR/terraform.$LAYER.plan") done # escrows applied revision +cd "$WORKING_DIR" REVISION="${CIRCLE_SHA1:-$(git rev-parse HEAD)}" echo "$REVISION" > tf-applied-revision.sha aws s3 cp ./tf-applied-revision.sha "s3://${TF_STATE_BUCKET}/" diff --git a/toolbox/bin/plan.sh b/toolbox/bin/plan.sh index 7c6b6e1..8f7e048 100755 --- a/toolbox/bin/plan.sh +++ b/toolbox/bin/plan.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -set -eu +set -eu -o pipefail source $(dirname $(realpath $0))/variables.sh @@ -16,27 +16,30 @@ if [ -z "$CHANGED_LAYERS" ]; then fi for LAYER in $CHANGED_LAYERS; do - echo "> Planning layer: $LAYER" + mkdir -p /tmp/artifacts/ + ALL_OUTPUT="/tmp/artifacts/terraform_all_outputs.log" + ALL_PLANS="/tmp/artifacts/terraform_all_plans.log" + LAYER_OUTPUT="/tmp/artifacts/terraform_output.${LAYER}.log" + LAYER_PLAN="/tmp/artifacts/terraform_plan.${LAYER}.log" + echo "> Planning layer: $LAYER" | tee -a "$ALL_OUTPUT" "$ALL_PLANS" "$LAYER_OUTPUT" "$LAYER_PLAN" # ensure even deleted layers are plannable if [ ! -d "$LAYERS_DIR/$LAYER" ]; then - echo "> Layer directory $LAYERS_DIR/$LAYER was not found, creating an empty version." | tee -a "$WORKSPACE_DIR/full_plan_output.log" + echo "> Layer directory $LAYERS_DIR/$LAYER was not found, creating an empty version." | tee -a "$ALL_OUTPUT" "$ALL_PLANS" "$LAYER_OUTPUT" "$LAYER_PLAN" mkdir -p "$LAYERS_DIR/$LAYER/.terraform" touch "$LAYERS_DIR/$LAYER/deleted.tf" fi - set -x - (cd "$LAYERS_DIR/$LAYER" && terraform init -backend=true -backend-config="bucket=$TF_STATE_BUCKET" -backend-config="region=$TF_STATE_REGION" -backend-config="encrypt=true" -input=false -no-color) - set +x + cd "$LAYERS_DIR/$LAYER" + (set -x && terraform init -no-color -input=false -backend=true -backend-config="bucket=$TF_STATE_BUCKET" -backend-config="region=$TF_STATE_REGION" -backend-config="encrypt=true") \ + | tee -a "$ALL_OUTPUT" "$LAYER_OUTPUT" # cache .terraform during the plan - (cd "$LAYERS_DIR/$LAYER" && tar -czf "$WORKSPACE_DIR/.terraform.$LAYER.tar.gz" .terraform) + tar -czf "$WORKSPACE_DIR/.terraform.$LAYER.tar.gz" .terraform | tee -a "$ALL_OUTPUT" "$LAYER_OUTPUT" - set -x - (cd "$LAYERS_DIR/$LAYER" && terraform plan -no-color -input=false -out="$WORKSPACE_DIR/terraform.$LAYER.plan" | tee -a "$WORKSPACE_DIR/full_plan_output.log" | grep -v "Refreshing state") - set +x - # for debugging, show these files exist - ls -la "$WORKSPACE_DIR/.terraform.$LAYER.tar.gz" - ls -la "$WORKSPACE_DIR/terraform.$LAYER.plan" + TEMP_PLAN_LOG=$(mktemp) + (set -x && terraform plan -no-color -input=false -out="$WORKSPACE_DIR/terraform.$LAYER.plan") | tee -a "$TEMP_PLAN_LOG" | grep -v "Refreshing state" + cat "$TEMP_PLAN_LOG" | tee -a "$ALL_OUTPUT" "$LAYER_OUTPUT" | sed -n '/-----/,/-----/p' | sed -n '/-----/,/No changes. Infrastructure is up-to-date/p' \ + | tee -a "$ALL_PLANS" "$LAYER_PLAN" > /dev/null done From 2a5f5cfb31e6f2056e5e5c08970240cb49902e3e Mon Sep 17 00:00:00 2001 From: JP Bochi Date: Mon, 28 Jan 2019 15:55:15 +0100 Subject: [PATCH 2/2] adds 1.7.0 to changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8254e8..7407746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # rackspace-toolbox Changelog +## [1.7.0](https://github.com/rackspace-infrastructure-automation/rackspace-toolbox/releases/tag/1.7.0) (Jan 28, 2019) + +Writes both verbose logs file and succinct plan files to `/tmp/artifacts/`. + ## [1.6.4](https://github.com/rackspace-infrastructure-automation/rackspace-toolbox/releases/tag/1.6.4) (Jan 11, 2019) Make up to 3 attempts to request credentials.