Merge pull request #5 from mhlias/add_parallelism_option

Add terraform plan refresh parallelism option per project.

main.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -25,6 +28,7 @@ type conf struct {
 	Accounts_mapping map[string]string `yaml:"accounts-mapping"`
 	Use_sts          bool              `yaml:"use-sts"`
 	Encrypt_s3_state bool              `yaml:"encrypt-s3-state"`
+	Parallelism      int16             `yaml:"parallelism"`
 	environment      string
 	account          string
 }
@@ -115,11 +119,30 @@ func main() {
 		fmt.Sprintf("%s-prd", project_config.Project): true,
 	}
 
+	tf_version := get_tf_version()
+
+	tf_legacy := true
+
+	ver_int, _ := strconv.Atoi(strings.Split(tf_version, ".")[1])
+
+	if ver_int > 8 {
+		tf_legacy = false
+	} else {
+		log.Printf("[WARN] Running in legacy mode, current Terraform version: %s, install >=0.9.x for full features.\n", tf_version)
+	}
+
 	state_config := &tf_helper.Config{Bucket_name: fmt.Sprintf("%s-%s-%s-tfstate", project_config.Project, project_config.account, project_config.environment),
 		State_filename:   fmt.Sprintf("%s-%s-%s.tfstate", project_config.Project, project_config.account, project_config.environment),
 		Versioning:       true,
 		Encrypt_s3_state: project_config.Encrypt_s3_state,
 		TargetsTF:        targetsTF,
+		TFlegacy:         tf_legacy,
+	}
+
+	var tf_parallelism int16 = 10
+
+	if &project_config.Parallelism != nil && project_config.Parallelism > 0 {
+		tf_parallelism = project_config.Parallelism
 	}
 
 	modules := &tf_helper.Modules{}
@@ -178,7 +201,7 @@ func main() {
 		}
 
 	} else if *planPtr {
-		state_config.Plan(tholos_conf)
+		state_config.Plan(tholos_conf, tf_parallelism)
 	} else if *modulesPtr {
 		modules.Fetch_modules(tholos_conf)
 	} else if *outputsPtr {
@@ -209,3 +232,22 @@ func load_config(project_config_file string) *conf {
 	return project_config
 
 }
+
+func get_tf_version() string {
+
+	cmd := exec.Command("terraform", "version")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		log.Fatal("Failed to get Terraform version, please make sure terraform is installed and in the path", err)
+	}
+
+	out_str := out.String()
+
+	start := strings.Index(out_str, "v")
+
+	ver := out_str[start : start+6]
+
+	return ver
+}

main_test.go

@@ -11,7 +11,7 @@ func TestProjectConfig(t *testing.T) {
 
 	project_config := load_config(fmt.Sprintf("%s/project.yaml", fixtures_dir))
 
-	if project_config.Project != "test" || project_config.Region != "eu-west-1" || !project_config.Use_sts || !project_config.Encrypt_s3_state || len(project_config.Roam_roles[fmt.Sprintf("%s-dev", project_config.Project)]) <= 0 || len(project_config.Accounts_mapping[fmt.Sprintf("%s-dev", project_config.Project)]) <= 0 || len(project_config.Accounts_mapping[fmt.Sprintf("%s-prd", project_config.Project)]) <= 0 {
+	if project_config.Parallelism != 4 || project_config.Project != "test" || project_config.Region != "eu-west-1" || !project_config.Use_sts || !project_config.Encrypt_s3_state || len(project_config.Roam_roles[fmt.Sprintf("%s-dev", project_config.Project)]) <= 0 || len(project_config.Accounts_mapping[fmt.Sprintf("%s-dev", project_config.Project)]) <= 0 || len(project_config.Accounts_mapping[fmt.Sprintf("%s-prd", project_config.Project)]) <= 0 {
 		t.Fatal("Project configuration parameters in fixtures don't match expected values when parsed.")
 	}
 

test-fixtures/project.yaml

@@ -10,3 +10,4 @@ encrypt-s3-state: true
 accounts-mapping:
     test-dev: 1001
     test-prd: 1002
+parallelism: 4

tf_helper/plan.go

@@ -7,7 +7,7 @@ import (
 	"github.com/mhlias/tholos/tholos"
 )
 
-func (c *Config) Plan(tholos_conf *tholos.Tholos_config) {
+func (c *Config) Plan(tholos_conf *tholos.Tholos_config, parallelism int16) {
 
 	cmd_name := "rm"
 
@@ -41,7 +41,7 @@ func (c *Config) Plan(tholos_conf *tholos.Tholos_config) {
 
 	c.Setup_remote_state()
 
-	exec_args = []string{"plan", "-module-depth=3", "-refresh=true", "-out=plans/plan.tfplan", "-var-file=params/env.tfvars"}
+	exec_args = []string{"plan", fmt.Sprintf("-parallelism=%d", parallelism), "-module-depth=3", "-refresh=true", "-out=plans/plan.tfplan", "-var-file=params/env.tfvars"}
 
 	if len(c.TargetsTF) > 0 {
 		for _, t := range c.TargetsTF {

tf_helper/state.go

@@ -15,6 +15,7 @@ type Config struct {
 	Encrypt_s3_state bool
 	Versioning       bool
 	TargetsTF        []string
+	TFlegacy         bool
 }
 
 func (c *Config) Create_bucket(client interface{}) bool {
@@ -98,6 +99,10 @@ func (c *Config) enable_versioning(client interface{}) bool {
 
 }
 
+func (c *Config) setup_lock_DB() {
+
+}
+
 func (c *Config) Setup_remote_state() {
 
 	//log.Printf("[INFO] Environment variables: %s, %s, %s, %s", os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), os.Getenv("AWS_SECURITY_TOKEN"), os.Getenv("AWS_DEFAULT_REGION") )