From ecd38e79816b74d93c7cfde7a1b02ece3d6388a6 Mon Sep 17 00:00:00 2001 From: Hiroki Takatsuka <3445553+tk3fftk@users.noreply.github.com> Date: Tue, 4 Jun 2024 00:24:01 +0900 Subject: [PATCH] feat: enable to merge blocks inside block (#31) * add test files * update test cases * feat: enable to use merge block inside block * run go mod tidy * update readme * rename annotation to tfustomize:merge_block from tfustomize:block_merge * fix typo --- README.md | 45 ++++++++++++++++-- api/hcl_parser.go | 47 +++++++++++++++++-- api/hcl_parser_test.go | 18 ++++++- go.mod | 4 +- go.sum | 12 +---- test/base/data_with_block.tf | 1 + test/base/main.tf | 1 + test/overlay/data_with_block_merge.tf | 8 ++++ .../data_with_block_merge_and_append.tf | 11 +++++ test/overlay/overlay.tf | 5 ++ 10 files changed, 129 insertions(+), 23 deletions(-) create mode 100644 test/overlay/data_with_block_merge.tf create mode 100644 test/overlay/data_with_block_merge_and_append.tf diff --git a/README.md b/README.md index e9f2ffa..ee05275 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ## Motivation [Terraform Modules](https://developer.hashicorp.com/terraform/language/modules) is looks almost the only way to reuse resource configurations with Terraform. There is other option [`terragrunt`](https://terragrunt.gruntwork.io/), but it's looks expanding and wrapping the use of Terraform module. -Terraform modules are effective when they are widly spreaded as open sources, or when they are officially provided by kind of Platform Engineers for internal use in private environments. However, creating custom modules for one product seems like overkill. +Terraform modules are effective when they are widly spread as open sources, or when they are officially provided by kind of Platform Engineers for internal use in private environments. However, creating custom modules for one product seems like overkill. It was quite labor-intensive to transition from a copy-paste style of management to using custom modules, so I created something like a Terraform version of kustomize as a proof of concept. [Override Files](https://developer.hashicorp.com/terraform/language/files/override) feature is looks similar concept with `tfustomize` but overriding is not for reusing purpose. @@ -74,12 +74,49 @@ patches { ### Merging Behavior and Limitation -- A Top-level block has the same block type and labels in base and overlay will be merged. +- A Top-level block has the same block type and labels in base and overlay will be merged. - Except `moved`, `import`, `removed` block. These will be appended. - `locals` blocks will be merged. - Within a top-level block, an attribute argument within an overlay block will be replaced any argument of the same name in the base block. -- Within a top-level block, any block will be appended. - - [Limitation] can not be replaced (https://github.com/tk3fftk/tfustomize/issues/8) +- Within a top-level block, any block will be appended by default. + - To merge a block, use an annotation `# tfustimize:merge_block:` both a base and an overlay like below. + +```hcl +# base +data "aws_ami" "ubuntu" { + filter { + # tfustomize:merge_block:name + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] + } +} + +# overlay +data "aws_ami" "ubuntu" { + filter { + name = "arch" + values = ["arm64"] + } + filter { + # tfustomize:merge_block:name + name = "name_is_updated" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] + } +} + +# output +data "aws_ami" "ubuntu" { + filter { + name = "arch" + values = ["arm64"] + } + filter { + name = "name_is_updated" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] + } +} +``` + - [Limitation] The output order is randomized inside block level order. (https://github.com/tk3fftk/tfustomize/issues/6) ### A sample Terraform directory structure with `tfustomize` diff --git a/api/hcl_parser.go b/api/hcl_parser.go index 263ef6d..64c6113 100644 --- a/api/hcl_parser.go +++ b/api/hcl_parser.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "regexp" "sort" "strings" @@ -30,6 +31,8 @@ var tfNoLabelBlockTypes = []string{ "removed", } +var annotationBlockMergeRegexp = regexp.MustCompile(`tfustomize:merge_block:([\w]+)`) + type HCLParser struct { } @@ -247,15 +250,49 @@ func mergeBlock(baseBlock *hclwrite.Block, overlayBlock *hclwrite.Block) (*hclwr setBodyAttribute(resultBlockBody, name, tmpAttributes[name]) } - // TODO: User can choose patch or append block - // append blocks that are defined in overlay + tmpBlocksForMerge := map[string]*hclwrite.Block{} + tmpBlocksForAppend := []*hclwrite.Block{} + for _, baseBlockBodyBlock := range baseBlockBody.Blocks() { - resultBlockBody.AppendNewline() - resultBlockBody.AppendBlock(baseBlockBodyBlock) + annotationForBlockMerge := annotationBlockMergeRegexp.Find(baseBlockBodyBlock.Body().BuildTokens(nil).Bytes()) + if annotationForBlockMerge != nil { + mergeKey := string(annotationForBlockMerge) + slog.Debug("annotation is found in the base blocks", "annotation", mergeKey) + + tmpBlocksForMerge[mergeKey] = baseBlockBodyBlock + } else { + tmpBlocksForAppend = append(tmpBlocksForAppend, baseBlockBodyBlock) + } } + for _, overlayBlockBodyBlock := range overlayBlockBody.Blocks() { + annotationForBlockMerge := annotationBlockMergeRegexp.Find(overlayBlockBodyBlock.Body().BuildTokens(nil).Bytes()) + if annotationForBlockMerge != nil { + mergeKey := string(annotationForBlockMerge) + + if _, ok := tmpBlocksForMerge[mergeKey]; ok { + slog.Debug("annotation is found in the base and the overlay blocks", "annotation", mergeKey) + + mergedBlock, err := mergeBlock(tmpBlocksForMerge[mergeKey], overlayBlockBodyBlock) + if err != nil { + return nil, err + } + tmpBlocksForMerge[mergeKey] = mergedBlock + } else { + slog.Debug("annotation is found but it is not in the base blocks", "annotation", mergeKey) + } + } else { + tmpBlocksForAppend = append(tmpBlocksForAppend, overlayBlockBodyBlock) + } + } + + for _, block := range tmpBlocksForAppend { + resultBlockBody.AppendNewline() + resultBlockBody.AppendBlock(block) + } + for _, block := range tmpBlocksForMerge { resultBlockBody.AppendNewline() - resultBlockBody.AppendBlock(overlayBlockBodyBlock) + resultBlockBody.AppendBlock(block) } return resultBlock, nil diff --git a/api/hcl_parser_test.go b/api/hcl_parser_test.go index 78b79e3..ca47ade 100644 --- a/api/hcl_parser_test.go +++ b/api/hcl_parser_test.go @@ -184,12 +184,26 @@ func TestMergeFileBlocks(t *testing.T) { overlay: []string{"overlay/data_with_block.tf"}, expect: `data "aws_ami" "ubuntu" { filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + # tfustomize:merge_block:name name = "name" values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] } +} +`, + wantErr: false, + }, + { + name: "data source with merge block test", + base: []string{"base/data_with_block.tf"}, + overlay: []string{"overlay/data_with_block_merge.tf"}, + expect: `data "aws_ami" "ubuntu" { filter { - name = "virtualization-type" - values = ["hvm"] + name = "name_is_updated" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] } } `, diff --git a/go.mod b/go.mod index 751a1f2..e72acac 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/hashicorp/hcl/v2 v2.20.1 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f ) require ( @@ -20,9 +21,8 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/zclconf/go-cty v1.13.2 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.20.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 4cd8b51..beb6f7e 100644 --- a/go.sum +++ b/go.sum @@ -40,21 +40,13 @@ github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY3 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/test/base/data_with_block.tf b/test/base/data_with_block.tf index ba4188d..eab97af 100644 --- a/test/base/data_with_block.tf +++ b/test/base/data_with_block.tf @@ -1,5 +1,6 @@ data "aws_ami" "ubuntu" { filter { + # tfustomize:merge_block:name name = "name" values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] } diff --git a/test/base/main.tf b/test/base/main.tf index 497b582..ad5f247 100644 --- a/test/base/main.tf +++ b/test/base/main.tf @@ -8,6 +8,7 @@ data "aws_ami" "ubuntu" { most_recent = true filter { + # tfustomize:merge_block:name name = "name" values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] } diff --git a/test/overlay/data_with_block_merge.tf b/test/overlay/data_with_block_merge.tf new file mode 100644 index 0000000..45af507 --- /dev/null +++ b/test/overlay/data_with_block_merge.tf @@ -0,0 +1,8 @@ +data "aws_ami" "ubuntu" { + + filter { + # tfustomize:merge_block:name + name = "name_is_updated" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] + } +} diff --git a/test/overlay/data_with_block_merge_and_append.tf b/test/overlay/data_with_block_merge_and_append.tf new file mode 100644 index 0000000..9b9757b --- /dev/null +++ b/test/overlay/data_with_block_merge_and_append.tf @@ -0,0 +1,11 @@ +data "aws_ami" "ubuntu" { + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} diff --git a/test/overlay/overlay.tf b/test/overlay/overlay.tf index 52a12f0..45d6201 100644 --- a/test/overlay/overlay.tf +++ b/test/overlay/overlay.tf @@ -17,6 +17,11 @@ data "aws_ami" "ubuntu" { name = "arch" values = ["arm64"] } + filter { + # tfustomize:merge_block:name + name = "name_is_updated" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"] + } } resource "aws_instance" "be" {