feat: enable to merge blocks inside block (#31)

* add test files

* update test cases

* feat: enable to use merge block inside block

* run go mod tidy

* update readme

* rename annotation to tfustomize:merge_block from tfustomize:block_merge

* fix typo

README.md

@@ -8,7 +8,7 @@
 ## Motivation
 
 [Terraform Modules](https://developer.hashicorp.com/terraform/language/modules) is looks almost the only way to reuse resource configurations with Terraform. There is other option [`terragrunt`](https://terragrunt.gruntwork.io/), but it's looks expanding and wrapping the use of Terraform module.  
-Terraform modules are effective when they are widly spreaded as open sources, or when they are officially provided by kind of Platform Engineers for internal use in private environments. However, creating custom modules for one product seems like overkill.  
+Terraform modules are effective when they are widly spread as open sources, or when they are officially provided by kind of Platform Engineers for internal use in private environments. However, creating custom modules for one product seems like overkill.  
 It was quite labor-intensive to transition from a copy-paste style of management to using custom modules, so I created something like a Terraform version of kustomize as a proof of concept.  
 [Override Files](https://developer.hashicorp.com/terraform/language/files/override) feature is looks similar concept with `tfustomize` but overriding is not for reusing purpose.
 
@@ -74,12 +74,49 @@ patches {
 
 ### Merging Behavior and Limitation
 
-- A Top-level block has the same block type and labels in base and overlay will be merged. 
+- A Top-level block has the same block type and labels in base and overlay will be merged.
   - Except `moved`, `import`, `removed` block. These will be appended.
 - `locals` blocks will be merged.
 - Within a top-level block, an attribute argument within an overlay block will be replaced any argument of the same name in the base block.
-- Within a top-level block, any block will be appended. 
-  - [Limitation] can not be replaced (https://github.com/tk3fftk/tfustomize/issues/8) 
+- Within a top-level block, any block will be appended by default.
+  - To merge a block, use an annotation `# tfustimize:merge_block:<key>` both a base and an overlay like below.
+
+```hcl
+# base
+data "aws_ami" "ubuntu" {
+  filter {
+    # tfustomize:merge_block:name
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+  }
+}
+
+# overlay
+data "aws_ami" "ubuntu" {
+  filter {
+    name   = "arch"
+    values = ["arm64"]
+  }
+  filter {
+    # tfustomize:merge_block:name
+    name   = "name_is_updated"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
+  }
+}
+
+# output
+data "aws_ami" "ubuntu" {
+  filter {
+    name   = "arch"
+    values = ["arm64"]
+  }
+  filter {
+    name   = "name_is_updated"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
+  }
+}
+```
+
 - [Limitation] The output order is randomized inside block level order. (https://github.com/tk3fftk/tfustomize/issues/6)
 
 ### A sample Terraform directory structure with `tfustomize`

api/hcl_parser.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 
@@ -30,6 +31,8 @@ var tfNoLabelBlockTypes = []string{
 	"removed",
 }
 
+var annotationBlockMergeRegexp = regexp.MustCompile(`tfustomize:merge_block:([\w]+)`)
+
 type HCLParser struct {
 }
 
@@ -247,15 +250,49 @@ func mergeBlock(baseBlock *hclwrite.Block, overlayBlock *hclwrite.Block) (*hclwr
 		setBodyAttribute(resultBlockBody, name, tmpAttributes[name])
 	}
 
-	// TODO: User can choose patch or append block
-	// append blocks that are defined in overlay
+	tmpBlocksForMerge := map[string]*hclwrite.Block{}
+	tmpBlocksForAppend := []*hclwrite.Block{}
+
 	for _, baseBlockBodyBlock := range baseBlockBody.Blocks() {
-		resultBlockBody.AppendNewline()
-		resultBlockBody.AppendBlock(baseBlockBodyBlock)
+		annotationForBlockMerge := annotationBlockMergeRegexp.Find(baseBlockBodyBlock.Body().BuildTokens(nil).Bytes())
+		if annotationForBlockMerge != nil {
+			mergeKey := string(annotationForBlockMerge)
+			slog.Debug("annotation is found in the base blocks", "annotation", mergeKey)
+
+			tmpBlocksForMerge[mergeKey] = baseBlockBodyBlock
+		} else {
+			tmpBlocksForAppend = append(tmpBlocksForAppend, baseBlockBodyBlock)
+		}
 	}
+
 	for _, overlayBlockBodyBlock := range overlayBlockBody.Blocks() {
+		annotationForBlockMerge := annotationBlockMergeRegexp.Find(overlayBlockBodyBlock.Body().BuildTokens(nil).Bytes())
+		if annotationForBlockMerge != nil {
+			mergeKey := string(annotationForBlockMerge)
+
+			if _, ok := tmpBlocksForMerge[mergeKey]; ok {
+				slog.Debug("annotation is found in the base and the overlay blocks", "annotation", mergeKey)
+
+				mergedBlock, err := mergeBlock(tmpBlocksForMerge[mergeKey], overlayBlockBodyBlock)
+				if err != nil {
+					return nil, err
+				}
+				tmpBlocksForMerge[mergeKey] = mergedBlock
+			} else {
+				slog.Debug("annotation is found but it is not in the base blocks", "annotation", mergeKey)
+			}
+		} else {
+			tmpBlocksForAppend = append(tmpBlocksForAppend, overlayBlockBodyBlock)
+		}
+	}
+
+	for _, block := range tmpBlocksForAppend {
+		resultBlockBody.AppendNewline()
+		resultBlockBody.AppendBlock(block)
+	}
+	for _, block := range tmpBlocksForMerge {
 		resultBlockBody.AppendNewline()
-		resultBlockBody.AppendBlock(overlayBlockBodyBlock)
+		resultBlockBody.AppendBlock(block)
 	}
 
 	return resultBlock, nil

api/hcl_parser_test.go

@@ -184,12 +184,26 @@ func TestMergeFileBlocks(t *testing.T) {
 			overlay: []string{"overlay/data_with_block.tf"},
 			expect: `data "aws_ami" "ubuntu" {
   filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+  filter {
+    # tfustomize:merge_block:name
     name   = "name"
     values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
   }
+}
+`,
+			wantErr: false,
+		},
+		{
+			name:    "data source with merge block test",
+			base:    []string{"base/data_with_block.tf"},
+			overlay: []string{"overlay/data_with_block_merge.tf"},
+			expect: `data "aws_ami" "ubuntu" {
   filter {
-    name   = "virtualization-type"
-    values = ["hvm"]
+    name   = "name_is_updated"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
   }
 }
 `,

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.20.1
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
 )
 
 require (
@@ -20,9 +21,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/zclconf/go-cty v1.13.2 // indirect
-	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect

go.sum

@@ -40,21 +40,13 @@ github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY3
 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY=
 golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=

test/base/data_with_block.tf

@@ -1,5 +1,6 @@
 data "aws_ami" "ubuntu" {
   filter {
+    # tfustomize:merge_block:name
     name   = "name"
     values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
   }

test/base/main.tf

@@ -8,6 +8,7 @@ data "aws_ami" "ubuntu" {
   most_recent = true
 
   filter {
+    # tfustomize:merge_block:name
     name   = "name"
     values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
   }

test/overlay/data_with_block_merge.tf

@@ -0,0 +1,8 @@
+data "aws_ami" "ubuntu" {
+
+  filter {
+    # tfustomize:merge_block:name
+    name   = "name_is_updated"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
+  }
+}

test/overlay/data_with_block_merge_and_append.tf

@@ -0,0 +1,11 @@
+data "aws_ami" "ubuntu" {
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}

test/overlay/overlay.tf

@@ -17,6 +17,11 @@ data "aws_ami" "ubuntu" {
     name   = "arch"
     values = ["arm64"]
   }
+  filter {
+    # tfustomize:merge_block:name
+    name   = "name_is_updated"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-24.04-amd64-server-*"]
+  }
 }
 
 resource "aws_instance" "be" {