Merge pull request #30 from Scalr/SCALRCORE-31913

SCALRCORE-31913 pygohcl > Improve configuration parsing

hcl_to_json.go

@@ -16,20 +16,21 @@ type jsonObj map[string]interface{}
 
 // Convert an hcl File to a json serializable object
 // This assumes that the body is a hclsyntax.Body
-func convertFile(file *hcl.File) (jsonObj, error) {
-	c := converter{bytes: file.Bytes}
+func convertFile(file *hcl.File, keepInterp bool) (jsonObj, error) {
+	c := converter{bytes: file.Bytes, keepInterp: keepInterp}
 	body := file.Body.(*hclsyntax.Body)
 	return c.convertBody(body)
 }
 
 type converter struct {
-	bytes []byte
+	bytes      []byte
+	keepInterp bool
 }
 
 func (c *converter) rangeSource(r hcl.Range) string {
 	data := string(c.bytes[r.Start.Byte:r.End.Byte])
-	data = strings.ReplaceAll(data, "\n", "")
-	data = strings.ReplaceAll(data, " ", "")
+	data = strings.ReplaceAll(data, "\n", " ")
+	data = strings.Join(strings.Fields(data), " ")
 	return data
 }
 
@@ -165,6 +166,9 @@ func (c *converter) convertStringPart(expr hclsyntax.Expression) (string, error)
 		return c.convertTemplateConditional(v)
 	case *hclsyntax.TemplateJoinExpr:
 		return c.convertTemplateFor(v.Tuple.(*hclsyntax.ForExpr))
+	case *hclsyntax.ScopeTraversalExpr:
+		return c.wrapTraversal(expr), nil
+
 	default:
 		// treating as an embedded expression
 		return c.wrapExpr(expr), nil
@@ -227,6 +231,14 @@ func (c *converter) wrapExpr(expr hclsyntax.Expression) string {
 	return c.rangeSource(expr.Range())
 }
 
+func (c *converter) wrapTraversal(expr hclsyntax.Expression) string {
+	res := c.wrapExpr(expr)
+	if c.keepInterp {
+		res = "${" + res + "}"
+	}
+	return res
+}
+
 func (c *converter) convertUnary(v *hclsyntax.UnaryOpExpr) (interface{}, error) {
 	_, isLiteral := v.Val.(*hclsyntax.LiteralValueExpr)
 	if !isLiteral {

pygohcl.go

@@ -8,6 +8,8 @@ import "C"
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
+
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/ext/tryfunc"
 	"github.com/hashicorp/hcl/v2/hclparse"
@@ -16,11 +18,10 @@ import (
 	"github.com/zclconf/go-cty/cty/convert"
 	"github.com/zclconf/go-cty/cty/function"
 	"github.com/zclconf/go-cty/cty/function/stdlib"
-	"strings"
 )
 
 //export Parse
-func Parse(a *C.char) (resp C.parseResponse) {
+func Parse(a *C.char, keepInterpFlag C.int) (resp C.parseResponse) {
 	defer func() {
 		if err := recover(); err != nil {
 			retValue := fmt.Sprintf("panic HCL: %v", err)
@@ -29,11 +30,12 @@ func Parse(a *C.char) (resp C.parseResponse) {
 	}()
 
 	input := C.GoString(a)
+	keepInterp := keepInterpFlag == 1
 	hclFile, diags := hclparse.NewParser().ParseHCL([]byte(input), "tmp.hcl")
 	if diags.HasErrors() {
 		return C.parseResponse{nil, C.CString(diagErrorsToString(diags, "invalid HCL: %s"))}
 	}
-	hclMap, err := convertFile(hclFile)
+	hclMap, err := convertFile(hclFile, keepInterp)
 	if err != nil {
 		return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot convert HCL to Go map representation: %s", err))}
 	}
@@ -63,7 +65,7 @@ func ParseAttributes(a *C.char) (resp C.parseResponse) {
 
 	var diags hcl.Diagnostics
 	hclMap := make(jsonObj)
-	c := converter{[]byte(input)}
+	c := converter{[]byte(input), false}
 
 	attrs, attrsDiags := hclFile.Body.JustAttributes()
 	diags = diags.Extend(attrsDiags)

pygohcl/__init__.py

@@ -32,9 +32,15 @@ class UnknownFunctionError(ValidationError):
     pass
 
 
-def loadb(data: bytes) -> tp.Dict:
+def loadb(data: bytes, keep_interpolations: bool = False) -> tp.Dict:
+    """
+    Parse and load HCL input into Python dictionary.
+    :param data: HCL to parse.
+    :param keep_interpolations: Set to True
+        to preserve template interpolation sequences (${ ... }) in strings. Defaults to False.
+    """
     s = ffi.new("char[]", data)
-    ret = lib.Parse(s)
+    ret = lib.Parse(s, int(keep_interpolations))
     if ret.err != ffi.NULL:
         err: bytes = ffi.string(ret.err)
         ffi.gc(ret.err, lib.free)
@@ -47,13 +53,13 @@ def loadb(data: bytes) -> tp.Dict:
     return json.loads(ret_json)
 
 
-def loads(data: str) -> tp.Dict:
-    return loadb(data.encode("utf8"))
+def loads(data: str, keep_interpolations: bool = False) -> tp.Dict:
+    return loadb(data.encode("utf8"), keep_interpolations)
 
 
-def load(stream: tp.IO) -> tp.Dict:
+def load(stream: tp.IO, keep_interpolations: bool = False) -> tp.Dict:
     data = stream.read()
-    return loadb(data)
+    return loadb(data, keep_interpolations)
 
 
 def attributes_loadb(data: bytes) -> tp.Dict:

pygohcl/build_cffi.py

@@ -17,7 +17,7 @@
             char *err;
         } parseResponse;
  
-        parseResponse Parse(char* a);
+        parseResponse Parse(char* a, int keepInterpFlag);
         parseResponse ParseAttributes(char* a);
         char* EvalValidationRule(char* c, char* e, char* n, char* v);
         void free(void *ptr);

tests/test_pygohcl.py

@@ -41,11 +41,11 @@ def test_numbers():
         }"""
     )
     assert out["locals"]["a"] == 0.19
-    assert out["locals"]["b"] == "1+9"
+    assert out["locals"]["b"] == "1 + 9"
     assert out["locals"]["c"] == -0.82
     assert out["locals"]["x"] == -10
     assert out["locals"]["y"] == "-x"
-    assert out["locals"]["z"] == "-(1+4)"
+    assert out["locals"]["z"] == "-(1 + 4)"
 
 
 def test_value_is_null():
@@ -64,3 +64,36 @@ def test_namespaced_functions():
     """locals {
         timestamp = provider::time::rfc3339_parse(plantimestamp())
     }""") == {"locals": {"timestamp": "provider::time::rfc3339_parse(plantimestamp())"}}
+
+
+def test_expression_whitespace_trimming():
+    out = pygohcl.loads("""
+    variable "a" {
+	  validation {
+	    condition = alltrue(
+	    
+	      [for
+	        value in values(var.a)   :
+	          regex("[a-z]+",value.my_key) == "val"
+	      ]
+	    )
+	  }
+	}""")
+    assert (out["variable"]["a"]["validation"]["condition"]
+        == "alltrue( [for value in values(var.a) : regex(\"[a-z]+\",value.my_key) == \"val\" ] )")
+
+
+def test_keep_template_interpolation():
+    s = """
+	variable "a" {
+	 validation {
+	   error_message = "Nope, ${var.a} is less than 5."
+	 }
+	}"""
+
+    assert pygohcl.loads(s) == {
+        "variable": {"a": {"validation": {"error_message": "Nope, var.a is less than 5."}}}
+    }
+    assert pygohcl.loads(s, True) == {
+        "variable": {"a": {"validation": {"error_message": "Nope, ${var.a} is less than 5."}}}
+    }