From a051dd9110b46b3784f8aa3592d8de6f0b57a8f4 Mon Sep 17 00:00:00 2001 From: Petro Protsakh Date: Mon, 30 Sep 2024 11:14:59 +0300 Subject: [PATCH] SCALRCORE-31913 Fix whitespace trimming corrupting expressions; add option to keep interpolation sequences. --- hcl_to_json.go | 22 +++++++++++++++++----- pygohcl.go | 10 ++++++---- pygohcl/__init__.py | 18 ++++++++++++------ pygohcl/build_cffi.py | 2 +- tests/test_pygohcl.py | 37 +++++++++++++++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/hcl_to_json.go b/hcl_to_json.go index 62c85a0..4241f57 100644 --- a/hcl_to_json.go +++ b/hcl_to_json.go @@ -16,20 +16,21 @@ type jsonObj map[string]interface{} // Convert an hcl File to a json serializable object // This assumes that the body is a hclsyntax.Body -func convertFile(file *hcl.File) (jsonObj, error) { - c := converter{bytes: file.Bytes} +func convertFile(file *hcl.File, keepInterp bool) (jsonObj, error) { + c := converter{bytes: file.Bytes, keepInterp: keepInterp} body := file.Body.(*hclsyntax.Body) return c.convertBody(body) } type converter struct { - bytes []byte + bytes []byte + keepInterp bool } func (c *converter) rangeSource(r hcl.Range) string { data := string(c.bytes[r.Start.Byte:r.End.Byte]) - data = strings.ReplaceAll(data, "\n", "") - data = strings.ReplaceAll(data, " ", "") + data = strings.ReplaceAll(data, "\n", " ") + data = strings.Join(strings.Fields(data), " ") return data } @@ -165,6 +166,9 @@ func (c *converter) convertStringPart(expr hclsyntax.Expression) (string, error) return c.convertTemplateConditional(v) case *hclsyntax.TemplateJoinExpr: return c.convertTemplateFor(v.Tuple.(*hclsyntax.ForExpr)) + case *hclsyntax.ScopeTraversalExpr: + return c.wrapTraversal(expr), nil + default: // treating as an embedded expression return c.wrapExpr(expr), nil @@ -227,6 +231,14 @@ func (c *converter) wrapExpr(expr hclsyntax.Expression) string { return c.rangeSource(expr.Range()) } +func (c *converter) wrapTraversal(expr hclsyntax.Expression) string { + res := c.wrapExpr(expr) + if c.keepInterp { + res = "${" + res + "}" + } + return res +} + func (c *converter) convertUnary(v *hclsyntax.UnaryOpExpr) (interface{}, error) { _, isLiteral := v.Val.(*hclsyntax.LiteralValueExpr) if !isLiteral { diff --git a/pygohcl.go b/pygohcl.go index d1f1626..0fb6110 100644 --- a/pygohcl.go +++ b/pygohcl.go @@ -8,6 +8,8 @@ import "C" import ( "encoding/json" "fmt" + "strings" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/ext/tryfunc" "github.com/hashicorp/hcl/v2/hclparse" @@ -16,11 +18,10 @@ import ( "github.com/zclconf/go-cty/cty/convert" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/function/stdlib" - "strings" ) //export Parse -func Parse(a *C.char) (resp C.parseResponse) { +func Parse(a *C.char, keepInterpFlag C.int) (resp C.parseResponse) { defer func() { if err := recover(); err != nil { retValue := fmt.Sprintf("panic HCL: %v", err) @@ -29,11 +30,12 @@ func Parse(a *C.char) (resp C.parseResponse) { }() input := C.GoString(a) + keepInterp := keepInterpFlag == 1 hclFile, diags := hclparse.NewParser().ParseHCL([]byte(input), "tmp.hcl") if diags.HasErrors() { return C.parseResponse{nil, C.CString(diagErrorsToString(diags, "invalid HCL: %s"))} } - hclMap, err := convertFile(hclFile) + hclMap, err := convertFile(hclFile, keepInterp) if err != nil { return C.parseResponse{nil, C.CString(fmt.Sprintf("cannot convert HCL to Go map representation: %s", err))} } @@ -63,7 +65,7 @@ func ParseAttributes(a *C.char) (resp C.parseResponse) { var diags hcl.Diagnostics hclMap := make(jsonObj) - c := converter{[]byte(input)} + c := converter{[]byte(input), false} attrs, attrsDiags := hclFile.Body.JustAttributes() diags = diags.Extend(attrsDiags) diff --git a/pygohcl/__init__.py b/pygohcl/__init__.py index cd4c59e..ed25e54 100644 --- a/pygohcl/__init__.py +++ b/pygohcl/__init__.py @@ -32,9 +32,15 @@ class UnknownFunctionError(ValidationError): pass -def loadb(data: bytes) -> tp.Dict: +def loadb(data: bytes, keep_interpolations: bool = False) -> tp.Dict: + """ + Parse and load HCL input into Python dictionary. + :param data: HCL to parse. + :param keep_interpolations: Set to True + to preserve template interpolation sequences (${ ... }) in strings. Defaults to False. + """ s = ffi.new("char[]", data) - ret = lib.Parse(s) + ret = lib.Parse(s, int(keep_interpolations)) if ret.err != ffi.NULL: err: bytes = ffi.string(ret.err) ffi.gc(ret.err, lib.free) @@ -47,13 +53,13 @@ def loadb(data: bytes) -> tp.Dict: return json.loads(ret_json) -def loads(data: str) -> tp.Dict: - return loadb(data.encode("utf8")) +def loads(data: str, keep_interpolations: bool = False) -> tp.Dict: + return loadb(data.encode("utf8"), keep_interpolations) -def load(stream: tp.IO) -> tp.Dict: +def load(stream: tp.IO, keep_interpolations: bool = False) -> tp.Dict: data = stream.read() - return loadb(data) + return loadb(data, keep_interpolations) def attributes_loadb(data: bytes) -> tp.Dict: diff --git a/pygohcl/build_cffi.py b/pygohcl/build_cffi.py index de5098f..5e4c3aa 100644 --- a/pygohcl/build_cffi.py +++ b/pygohcl/build_cffi.py @@ -17,7 +17,7 @@ char *err; } parseResponse; - parseResponse Parse(char* a); + parseResponse Parse(char* a, int keepInterpFlag); parseResponse ParseAttributes(char* a); char* EvalValidationRule(char* c, char* e, char* n, char* v); void free(void *ptr); diff --git a/tests/test_pygohcl.py b/tests/test_pygohcl.py index 5b2d2fc..5e4bd49 100644 --- a/tests/test_pygohcl.py +++ b/tests/test_pygohcl.py @@ -41,11 +41,11 @@ def test_numbers(): }""" ) assert out["locals"]["a"] == 0.19 - assert out["locals"]["b"] == "1+9" + assert out["locals"]["b"] == "1 + 9" assert out["locals"]["c"] == -0.82 assert out["locals"]["x"] == -10 assert out["locals"]["y"] == "-x" - assert out["locals"]["z"] == "-(1+4)" + assert out["locals"]["z"] == "-(1 + 4)" def test_value_is_null(): @@ -64,3 +64,36 @@ def test_namespaced_functions(): """locals { timestamp = provider::time::rfc3339_parse(plantimestamp()) }""") == {"locals": {"timestamp": "provider::time::rfc3339_parse(plantimestamp())"}} + + +def test_expression_whitespace_trimming(): + out = pygohcl.loads(""" + variable "a" { + validation { + condition = alltrue( + + [for + value in values(var.a) : + regex("[a-z]+",value.my_key) == "val" + ] + ) + } + }""") + assert (out["variable"]["a"]["validation"]["condition"] + == "alltrue( [for value in values(var.a) : regex(\"[a-z]+\",value.my_key) == \"val\" ] )") + + +def test_keep_template_interpolation(): + s = """ + variable "a" { + validation { + error_message = "Nope, ${var.a} is less than 5." + } + }""" + + assert pygohcl.loads(s) == { + "variable": {"a": {"validation": {"error_message": "Nope, var.a is less than 5."}}} + } + assert pygohcl.loads(s, True) == { + "variable": {"a": {"validation": {"error_message": "Nope, ${var.a} is less than 5."}}} + }