Merge pull request #122 from linuxboot/feature/expand_variables_to_pa…

…rameters

Parameter.Expand consumes optional StepsVariables argument

pkg/test/functions.go

@@ -32,6 +32,47 @@ func getFuncMap() map[string]interface{} {
 	return mapCopy
 }
 
+func parseLabelDotVariable(labelDotVariable string) (string, string, error) {
+	parts := strings.SplitN(labelDotVariable, ".", 2)
+	if len(parts) < 2 {
+		return "", "", fmt.Errorf("not 'label.variable' format of '%s'", labelDotVariable)
+	}
+	stepLabel, varname := parts[0], parts[1]
+	if err := CheckIdentifier(stepLabel); err != nil {
+		return "", "", fmt.Errorf("invalid step label: '%s': %w", stepLabel, err)
+	}
+	if err := CheckIdentifier(varname); err != nil {
+		return "", "", fmt.Errorf("invalid variable name: '%s': %w", varname, err)
+	}
+	return stepLabel, varname, nil
+}
+
+func registerStepVariableAccessor(fm map[string]interface{}, tgtID string, vars StepsVariablesReader) {
+	fm["StringVar"] = func(labelDotVariable string) (string, error) {
+		stepLabel, varName, err := parseLabelDotVariable(labelDotVariable)
+		if err != nil {
+			return "", err
+		}
+
+		var s string
+		if err := vars.Get(tgtID, stepLabel, varName, &s); err != nil {
+			return "", err
+		}
+		return s, nil
+	}
+	fm["IntVar"] = func(labelDotVariable string) (int, error) {
+		stepLabel, varName, err := parseLabelDotVariable(labelDotVariable)
+		if err != nil {
+			return 0, err
+		}
+		var i int
+		if err := vars.Get(tgtID, stepLabel, varName, &i); err != nil {
+			return 0, err
+		}
+		return i, nil
+	}
+}
+
 // RegisterFunction registers a template function suitable for text/template.
 // It can be either a func(string) string or a func(string) (string, error),
 // hence it's passed as an empty interface.

pkg/test/param_expander.go

@@ -13,16 +13,17 @@ import (
 )
 
 type ParamExpander struct {
-	t *target.Target
+	t    *target.Target
+	vars StepsVariablesReader
 }
 
-func NewParamExpander(target *target.Target) *ParamExpander {
-	return &ParamExpander{target}
+func NewParamExpander(target *target.Target, vars StepsVariablesReader) *ParamExpander {
+	return &ParamExpander{t: target, vars: vars}
 }
 
 func (pe *ParamExpander) Expand(value string) (string, error) {
 	p := NewParam(value)
-	return p.Expand(pe.t)
+	return p.Expand(pe.t, pe.vars)
 }
 
 func (pe *ParamExpander) ExpandObject(obj interface{}, out interface{}) error {

pkg/test/parameter.go

@@ -57,12 +57,16 @@ func (p Param) JSON() json.RawMessage {
 
 // Expand evaluates the raw expression and applies the necessary manipulation,
 // if any.
-func (p *Param) Expand(target *target.Target) (string, error) {
+func (p *Param) Expand(target *target.Target, vars StepsVariablesReader) (string, error) {
 	if p == nil {
 		return "", errors.New("parameter cannot be nil")
 	}
+	funcs := getFuncMap()
+	if vars != nil {
+		registerStepVariableAccessor(funcs, target.ID, vars)
+	}
 	// use Go text/template from here
-	tmpl, err := template.New("").Funcs(getFuncMap()).Parse(p.String())
+	tmpl, err := template.New("").Funcs(funcs).Parse(p.String())
 	if err != nil {
 		return "", fmt.Errorf("failed to parse template: %v", err)
 	}

pkg/test/parameter_test.go

@@ -6,7 +6,9 @@
 package test
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
 	"strings"
 	"testing"
 
@@ -24,7 +26,7 @@ func TestParameterExpand(t *testing.T) {
 	}
 	for _, x := range validExprs {
 		p := NewParam(x[0])
-		res, err := p.Expand(&target.Target{FQDN: x[1], ID: x[2]})
+		res, err := p.Expand(&target.Target{FQDN: x[1], ID: x[2]}, nil)
 		require.NoError(t, err, x[0])
 		require.Equal(t, x[3], res, x[0])
 	}
@@ -46,10 +48,91 @@ func TestParameterExpandUserFunctions(t *testing.T) {
 	}
 	for _, x := range validExprs {
 		p := NewParam(x[0])
-		res, err := p.Expand(&target.Target{FQDN: x[1], ID: x[2]})
+		res, err := p.Expand(&target.Target{FQDN: x[1], ID: x[2]}, nil)
 		require.NoError(t, err, x[0])
 		require.Equal(t, x[3], res, x[0])
 	}
 	require.NoError(t, UnregisterFunction("CustomFunc"))
 	require.Error(t, UnregisterFunction("NoSuchFunction"))
 }
+
+func TestStepVariablesExpand(t *testing.T) {
+	p := NewParam("{{ StringVar \"step1.string_var\" }}: {{ IntVar \"step1.int_var\" }}")
+	svm := newStepsVariablesMock()
+
+	tgt := target.Target{ID: "1"}
+	require.NoError(t, svm.add(tgt.ID, "step1", "string_var", "Hello"))
+	require.NoError(t, svm.add(tgt.ID, "step1", "int_var", 42))
+
+	res, err := p.Expand(&tgt, svm)
+	require.NoError(t, err)
+	require.Equal(t, "Hello: 42", res)
+}
+
+func TestInvalidStepVariablesExpand(t *testing.T) {
+	t.Run("no_dot", func(t *testing.T) {
+		p := NewParam("{{ StringVar \"step1string_var\" }}")
+		_, err := p.Expand(&target.Target{ID: "1"}, newStepsVariablesMock())
+		require.Error(t, err)
+	})
+
+	t.Run("just_variable_name", func(t *testing.T) {
+		p := NewParam("{{ StringVar \"string_var\" }}")
+
+		svm := newStepsVariablesMock()
+		tgt := target.Target{ID: "1"}
+		require.NoError(t, svm.add(tgt.ID, "step1", "string_var", "Hello"))
+
+		_, err := p.Expand(&tgt, svm)
+		require.Error(t, err)
+	})
+
+	t.Run("invalid_variable_name", func(t *testing.T) {
+		p := NewParam("{{ StringVar \"step1.22string_var\" }}")
+
+		svm := newStepsVariablesMock()
+		tgt := target.Target{ID: "1"}
+		// we can add invalid values to our mock
+		require.NoError(t, svm.add(tgt.ID, "step1", "22string_var", "Hello"))
+
+		_, err := p.Expand(&tgt, svm)
+		require.Error(t, err)
+	})
+}
+
+type stepsVariablesMock struct {
+	variables map[string]map[string]json.RawMessage
+}
+
+func newStepsVariablesMock() *stepsVariablesMock {
+	return &stepsVariablesMock{
+		variables: make(map[string]map[string]json.RawMessage),
+	}
+}
+
+func (svm *stepsVariablesMock) add(tgtID string, label, name string, in interface{}) error {
+	b, err := json.Marshal(in)
+	if err != nil {
+		return err
+	}
+
+	targetVars := svm.variables[tgtID]
+	if targetVars == nil {
+		targetVars = make(map[string]json.RawMessage)
+		svm.variables[tgtID] = targetVars
+	}
+	targetVars[label+"."+name] = b
+	return nil
+}
+
+func (svm *stepsVariablesMock) Get(tgtID string, stepLabel, name string, out interface{}) error {
+	targetVars := svm.variables[tgtID]
+	if targetVars == nil {
+		return fmt.Errorf("no target: %s", tgtID)
+	}
+	b, found := targetVars[stepLabel+"."+name]
+	if !found {
+		return fmt.Errorf("no variable %s %s", stepLabel, name)
+	}
+	return json.Unmarshal(b, out)
+}

pkg/test/step.go

@@ -109,7 +109,7 @@ type TestStepChannels struct {
 	Out chan<- TestStepResult
 }
 
-// StepsVariables represents a read/write access for step variables
+// StepsVariablesReader represents a read access for step variables
 // Example:
 // var sv StepsVariables
 // intVar := 42
@@ -118,12 +118,16 @@ type TestStepChannels struct {
 // var recvIntVar int
 // sv.Get(("dummy-target-id", "varname", &recvIntVar)
 // assert recvIntVar == 42
+type StepsVariablesReader interface {
+	// Get obtains existing variable that was added in one of the previous steps
+	Get(tgtID string, stepLabel, name string, out interface{}) error
+}
+
+// StepsVariables represents a read/write access for step variables
 type StepsVariables interface {
+	StepsVariablesReader
 	// Add adds a new or replaces existing variable associated with current test step and target
 	Add(tgtID string, name string, in interface{}) error
-
-	// Get obtains existing variable by a mappedName which should be specified in variables mapping
-	Get(tgtID string, stepLabel, name string, out interface{}) error
 }
 
 // TestStep is the interface that all steps need to implement to be executed

plugins/teststeps/cmd/cmd.go

@@ -110,14 +110,14 @@ func (ts *Cmd) Run(
 		// expand args
 		var args []string
 		for _, arg := range ts.args {
-			expArg, err := arg.Expand(target)
+			expArg, err := arg.Expand(target, stepsVars)
 			if err != nil {
 				return fmt.Errorf("failed to expand argument '%s': %v", arg, err)
 			}
 			args = append(args, expArg)
 		}
 		cmd := exec.CommandContext(ctx, ts.executable, args...)
-		pwd, err := ts.dir.Expand(target)
+		pwd, err := ts.dir.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("failed to expand argument dir '%s': %v", ts.dir, err)
 		}

plugins/teststeps/cpucmd/cpucmd.go

@@ -99,7 +99,7 @@ func (ts *CPUCmd) Run(
 		// 	return fmt.Errorf("cannot expand user parameter: %v", err)
 		// }
 
-		host, err := ts.Host.Expand(target)
+		host, err := ts.Host.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("cannot expand host parameter: %v", err)
 		}
@@ -120,7 +120,7 @@ func (ts *CPUCmd) Run(
 			return fmt.Errorf("host value is empty")
 		}
 
-		portStr, err := ts.Port.Expand(target)
+		portStr, err := ts.Port.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("cannot expand port parameter: %v", err)
 		}
@@ -130,7 +130,7 @@ func (ts *CPUCmd) Run(
 		if err != nil {
 			return fmt.Errorf("Can not expand %q:%q to a cpu port", host, portStr)
 		}
-		timeoutStr, err := ts.Timeout.Expand(target)
+		timeoutStr, err := ts.Timeout.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("cannot expand timeout parameter %s: %v", timeoutStr, err)
 		}
@@ -142,20 +142,20 @@ func (ts *CPUCmd) Run(
 
 		timeTimeout := time.Now().Add(timeout)
 
-		privKeyFile, err := ts.PrivateKeyFile.Expand(target)
+		privKeyFile, err := ts.PrivateKeyFile.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("cannot expand private key file parameter: %v", err)
 		}
 
-		executable, err := ts.Executable.Expand(target)
+		executable, err := ts.Executable.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("cannot expand executable parameter: %v", err)
 		}
 
 		// apply functions to the command args, if any
 		args := []string{executable}
 		for _, arg := range ts.Args {
-			earg, err := arg.Expand(target)
+			earg, err := arg.Expand(target, stepsVars)
 			if err != nil {
 				return fmt.Errorf("cannot expand command argument '%s': %v", arg, err)
 			}

plugins/teststeps/echo/echo.go

@@ -65,7 +65,7 @@ func (e Step) Run(
 			if !ok {
 				return nil, nil
 			}
-			output, err := params.GetOne("text").Expand(target)
+			output, err := params.GetOne("text").Expand(target, stepsVars)
 			if err != nil {
 				return nil, err
 			}

plugins/teststeps/exec/exec.go

@@ -64,7 +64,7 @@ func (ts *TestStep) Run(
 		return nil, err
 	}
 
-	tr := NewTargetRunner(ts, ev)
+	tr := NewTargetRunner(ts, ev, stepsVars)
 	return teststeps.ForEachTarget(Name, ctx, ch, tr.Run)
 }
 

plugins/teststeps/exec/runner.go

@@ -21,11 +21,12 @@ import (
 type outcome error
 
 type TargetRunner struct {
-	ts *TestStep
-	ev testevent.Emitter
+	ts        *TestStep
+	ev        testevent.Emitter
+	stepsVars test.StepsVariablesReader
 }
 
-func NewTargetRunner(ts *TestStep, ev testevent.Emitter) *TargetRunner {
+func NewTargetRunner(ts *TestStep, ev testevent.Emitter, stepsVars test.StepsVariablesReader) *TargetRunner {
 	return &TargetRunner{
 		ts: ts,
 		ev: ev,
@@ -127,7 +128,7 @@ func (r *TargetRunner) Run(ctx xcontext.Context, target *target.Target) error {
 		defer cancel()
 	}
 
-	pe := test.NewParamExpander(target)
+	pe := test.NewParamExpander(target, r.stepsVars)
 
 	var params stepParams
 	if err := pe.ExpandObject(r.ts.stepParams, &params); err != nil {

plugins/teststeps/gathercmd/gathercmd.go

@@ -250,9 +250,9 @@ func (ts *GatherCmd) setParams(params test.TestStepParameters) error {
 
 	ts.args = []string{}
 	args := params.Get("args")
-	// expand args in case they use functions, but they shouldnt be target aware
+	// expand args in case they use functions, but they shouldn't be target aware
 	for _, arg := range args {
-		expanded, err := arg.Expand(&target.Target{})
+		expanded, err := arg.Expand(&target.Target{}, nil)
 		if err != nil {
 			return fmt.Errorf("failed to expand argument: %s -> %v", arg, err)
 		}

plugins/teststeps/s3fileupload/s3fileupload.go

@@ -97,11 +97,11 @@ func (ts *FileUpload) Run(
 	}
 	f := func(ctx xcontext.Context, target *target.Target) error {
 		// expand args
-		path, err := ts.localPath.Expand(target)
+		path, err := ts.localPath.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("failed to expand argument '%s': %v", ts.localPath, err)
 		}
-		filename, err := ts.fileName.Expand(target)
+		filename, err := ts.fileName.Expand(target, stepsVars)
 		if err != nil {
 			return fmt.Errorf("failed to expand argument dir '%s': %v", ts.fileName, err)
 		}