RFC: pure go stateful fuzz test generator

Purpose of this patch is to drop dependency to fMBT in e2e fuzz test
generation. This patch implements parts that were needed from fMBT in
go and includes an example for generating tests: a model and a logic
what to cover and how to cover it.

Signed-off-by: Antti Kervinen <[email protected]>

test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"fmt"
+
+	m "github.com/containers/nri-plugins/test/gofmbt"
+)
+
+type TestState struct {
+	cpu    int
+	mem    int
+	rescpu int
+	podCpuMem  map[string][2]int
+}
+
+func (s *TestState) String() string {
+	return fmt.Sprintf("[cpu:%d mem:%d pods:%v]", s.cpu, s.mem, s.podCpuMem)
+}
+
+func createPod(pod string, contcount, cpu, mem int) m.Body {
+	return func(current m.State) m.State {
+		s := current.(*TestState)
+		if s.cpu < cpu*contcount || s.mem < mem*contcount {
+			// refuse from state change if not enough resources
+			return nil
+		}
+		if _, ok := s.podCpuMem[pod]; ok {
+			// refuse to create pod if it is already running
+			return nil
+		}
+		newPodCpuMem := make(map[string][2]int)
+		for k, v := range s.podCpuMem {
+			newPodCpuMem[k] = v
+		}
+		newPodCpuMem[pod] = [2]int{cpu*contcount, mem*contcount}
+		return &TestState{
+			cpu: s.cpu - cpu*contcount,
+			mem: s.mem - mem*contcount,
+			podCpuMem: newPodCpuMem,
+		}
+	}
+}
+
+func deletePod(pod string) m.Body {
+	return func(current m.State) m.State {
+		s := current.(*TestState)
+		cpumem, ok := s.podCpuMem[pod]
+		if !ok {
+			// refuse to delete pod if it is not running
+			return nil
+		}
+		newPodCpuMem := make(map[string][2]int)
+		for k, v := range s.podCpuMem {
+			if k != pod {
+				newPodCpuMem[k] = v
+			}
+		}
+		return &TestState{
+			cpu: s.cpu + cpumem[0],
+			mem: s.mem + cpumem[1],
+			podCpuMem: newPodCpuMem,
+		}
+	}
+}
+
+func main() {
+	maxMem := 7500         // 7.5GB
+	maxCpu := 15000        // 15 cores
+	maxReservedCpu := 1000 // 1 core
+	maxTestSteps := 3000
+
+	podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"}
+
+	model := m.NewModel()
+
+	model.From(func(current m.State) []*m.Transition {
+		s := current.(*TestState)
+		return m.When(s.cpu > 0 && s.mem > 0,
+			m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1, 200, 1500)),
+			m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2, 1000, 500)),
+			m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2, 1200, 4500)),
+			m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3, 2000, 500)),
+			m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1, 4200, 100)),
+			m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1, 1200, 50)),
+			m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2, 1900, 300)),
+			m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1, 0, 0)),
+			m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3, 0, 0)))
+	})
+
+	model.From(func(current m.State) []*m.Transition {
+		s := current.(*TestState)
+		ts := []*m.Transition{}
+		for _, pod := range podNames {
+			if _, ok := s.podCpuMem[pod]; ok {
+				ts = append(ts, m.OnAction("NAME=%s kubectl delete pod %s --now", pod, pod).Do(deletePod(pod))...)
+			}
+		}
+		return ts
+	})
+
+	coverer := m.NewCoverer()
+	coverer.CoverActionNameCombinations(3)
+
+	var state m.State
+
+	state = &TestState{
+		cpu:    maxCpu,
+		mem:    maxMem,
+		rescpu: maxReservedCpu,
+	}
+	testStep := 0
+	for testStep < maxTestSteps {
+		path, covStats := coverer.BestPath(model, state, 4)
+		if len(path) == 0 {
+			fmt.Printf("# did not find anything to cover\n")
+			break
+		}
+		for i := 0; i < covStats.StepsToMax+1; i++ {
+			testStep++
+			step := path[i]
+			fmt.Printf("\n# step %d, coverage: %d, state: %v\n", testStep, coverer.Coverage(), state)
+			fmt.Println(step.Action())
+			state = step.EndState()
+			coverer.MarkCovered(step)
+			coverer.UpdateCoverage()
+		}
+	}
+}

test/gofmbt/action.go

@@ -0,0 +1,37 @@
+package gofmbt
+
+import (
+	"fmt"
+)
+
+type Action struct {
+	name   string
+	format string
+	args   []interface{}
+}
+
+func NewAction(format string, args ...interface{}) *Action {
+	return &Action{
+		format: format,
+		args:   args,
+		name:   fmt.Sprintf(format, args...),
+	}
+}
+
+func (a *Action) String() string {
+	return a.name
+}
+
+func OnAction(format string, args ...interface{}) *Action {
+	return NewAction(format, args...)
+}
+
+func (a *Action) Do(bodies ...Body) []*Transition {
+	body := func (s State) State {
+		for _, b := range bodies {
+			s = b(s)
+		}
+		return s
+	}
+	return []*Transition{NewTransition(a, body)}
+}

test/gofmbt/cover.go

@@ -0,0 +1,196 @@
+package gofmbt
+
+import (
+	"strings"
+)
+
+type CoveredInPath func(Path) []string
+
+type Coverer struct {
+	coverCount  map[string]int
+	coveredPath Path
+	covFuncs    []CoveredInPath
+	historyLen  int
+}
+
+func NewCoverer() *Coverer {
+	return &Coverer{
+		coverCount: map[string]int{},
+	}
+}
+
+func ActionNames(path Path) []string {
+	names := []string{}
+	for _, t := range path {
+		names = append(names, t.action.name)
+	}
+	return names
+}
+
+func ActionFormats(path Path) []string {
+	formats := []string{}
+	for _, t := range path {
+		formats = append(formats, t.action.format)
+	}
+	return formats
+}
+
+func (c *Coverer) CoverActionNameCombinations(combLenMax int) {
+	if c.historyLen < combLenMax {
+		c.historyLen = combLenMax
+	}
+	actionSep := "\x00"
+	c.addCovFunc(func(path Path) []string {
+		actionCombs := []string{}
+		for combLen := 1; combLen <= combLenMax; combLen++ {
+			for first := 0; first <= len(path)-combLen; first++ {
+				actionCombs = append(actionCombs, strings.Join(ActionNames(path[first:first+combLen]), actionSep))
+			}
+		}
+		return actionCombs
+	})
+}
+
+func (c *Coverer) CoverActionFormatCombinations(combLenMax int) {
+	if c.historyLen < combLenMax {
+		c.historyLen = combLenMax
+	}
+	actionSep := "\x00"
+	c.addCovFunc(func(path Path) []string {
+		actionCombs := []string{}
+		for combLen := 1; combLen <= combLenMax; combLen++ {
+			for first := 0; first <= len(path)-combLen; first++ {
+				actionCombs = append(actionCombs, strings.Join(ActionFormats(path[first:first+combLen]), actionSep))
+			}
+		}
+		return actionCombs
+	})
+}
+
+func StateStrings(path Path) []string {
+	if len(path) == 0 {
+		return nil
+	}
+	s := []string{path[0].start.String()}
+	for _, step := range path {
+		s = append(s, step.end.String())
+	}
+	return s
+}
+
+func (c *Coverer) CoverStates() {
+	c.addCovFunc(func(path Path) []string {
+		return StateStrings(path)
+	})
+}
+
+func (c *Coverer) CoverStateCombinations(combLenMax int) {
+	if c.historyLen < combLenMax {
+		c.historyLen = combLenMax
+	}
+	stateSep := "\x00"
+	c.addCovFunc(func(path Path) []string {
+		stateCombs := []string{}
+		for combLen := 1; combLen <= combLenMax; combLen++ {
+			for first := 0; first <= len(path)-combLen; first++ {
+				stateCombs = append(stateCombs, strings.Join(StateStrings(path[first:first+combLen]), stateSep))
+			}
+		}
+		return stateCombs
+	})
+}
+
+func (c *Coverer) addCovFunc(covFunc CoveredInPath) {
+	c.covFuncs = append(c.covFuncs, covFunc)
+}
+
+func (c *Coverer) covFunc(path Path) []string {
+	allCovered := []string{}
+	for _, covFunc := range c.covFuncs {
+		allCovered = append(allCovered, covFunc(path)...)
+	}
+	return allCovered
+}
+
+func (c *Coverer) Coverage() int {
+	return len(c.coverCount)
+}
+
+func (c *Coverer) UpdateCoverage() {
+	c.coverCount = map[string]int{}
+	for _, s := range c.covFunc(c.coveredPath) {
+		c.coverCount[s]++
+	}
+}
+
+func (c *Coverer) MarkCovered(step ...*Step) {
+	c.coveredPath = append(c.coveredPath, step...)
+}
+
+type CoverageIncreaseStats struct {
+	StepsToMax       int
+	MaxIncrease int
+	StepsToFirst       int
+	FirstIncrease int
+}
+
+func (c *Coverer) EstimateGrowth(path Path) *CoverageIncreaseStats {
+	est := &CoverageIncreaseStats{StepsToFirst: -1, StepsToMax: -1}
+	fullCoverCount := map[string]int{}
+	historyLen := c.historyLen
+	if historyLen > len(c.coveredPath) {
+		historyLen = len(c.coveredPath)
+	}
+	pathWithHistory := append(c.coveredPath[len(c.coveredPath)-historyLen:], path...)
+	allNewCovered := c.covFunc(pathWithHistory)
+	for _, s := range allNewCovered {
+		if c.coverCount[s] == 0 {
+			fullCoverCount[s]++
+		}
+	}
+	est.MaxIncrease = len(fullCoverCount)
+	firstCoverCount := map[string]int{}
+	for i := historyLen; i < len(pathWithHistory); i++ {
+		newCovered := c.covFunc(pathWithHistory[:i+1])
+		for _, s := range newCovered {
+			if c.coverCount[s] == 0 {
+				firstCoverCount[s]++
+			}
+		}
+		if est.StepsToFirst == -1 && len(firstCoverCount) > 0 {
+			est.StepsToFirst = i - historyLen
+			est.FirstIncrease = len(firstCoverCount)
+		}
+		if est.StepsToMax == -1 && len(firstCoverCount) == est.MaxIncrease {
+			est.StepsToMax = i - historyLen
+			break
+		}
+	}
+	return est
+}
+
+func (c *Coverer) UnmarkCovered(steps ...*Step) {
+	c.coveredPath = c.coveredPath[:len(c.coveredPath)-len(steps)]
+}
+
+func (c *Coverer) BestPath(m *Model, s State, searchDepth int) (Path, *CoverageIncreaseStats) {
+	var best *CoverageIncreaseStats
+	bestPath := Path{}
+	for _, path := range m.Paths(s, searchDepth) {
+		est := c.EstimateGrowth(path)
+		if best == nil ||
+			est.MaxIncrease > best.MaxIncrease ||
+			est.MaxIncrease == best.MaxIncrease && est.StepsToMax < best.StepsToMax ||
+			est.MaxIncrease == best.MaxIncrease && est.StepsToMax == best.StepsToMax && est.StepsToFirst < best.StepsToFirst ||
+			est.MaxIncrease == best.MaxIncrease && est.StepsToMax == best.StepsToMax && est.StepsToFirst == best.StepsToFirst && est.FirstIncrease > best.FirstIncrease {
+			if est.MaxIncrease > 0 {
+				bestPath = path
+				best = est
+			}
+		}
+	}
+	if len(bestPath) == 0 {
+		return nil, nil
+	}
+	return bestPath, best
+}