From 400e12b8f53845e68146a9ba3f469c6d3a3b0073 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Sun, 12 Jan 2025 20:53:31 +0200 Subject: [PATCH] e2e: add pure go stateful fuzz test generator Drop dependency to fMBT in e2e fuzz test generation. This patch adds go implementations of necessary parts of fMBT and replaces AAL/Python models with go models for topology-aware fuzz test. Signed-off-by: Antti Kervinen --- .../n4c16/test06-fuzz/generate.go | 185 ++++++++++++++ .../n4c16/test06-fuzz/generate.sh | 28 +-- test/gofmbt/action.go | 51 ++++ test/gofmbt/cover.go | 237 ++++++++++++++++++ test/gofmbt/doc.go | 161 ++++++++++++ test/gofmbt/model.go | 67 +++++ test/gofmbt/model_test.go | 234 +++++++++++++++++ test/gofmbt/state.go | 19 ++ test/gofmbt/step.go | 55 ++++ test/gofmbt/transition.go | 47 ++++ 10 files changed, 1065 insertions(+), 19 deletions(-) create mode 100644 test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go create mode 100644 test/gofmbt/action.go create mode 100644 test/gofmbt/cover.go create mode 100644 test/gofmbt/doc.go create mode 100644 test/gofmbt/model.go create mode 100644 test/gofmbt/model_test.go create mode 100644 test/gofmbt/state.go create mode 100644 test/gofmbt/step.go create mode 100644 test/gofmbt/transition.go diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go new file mode 100644 index 000000000..0e9bbeaa7 --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go @@ -0,0 +1,185 @@ +package main + +import ( + "flag" + "fmt" + "sort" + "strings" + + m "github.com/containers/nri-plugins/test/gofmbt" +) + +type PodResources struct { + cpu int // total CPU allocated for all containers in the pod + rcpu int // total reserved CPU allocated for all containers in the pod + mem int // total memory allocated for all containers in the pod +} + +type TestState struct { + cpu int // free CPU on node for non-reserved pods + rcpu int // free CPU on node for reserved pods + mem int // free memory on node + podRes map[string]*PodResources // map running pod name to resources allocated to it +} + +func (s *TestState) String() string { + pr := []string{} + pods := make([]string, 0, len(s.podRes)) + for pod := range s.podRes { + pods = append(pods, pod) + } + sort.Strings(pods) + for _, pod := range pods { + res := s.podRes[pod] + switch { + case res.rcpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dM", pod, res.cpu, res.mem)) + case res.cpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmRCPU/%dM", pod, res.rcpu, res.mem)) + default: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dmRCPU/%dM", pod, res.cpu, res.rcpu, res.mem)) + } + } + return fmt.Sprintf("[free:%dmCPU/%dmRCPU/%dM pods:[%s]]", s.cpu, s.rcpu, s.mem, strings.Join(pr, " ")) +} + +func createPod(pod string, cpu, rcpu, mem int) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + if s.cpu < cpu || s.rcpu < rcpu || s.mem < mem { + // refuse from state change if not enough resources + return nil + } + if _, ok := s.podRes[pod]; ok { + // refuse to create pod if it is already running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + newPodRes[k] = v + } + newPodRes[pod] = &PodResources{cpu, rcpu, mem} + return &TestState{ + cpu: s.cpu - cpu, + rcpu: s.rcpu - rcpu, + mem: s.mem - mem, + podRes: newPodRes, + } + } +} + +func deletePod(pod string) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + res, ok := s.podRes[pod] + if !ok { + // refuse to delete pod if it is not running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + if k != pod { + newPodRes[k] = v + } + } + return &TestState{ + cpu: s.cpu + res.cpu, + rcpu: s.rcpu + res.rcpu, + mem: s.mem + res.mem, + podRes: newPodRes, + } + } +} + +var ( + maxMem int + maxCpu int + maxReservedCpu int + maxTestSteps int +) + +func newModel() *m.Model { + podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"} + rPodNames := []string{"rbe0", "rgu0", "rbu0"} + + model := m.NewModel() + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + return m.When(true, + m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1*0, 0, 1*0)), + m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3*0, 0, 3*0)), + m.OnAction("NAME=rbe0 CONTCOUNT=2 CPU=0 MEM=0 namespace=kube-system create besteffort").Do(createPod("rbe0", 0, 2*0, 2*0)), + m.When(s.mem > 0, + m.When(s.cpu >= 200, + m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1*200, 0, 1*1500)), + m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2*1000, 0, 2*500)), + m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2*1200, 0, 2*4500)), + m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3*2000, 0, 3*500)), + m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1*4200, 0, 1*100)), + m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1*1200, 0, 1*50)), + m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2*1900, 0, 2*300)), + ), + m.When(s.rcpu > 99, + m.OnAction("NAME=rgu0 CONTCOUNT=2 CPU=100m MEM=1000M namespace=kube-system create guaranteed").Do(createPod("rgu0", 0, 2*100, 2*1000)), + m.OnAction("NAME=rbu0 CONTCOUNT=1 CPU=100m MEM=100M CPUREQ=99m MEMREQ=99M CPULIM=100m MEMLIM=100M namespace=kube-system create burstable").Do(createPod("rbu0", 0, 1*100, 1*100)), + ), + ), + ) + }) + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + ts := []*m.Transition{} + for _, pod := range podNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + for _, pod := range rPodNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod --namespace kube-system %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + return ts + }) + + return model +} + +func main() { + flag.IntVar(&maxMem, "mem", 7500, "memory available for test pods") + flag.IntVar(&maxCpu, "cpu", 15000, "non-reserved milli-CPU available for test pods") + flag.IntVar(&maxReservedCpu, "reserved-cpu", 1000, "reserved milli-CPU availble for test pods") + flag.IntVar(&maxTestSteps, "test-steps", 3000, "number of test steps") + flag.Parse() + + model := newModel() + coverer := m.NewCoverer() + coverer.CoverActionCombinations(3) + + var state m.State + + state = &TestState{ + cpu: maxCpu, + rcpu: maxReservedCpu, + mem: maxMem, + } + testStep := 0 + for testStep < maxTestSteps { + path, covStats := coverer.BestPath(model, state, 4) + if len(path) == 0 { + fmt.Printf("# did not find anything to cover\n") + break + } + for i := 0; i < covStats.MaxStep+1; i++ { + testStep++ + step := path[i] + fmt.Printf("echo ==== step %d, coverage: %d, state: %v\n", testStep, coverer.Coverage(), state) + fmt.Println(step.Action()) + state = step.EndState() + coverer.MarkCovered(step) + coverer.UpdateCoverage() + } + } +} diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh index 6d56afefb..0b7542473 100755 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh @@ -11,8 +11,6 @@ Configuring test generation with environment variables: RESERVED_CPU= Reserved CPU [mCPU] available for test pods in the system. STEPS= Total number of test steps in all parallel tests. - FMBT_IMAGE= Generate the test using fmbt from docker image IMG:TAG. - The default is fmbt-cli:latest. EOF exit 0 } @@ -28,36 +26,28 @@ MEM=${MEM:-7500} CPU=${CPU:-14050} RESERVED_CPU=${RESERVED_CPU:-1000} STEPS=${STEPS:-100} -FMBT_IMAGE=${FMBT_IMAGE:-"fmbt-cli:latest"} mem_per_test=$(( MEM / TESTCOUNT )) cpu_per_test=$(( CPU / TESTCOUNT )) reserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT )) steps_per_test=$(( STEPS / TESTCOUNT )) -# Check fmbt Docker image -docker run "$FMBT_IMAGE" fmbt --version 2>&1 | grep ^Version: || { - echo "error: cannot run fmbt from Docker image '$FMBT_IMAGE'" - echo "You can build the image locally by running:" - echo "( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )" - exit 1 -} - cd "$(dirname "$0")" || { echo "cannot cd to the directory of $0" exit 1 } + for testnum in $(seq 1 "$TESTCOUNT"); do testid=$(( testnum - 1)) - sed -e "s/max_mem=.*/max_mem=${mem_per_test}/" \ - -e "s/max_cpu=.*/max_cpu=${cpu_per_test}/" \ - -e "s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/" \ - < fuzz.aal > tmp.fuzz.aal - sed -e "s/fuzz\.aal/tmp.fuzz.aal/" \ - -e "s/pass = steps(.*/pass = steps(${steps_per_test})/" \ - < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf OUTFILE=generated${testid}.sh + prefix_pods_with_testid="s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" echo "generating $OUTFILE..." - docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" + go run ./generate.go \ + --mem $mem_per_test \ + --cpu $cpu_per_test \ + --reserved-cpu $reserved_cpu_per_test \ + --test-steps $steps_per_test \ + | sed -e "$prefix_pods_with_testid" \ + > "$OUTFILE" done diff --git a/test/gofmbt/action.go b/test/gofmbt/action.go new file mode 100644 index 000000000..4dff253ba --- /dev/null +++ b/test/gofmbt/action.go @@ -0,0 +1,51 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +type Action struct { + name string + format string + args []interface{} +} + +func NewAction(format string, args ...interface{}) *Action { + return &Action{ + format: format, + args: args, + name: fmt.Sprintf(format, args...), + } +} + +func (a *Action) String() string { + return a.name +} + +func OnAction(format string, args ...interface{}) *Action { + return NewAction(format, args...) +} + +func (a *Action) Do(stateChanges ...StateChange) []*Transition { + stateChange := func(s State) State { + for _, sc := range stateChanges { + s = sc(s) + } + return s + } + return []*Transition{NewTransition(a, stateChange)} +} diff --git a/test/gofmbt/cover.go b/test/gofmbt/cover.go new file mode 100644 index 000000000..2e4d622b1 --- /dev/null +++ b/test/gofmbt/cover.go @@ -0,0 +1,237 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "strings" +) + +type CoveredInPath func(Path) []string + +type Coverer struct { + coverCount map[string]int + coveredPath Path + covFuncs []CoveredInPath + historyLen int +} + +func NewCoverer() *Coverer { + return &Coverer{ + coverCount: map[string]int{}, + } +} + +func ActionNames(path Path) []string { + names := []string{} + for _, t := range path { + names = append(names, t.action.name) + } + return names +} + +func ActionFormats(path Path) []string { + formats := []string{} + for _, t := range path { + formats = append(formats, t.action.format) + } + return formats +} + +func (c *Coverer) CoverActions() { + c.addCovFunc(ActionNames) +} + +func (c *Coverer) CoverActionFormats() { + c.addCovFunc(ActionFormats) +} + +func (c *Coverer) CoverActionCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionNames(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +func (c *Coverer) CoverActionFormatCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionFormats(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +func StateStrings(path Path) []string { + if len(path) == 0 { + return nil + } + s := []string{path[0].start.String()} + for _, step := range path { + s = append(s, step.end.String()) + } + return s +} + +func StateActionStrings(path Path) []string { + stateActionSep := "\x00" + stateActions := make([]string, 0, len(path)) + for _, step := range path { + stateActions = append(stateActions, step.StartState().String()+stateActionSep+step.Action().String()) + } + return stateActions +} + +func (c *Coverer) CoverStates() { + c.addCovFunc(StateStrings) +} + +func (c *Coverer) CoverStateActions() { + c.addCovFunc(StateActionStrings) +} + +func (c *Coverer) CoverStateCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + stateSep := "\x00" + c.addCovFunc(func(path Path) []string { + stateCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + stateCombs = append(stateCombs, strings.Join(StateStrings(path[first:first+combLen]), stateSep)) + } + } + return stateCombs + }) +} + +func (c *Coverer) addCovFunc(covFunc CoveredInPath) { + c.covFuncs = append(c.covFuncs, covFunc) +} + +func (c *Coverer) covFunc(path Path) []string { + allCovered := []string{} + for _, covFunc := range c.covFuncs { + allCovered = append(allCovered, covFunc(path)...) + } + return allCovered +} + +func (c *Coverer) Coverage() int { + return len(c.coverCount) +} + +func (c *Coverer) CoveredStrings() []string { + cs := make([]string, 0, len(c.coverCount)) + for s := range c.coverCount { + cs = append(cs, s) + } + return cs +} + +func (c *Coverer) UpdateCoverage() { + c.coverCount = map[string]int{} + for _, s := range c.covFunc(c.coveredPath) { + c.coverCount[s]++ + } +} + +func (c *Coverer) MarkCovered(step ...*Step) { + c.coveredPath = append(c.coveredPath, step...) +} + +type CoverageIncreaseStats struct { + MaxStep int + MaxIncrease int + FirstStep int + FirstIncrease int +} + +func (c *Coverer) EstimateGrowth(path Path) *CoverageIncreaseStats { + est := &CoverageIncreaseStats{FirstStep: -1, MaxStep: -1} + fullCoverCount := map[string]int{} + historyLen := c.historyLen + if historyLen > len(c.coveredPath) { + historyLen = len(c.coveredPath) + } + pathWithHistory := append(c.coveredPath[len(c.coveredPath)-historyLen:], path...) + allNewCovered := c.covFunc(pathWithHistory) + for _, s := range allNewCovered { + if c.coverCount[s] == 0 { + fullCoverCount[s]++ + } + } + est.MaxIncrease = len(fullCoverCount) + firstCoverCount := map[string]int{} + for i := historyLen; i < len(pathWithHistory); i++ { + newCovered := c.covFunc(pathWithHistory[:i+1]) + for _, s := range newCovered { + if c.coverCount[s] == 0 { + firstCoverCount[s]++ + } + } + if est.FirstStep == -1 && len(firstCoverCount) > 0 { + est.FirstStep = i - historyLen + est.FirstIncrease = len(firstCoverCount) + } + if est.MaxStep == -1 && len(firstCoverCount) == est.MaxIncrease { + est.MaxStep = i - historyLen + break + } + } + return est +} + +func (c *Coverer) UnmarkCovered(steps ...*Step) { + c.coveredPath = c.coveredPath[:len(c.coveredPath)-len(steps)] +} + +func (c *Coverer) BestPath(m *Model, s State, maxLen int) (Path, *CoverageIncreaseStats) { + var best *CoverageIncreaseStats + bestPath := Path{} + for _, path := range m.Paths(s, maxLen) { + est := c.EstimateGrowth(path) + if best == nil || + est.MaxIncrease > best.MaxIncrease || + est.MaxIncrease == best.MaxIncrease && est.MaxStep < best.MaxStep || + est.MaxIncrease == best.MaxIncrease && est.MaxStep == best.MaxStep && est.FirstStep < best.FirstStep || + est.MaxIncrease == best.MaxIncrease && est.MaxStep == best.MaxStep && est.FirstStep == best.FirstStep && est.FirstIncrease > best.FirstIncrease { + if est.MaxIncrease > 0 { + bestPath = path + best = est + } + } + } + if len(bestPath) == 0 { + return nil, nil + } + return bestPath, best +} diff --git a/test/gofmbt/doc.go b/test/gofmbt/doc.go new file mode 100644 index 000000000..432867798 --- /dev/null +++ b/test/gofmbt/doc.go @@ -0,0 +1,161 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gofmbt implements a model-based testing library, including +// tools for +// 1. defining test models +// 2. defining what a test should cover +// 3. generating tests sequences with optimal coverage +// +// # Models +// +// A model specifies what and when can be tested. "What" is specified +// by an Action while "when" is expressed using states and +// transitions. +// +// State is implemented by user. It needs to implement String(). For +// example, if testing a music player, a State can be defined as: +// +// type PlayerState struct { +// playing bool // player is either playing or paused +// song int // the number of the song being played +// } +// +// StateChange is a function that takes a start state as an input and +// returns an end state as an output. If a StateChange function +// returns nil, the state change is unspecified at the start +// state. User defines typically many StateChange functions, each of +// which modify one or more attributes of State. For example, a state +// change that starts a paused player, but is unspecified if the +// player is already playing, can be defined as: +// +// func startPlaying (current State) State { +// s := current.(*PlayerState) +// if s.playing { +// return nil +// } +// return &PlayerState{ +// playing: true, +// song: s.song, +// } +// } +// +// Action is a string, possibly specified by separate +// format+arguments, that identifies what exactly should be done when +// the test generator suggests executing a test step with this +// action. For example, an action can be a keyword or a test step name +// possibly with parameters, or an executable line of any script +// language like "press-button 'play'". +// +// Transition is a combination of an Action and a StateChange +// function. For example, +// +// play := NewTransition(NewAction("press-button '%s'", "play"), startPlaying) +// +// Finally, Transitions are added to a Model using transition +// generator functions. They are functions that return a slice of +// Transitions that may be specified in a State. For instance: +// +// model.From(func(current State) []*Transition { +// return []*Transition{play} +// }) +// +// Note that the transition generator function can already do checks +// on the current State attributes and return only transitions that +// are specified at the state. However, as the StateChange function of +// a returned transition may return nil, not all returned transitions +// need to be defined at the state. Rather, transition generator +// functions enable having common preconditions for all transitions +// that the generator may return. Model.From() can be called multiple +// times to add multiple transition generator functions. +// +// Refer to model_test.go to find examples of defining the same model +// for a player in two different ways: first with StateChanges and +// Transitions, and then with convenience functions When/OnAction/Do. +// +// # Test generation +// +// Tests are sequences of Steps. Every Step has an Action, a start +// State, and an end State. Model.StepsFrom(State) returns all +// possible Steps whose start state is State. +// +// Path is a sequence of Steps where the end state of a Step is the +// start state of the next Step. Model.Paths(State, maxLen) returns +// all possible Paths of at most maxLen Steps where the first Step of +// every Path starts from the State. +// +// Coverer helps finding Paths that increase coverage of wanted +// elements. Elements to be covered are specified by Coverer methods: +// - CoverStates(): cover unique State.String()s: +// visit every state. +// - CoverStateActions(): unique StartState().String() + Action().String(): +// test every action in every state. +// - CoverStateCombination(n): unique State_1, ..., State_n combinations: +// test all state-paths of length n. +// - CoverActions(): unique Action.Strings()s: +// test every action. Different parameters counts as different actions. +// - CoverActionCombinations(n): unique Action_1, ..., Action_n combinations: +// test all action-paths of length n. +// - CoverActionFormats(): unique Action formats: +// test every action format, ignoring action parameters. +// +// Calling multiple Cover*() functions allows specifying multiple +// elements whose coverage counts. For example, CoverActions() and +// CoverStates() counts every new Action and every new State, but it +// does not require executing every Action and in every State like +// CoverStateActions() would do. On the other hand, if all these three +// functions are called, then the best test Path is one that gives +// greatest increase the coverage of all the three elements at the +// same time. In practice, this prioritises testing new Actions and +// new States as long as they are found, at the same time when trying +// to cover every Action in every State. +// +// Cover.BestPath(Model, State, maxLen) returns a Path, starting +// from a State in a Model, that results in largest increase in +// whatever elements are covered. The Path is nil if coverage cannot +// be increased by any Path of at most maxLen Steps. +// +// When one or more Steps in a Path have been handled, they are marked +// as covered by calling Coverer.MarkCovered(Step...). Having all +// marked, Coverer.UpdateCoverage() must be called. This can be heavy +// operation, depending on how coverage is measured and how many steps +// have been covered, and therefore it is done separately from marking +// individual Steps covered. Once updated, Coverer.Coverage() returns +// the total number of elements that have been covered by in all +// marked Steps, and Coverer.BestPath() will use new coverage as basis +// when searching for new BestPaths(). +// +// Test generation loop example: +// +// model := myModel() +// state := &MyModelState { ... } // initial state of generated test +// coverer := NewCoverer() +// coverer.CoverActions() +// coverer.CoverStates() +// coverer.CoverStateActions() +// for { +// path, stats := coverer.BestPath(model, state, 6) +// if len(path) == 0 { +// break // could not find a path that increased coverage +// } +// for _, step := range path[:stats.FirstStep+1] { +// fmt.Printf("# coverage: %d, step: %s\n", coverer.Coverage(), step) +// fmt.Printf("%s\n", step.Action()) +// coverer.MarkCovered(step) +// coverer.UpdateCoverage() +// } +// state = path[stats.FirstStep].EndState() +// } + +package gofmbt diff --git a/test/gofmbt/model.go b/test/gofmbt/model.go new file mode 100644 index 000000000..f86579222 --- /dev/null +++ b/test/gofmbt/model.go @@ -0,0 +1,67 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type Model struct { + gen []TransitionGen +} + +func NewModel() *Model { + return &Model{} +} + +func (m *Model) From(transitionGen func(State) []*Transition) { + m.gen = append(m.gen, transitionGen) +} + +func (m *Model) TransitionsFrom(s State) []*Transition { + ts := []*Transition{} + for _, gen := range m.gen { + ts = append(ts, gen(s)...) + } + return ts +} + +func (m *Model) StepsFrom(s State) []*Step { + steps := []*Step{} + for _, t := range m.TransitionsFrom(s) { + if endState := t.stateChange(s); endState != nil { + steps = append(steps, NewStep(s, t.action, endState)) + } + } + return steps +} + +func (m *Model) Paths(s State, maxLen int) []Path { + paths := []Path{} + if maxLen == 0 { + return nil + } + for _, step := range m.StepsFrom(s) { + newPath := NewPath(step) + childPaths := m.Paths(step.EndState(), maxLen-1) + if len(childPaths) == 0 { + paths = append(paths, newPath) + } else { + for _, childPath := range childPaths { + paths = append(paths, append(newPath, childPath...)) + } + } + } + if len(paths) == 0 { + return nil + } + return paths +} diff --git a/test/gofmbt/model_test.go b/test/gofmbt/model_test.go new file mode 100644 index 000000000..3161eeaf4 --- /dev/null +++ b/test/gofmbt/model_test.go @@ -0,0 +1,234 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" + "testing" +) + +type PlayerState struct { + playing bool + song int +} + +func (ps *PlayerState) String() string { + return fmt.Sprintf("{playing:%v,song:%v}", ps.playing, ps.song) +} + +// This is Rosetta's stone on modelling directly with StateChange +// functions and NewTransitions in newPlayerModelWithRawTransitions, +// and writing exactly the same model using When/OnAction/Do. +func newPlayerModelWithRawTransitions() *Model { + model := NewModel() + play := func(current State) State { + s := current.(*PlayerState) + if s.playing { + return nil + } + return &PlayerState{ + playing: true, + song: s.song, + } + } + pause := func(current State) State { + s := current.(*PlayerState) + if !s.playing { + return nil + } + return &PlayerState{ + playing: false, + song: s.song, + } + } + nextsong := func(current State) State { + s := current.(*PlayerState) + if s.song >= 3 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song + 1, + } + } + prevsong := func(current State) State { + s := current.(*PlayerState) + if s.song <= 1 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song - 1, + } + } + + model.From(func(current State) []*Transition { + return []*Transition{ + NewTransition(NewAction("play"), play), + NewTransition(NewAction("pause"), pause), + NewTransition(NewAction("nextsong"), nextsong), + NewTransition(NewAction("prevsong"), prevsong), + } + }) + + return model +} + +func newPlayerModelWithWhenOnAction() *Model { + setState := func(playing bool, song int) StateChange { + return func(_ State) State { + return &PlayerState{playing, song} + } + } + + model := NewModel() + model.From(func(start State) []*Transition { + s := start.(*PlayerState) + return When(true, + When(s.playing, + OnAction("pause").Do(setState(false, s.song))), + When(!s.playing, + OnAction("play").Do(setState(true, s.song))), + When(s.song < 3, + OnAction("nextsong").Do(setState(s.playing, s.song+1))), + When(s.song > 1, + OnAction("prevsong").Do(setState(s.playing, s.song-1))), + ) + }) + return model +} + +var playerModels map[string]*Model = map[string]*Model{ + "raw": newPlayerModelWithRawTransitions(), + "when": newPlayerModelWithWhenOnAction(), +} + +func TestCoverPlayerActions(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverActionCombinations(1) + path, stats := coverer.BestPath(model, state, 6) + if len(path) != 6 { + t.Fatalf("model %q: expected len(path)==6, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 4 { + t.Fatalf("model %q: expected reaching coverage 4, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 3 { + t.Fatalf("model %q: expected all actions covered at step 4, got %d", modelName, stats.MaxStep) + } + + covered := map[string]int{} + for _, step := range path[:stats.MaxStep+1] { + covered[step.Action().String()] += 1 + } + if len(covered) != 4 { + t.Fatalf("model %q: expected 4 different actions, got %d", modelName, len(covered)) + } + for _, expectedStep := range []string{"play", "pause", "nextsong", "prevsong"} { + if _, ok := covered[expectedStep]; !ok { + t.Fatalf("model %q: expected %q in covered, got: %v", modelName, expectedStep, covered) + } + } + } +} + +func TestCoverPlayerStates(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStates() + path, stats := coverer.BestPath(model, state, 8) + if len(path) != 8 { + t.Fatalf("model %q: expected len(path)==10, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 6 { + t.Fatalf("model %q: expected reaching 6 different states, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 4 { + t.Fatalf("model %q: expected all stats visited at step 4, got %d", modelName, stats.MaxStep) + } + if len(coverer.CoveredStrings()) != 0 { + t.Fatalf("model %q: expected nothing to be covered yet, got %v", modelName, coverer.CoveredStrings()) + } + coverer.MarkCovered(path[:stats.MaxStep+1]...) + coverer.UpdateCoverage() + if len(coverer.CoveredStrings()) != 6 { + t.Fatalf("model %q: expected 6 states covered, got %d %v", modelName, len(coverer.CoveredStrings()), coverer.CoveredStrings()) + } + } +} + +func TestStepsFrom(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + altSteps := model.StepsFrom(state) + if len(altSteps) != 2 { + t.Fatalf("model %q: expected 2 StepsFrom %s, got %d", modelName, state, len(altSteps)) + } + for _, step := range altSteps { + if step.Action().String() == "play" { + continue + } + if step.Action().String() == "nextsong" { + continue + } + t.Fatalf("model %q: unexpected action %q in state %s. Step: %s", modelName, step.Action(), state, step) + } + } +} + +func TestSeachPathsTestSteps(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStateActions() + t.Log(modelName) + i := 0 + for { + path, stats := coverer.BestPath(model, state, 6) + if len(path) == 0 { + break + } + // Verify prev step.EndState() == next step.StartState() + var prev *Step + for _, step := range path { + if prev != nil && step.StartState().String() != prev.EndState().String() { + t.Fatalf("model %q: prev end state and next start state differ: %s != %s", modelName, prev, step) + } + prev = step + } + + // Execute path up to the first step where + // coverage increases and search for the + // BestPath continuing from there. + for _, step := range path[:stats.FirstStep+1] { + i++ + t.Log("step:", step) + coverer.MarkCovered(step) + coverer.UpdateCoverage() + t.Log("coverage:", coverer.Coverage(), "strings:", coverer.CoveredStrings()) + } + state = path[stats.FirstStep].EndState().(*PlayerState) + } + if coverer.Coverage() != 14 { + t.Fatalf("model %q: expected 14 {state}--action--> combinations to be covered, got %d", modelName, coverer.Coverage()) + } + if i != 14 { + t.Fatalf("model %q: expected reach max coverage with 14 steps, needed: %d", modelName, i) + } + } +} diff --git a/test/gofmbt/state.go b/test/gofmbt/state.go new file mode 100644 index 000000000..e676ceef4 --- /dev/null +++ b/test/gofmbt/state.go @@ -0,0 +1,19 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type State interface { + String() string +} diff --git a/test/gofmbt/step.go b/test/gofmbt/step.go new file mode 100644 index 000000000..6cb127e33 --- /dev/null +++ b/test/gofmbt/step.go @@ -0,0 +1,55 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +type Step struct { + start State + action *Action + end State +} + +type Path []*Step + +func NewStep(start State, action *Action, end State) *Step { + return &Step{start, action, end} +} + +func (step *Step) String() string { + return fmt.Sprintf("[%s--%s->%s]", step.start, step.action, step.end) +} + +func (step *Step) StartState() State { + return step.start +} + +func (step *Step) Action() *Action { + return step.action +} + +func (step *Step) EndState() State { + return step.end +} + +func NewPath(steps ...*Step) Path { + path := Path{} + for _, step := range steps { + path = append(path, step) + } + return path +} diff --git a/test/gofmbt/transition.go b/test/gofmbt/transition.go new file mode 100644 index 000000000..87b8370c6 --- /dev/null +++ b/test/gofmbt/transition.go @@ -0,0 +1,47 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type StateChange func(State) State + +type TransitionGen func(State) []*Transition + +type Transition struct { + action *Action + stateChange StateChange +} + +func NewTransition(a *Action, sc StateChange) *Transition { + return &Transition{a, sc} +} + +func (t *Transition) Action() *Action { + return t.action +} + +func (t *Transition) StateChange(s State) State { + return t.stateChange(s) +} + +func When(enabled bool, tss ...[]*Transition) []*Transition { + if !enabled { + return nil + } + ts := []*Transition{} + for _, origTs := range tss { + ts = append(ts, origTs...) + } + return ts +}