From 330a25b1ec08932d91ba0d5a0995e543cbbab625 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Sun, 12 Jan 2025 20:53:31 +0200 Subject: [PATCH] RFC: pure go stateful fuzz test generator Purpose of this patch is to drop dependency to fMBT in e2e fuzz test generation. This patch implements parts that were needed from fMBT in go and includes an example for generating tests: a model and a logic what to cover and how to cover it. Signed-off-by: Antti Kervinen --- .../n4c16/test06-fuzz/generate.go | 137 ++++++++++ .../n4c16/test06-fuzz/generate.sh | 24 +- test/gofmbt/action.go | 51 ++++ test/gofmbt/cover.go | 237 ++++++++++++++++++ test/gofmbt/doc.go | 161 ++++++++++++ test/gofmbt/model.go | 67 +++++ test/gofmbt/model_test.go | 234 +++++++++++++++++ test/gofmbt/state.go | 19 ++ test/gofmbt/step.go | 55 ++++ test/gofmbt/transition.go | 47 ++++ 10 files changed, 1013 insertions(+), 19 deletions(-) create mode 100644 test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go create mode 100644 test/gofmbt/action.go create mode 100644 test/gofmbt/cover.go create mode 100644 test/gofmbt/doc.go create mode 100644 test/gofmbt/model.go create mode 100644 test/gofmbt/model_test.go create mode 100644 test/gofmbt/state.go create mode 100644 test/gofmbt/step.go create mode 100644 test/gofmbt/transition.go diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go new file mode 100644 index 000000000..5c26b24be --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go @@ -0,0 +1,137 @@ +package main + +import ( + "flag" + "fmt" + + m "github.com/containers/nri-plugins/test/gofmbt" +) + +type TestState struct { + cpu int + mem int + rescpu int + podCpuMem map[string][2]int +} + +func (s *TestState) String() string { + return fmt.Sprintf("[cpu:%d mem:%d pods:%v]", s.cpu, s.mem, s.podCpuMem) +} + +func createPod(pod string, contcount, cpu, mem int) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + if s.cpu < cpu*contcount || s.mem < mem*contcount { + // refuse from state change if not enough resources + return nil + } + if _, ok := s.podCpuMem[pod]; ok { + // refuse to create pod if it is already running + return nil + } + newPodCpuMem := make(map[string][2]int) + for k, v := range s.podCpuMem { + newPodCpuMem[k] = v + } + newPodCpuMem[pod] = [2]int{cpu * contcount, mem * contcount} + return &TestState{ + cpu: s.cpu - cpu*contcount, + mem: s.mem - mem*contcount, + podCpuMem: newPodCpuMem, + } + } +} + +func deletePod(pod string) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + cpumem, ok := s.podCpuMem[pod] + if !ok { + // refuse to delete pod if it is not running + return nil + } + newPodCpuMem := make(map[string][2]int) + for k, v := range s.podCpuMem { + if k != pod { + newPodCpuMem[k] = v + } + } + return &TestState{ + cpu: s.cpu + cpumem[0], + mem: s.mem + cpumem[1], + podCpuMem: newPodCpuMem, + } + } +} + +var ( + maxMem int + maxCpu int + maxReservedCpu int + maxTestSteps int +) + +func main() { + flag.IntVar(&maxMem, "mem", 7500, "memory available for test pods") + flag.IntVar(&maxCpu, "cpu", 15000, "non-reserved milli-CPU available for test pods") + flag.IntVar(&maxReservedCpu, "reserved-cpu", 1000, "reserved milli-CPU availble for test pods") + flag.IntVar(&maxTestSteps, "test-steps", 3000, "number of test steps") + flag.Parse() + + podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"} + + model := m.NewModel() + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + return m.When(s.cpu > 0 && s.mem > 0, + m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1, 200, 1500)), + m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2, 1000, 500)), + m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2, 1200, 4500)), + m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3, 2000, 500)), + m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1, 4200, 100)), + m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1, 1200, 50)), + m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2, 1900, 300)), + m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1, 0, 0)), + m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3, 0, 0))) + }) + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + ts := []*m.Transition{} + for _, pod := range podNames { + if _, ok := s.podCpuMem[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s kubectl delete pod %s --now", pod, pod).Do(deletePod(pod))...) + } + } + return ts + }) + + coverer := m.NewCoverer() + coverer.CoverActionCombinations(3) + + var state m.State + + state = &TestState{ + cpu: maxCpu, + mem: maxMem, + rescpu: maxReservedCpu, + } + testStep := 0 + for testStep < maxTestSteps { + path, covStats := coverer.BestPath(model, state, 4) + if len(path) == 0 { + fmt.Printf("# did not find anything to cover\n") + break + } + for i := 0; i < covStats.MaxStep+1; i++ { + testStep++ + step := path[i] + fmt.Printf("\n# step %d, coverage: %d, state: %v\n", testStep, coverer.Coverage(), state) + fmt.Println(step.Action()) + state = step.EndState() + coverer.MarkCovered(step) + coverer.UpdateCoverage() + } + } +} diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh index 6d56afefb..af8c00838 100755 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh @@ -11,8 +11,6 @@ Configuring test generation with environment variables: RESERVED_CPU= Reserved CPU [mCPU] available for test pods in the system. STEPS= Total number of test steps in all parallel tests. - FMBT_IMAGE= Generate the test using fmbt from docker image IMG:TAG. - The default is fmbt-cli:latest. EOF exit 0 } @@ -28,21 +26,12 @@ MEM=${MEM:-7500} CPU=${CPU:-14050} RESERVED_CPU=${RESERVED_CPU:-1000} STEPS=${STEPS:-100} -FMBT_IMAGE=${FMBT_IMAGE:-"fmbt-cli:latest"} mem_per_test=$(( MEM / TESTCOUNT )) cpu_per_test=$(( CPU / TESTCOUNT )) reserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT )) steps_per_test=$(( STEPS / TESTCOUNT )) -# Check fmbt Docker image -docker run "$FMBT_IMAGE" fmbt --version 2>&1 | grep ^Version: || { - echo "error: cannot run fmbt from Docker image '$FMBT_IMAGE'" - echo "You can build the image locally by running:" - echo "( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )" - exit 1 -} - cd "$(dirname "$0")" || { echo "cannot cd to the directory of $0" exit 1 @@ -50,14 +39,11 @@ cd "$(dirname "$0")" || { for testnum in $(seq 1 "$TESTCOUNT"); do testid=$(( testnum - 1)) - sed -e "s/max_mem=.*/max_mem=${mem_per_test}/" \ - -e "s/max_cpu=.*/max_cpu=${cpu_per_test}/" \ - -e "s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/" \ - < fuzz.aal > tmp.fuzz.aal - sed -e "s/fuzz\.aal/tmp.fuzz.aal/" \ - -e "s/pass = steps(.*/pass = steps(${steps_per_test})/" \ - < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf OUTFILE=generated${testid}.sh echo "generating $OUTFILE..." - docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" + go run ./generate.go \ + --mem $mem_per_test \ + --cpu $cpu_per_test \ + --reserved-cpu $reserved_cpu_per_test \ + --test-steps $steps_per_test > "$OUTFILE" done diff --git a/test/gofmbt/action.go b/test/gofmbt/action.go new file mode 100644 index 000000000..4dff253ba --- /dev/null +++ b/test/gofmbt/action.go @@ -0,0 +1,51 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +type Action struct { + name string + format string + args []interface{} +} + +func NewAction(format string, args ...interface{}) *Action { + return &Action{ + format: format, + args: args, + name: fmt.Sprintf(format, args...), + } +} + +func (a *Action) String() string { + return a.name +} + +func OnAction(format string, args ...interface{}) *Action { + return NewAction(format, args...) +} + +func (a *Action) Do(stateChanges ...StateChange) []*Transition { + stateChange := func(s State) State { + for _, sc := range stateChanges { + s = sc(s) + } + return s + } + return []*Transition{NewTransition(a, stateChange)} +} diff --git a/test/gofmbt/cover.go b/test/gofmbt/cover.go new file mode 100644 index 000000000..2e4d622b1 --- /dev/null +++ b/test/gofmbt/cover.go @@ -0,0 +1,237 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "strings" +) + +type CoveredInPath func(Path) []string + +type Coverer struct { + coverCount map[string]int + coveredPath Path + covFuncs []CoveredInPath + historyLen int +} + +func NewCoverer() *Coverer { + return &Coverer{ + coverCount: map[string]int{}, + } +} + +func ActionNames(path Path) []string { + names := []string{} + for _, t := range path { + names = append(names, t.action.name) + } + return names +} + +func ActionFormats(path Path) []string { + formats := []string{} + for _, t := range path { + formats = append(formats, t.action.format) + } + return formats +} + +func (c *Coverer) CoverActions() { + c.addCovFunc(ActionNames) +} + +func (c *Coverer) CoverActionFormats() { + c.addCovFunc(ActionFormats) +} + +func (c *Coverer) CoverActionCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionNames(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +func (c *Coverer) CoverActionFormatCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionFormats(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +func StateStrings(path Path) []string { + if len(path) == 0 { + return nil + } + s := []string{path[0].start.String()} + for _, step := range path { + s = append(s, step.end.String()) + } + return s +} + +func StateActionStrings(path Path) []string { + stateActionSep := "\x00" + stateActions := make([]string, 0, len(path)) + for _, step := range path { + stateActions = append(stateActions, step.StartState().String()+stateActionSep+step.Action().String()) + } + return stateActions +} + +func (c *Coverer) CoverStates() { + c.addCovFunc(StateStrings) +} + +func (c *Coverer) CoverStateActions() { + c.addCovFunc(StateActionStrings) +} + +func (c *Coverer) CoverStateCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + stateSep := "\x00" + c.addCovFunc(func(path Path) []string { + stateCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + stateCombs = append(stateCombs, strings.Join(StateStrings(path[first:first+combLen]), stateSep)) + } + } + return stateCombs + }) +} + +func (c *Coverer) addCovFunc(covFunc CoveredInPath) { + c.covFuncs = append(c.covFuncs, covFunc) +} + +func (c *Coverer) covFunc(path Path) []string { + allCovered := []string{} + for _, covFunc := range c.covFuncs { + allCovered = append(allCovered, covFunc(path)...) + } + return allCovered +} + +func (c *Coverer) Coverage() int { + return len(c.coverCount) +} + +func (c *Coverer) CoveredStrings() []string { + cs := make([]string, 0, len(c.coverCount)) + for s := range c.coverCount { + cs = append(cs, s) + } + return cs +} + +func (c *Coverer) UpdateCoverage() { + c.coverCount = map[string]int{} + for _, s := range c.covFunc(c.coveredPath) { + c.coverCount[s]++ + } +} + +func (c *Coverer) MarkCovered(step ...*Step) { + c.coveredPath = append(c.coveredPath, step...) +} + +type CoverageIncreaseStats struct { + MaxStep int + MaxIncrease int + FirstStep int + FirstIncrease int +} + +func (c *Coverer) EstimateGrowth(path Path) *CoverageIncreaseStats { + est := &CoverageIncreaseStats{FirstStep: -1, MaxStep: -1} + fullCoverCount := map[string]int{} + historyLen := c.historyLen + if historyLen > len(c.coveredPath) { + historyLen = len(c.coveredPath) + } + pathWithHistory := append(c.coveredPath[len(c.coveredPath)-historyLen:], path...) + allNewCovered := c.covFunc(pathWithHistory) + for _, s := range allNewCovered { + if c.coverCount[s] == 0 { + fullCoverCount[s]++ + } + } + est.MaxIncrease = len(fullCoverCount) + firstCoverCount := map[string]int{} + for i := historyLen; i < len(pathWithHistory); i++ { + newCovered := c.covFunc(pathWithHistory[:i+1]) + for _, s := range newCovered { + if c.coverCount[s] == 0 { + firstCoverCount[s]++ + } + } + if est.FirstStep == -1 && len(firstCoverCount) > 0 { + est.FirstStep = i - historyLen + est.FirstIncrease = len(firstCoverCount) + } + if est.MaxStep == -1 && len(firstCoverCount) == est.MaxIncrease { + est.MaxStep = i - historyLen + break + } + } + return est +} + +func (c *Coverer) UnmarkCovered(steps ...*Step) { + c.coveredPath = c.coveredPath[:len(c.coveredPath)-len(steps)] +} + +func (c *Coverer) BestPath(m *Model, s State, maxLen int) (Path, *CoverageIncreaseStats) { + var best *CoverageIncreaseStats + bestPath := Path{} + for _, path := range m.Paths(s, maxLen) { + est := c.EstimateGrowth(path) + if best == nil || + est.MaxIncrease > best.MaxIncrease || + est.MaxIncrease == best.MaxIncrease && est.MaxStep < best.MaxStep || + est.MaxIncrease == best.MaxIncrease && est.MaxStep == best.MaxStep && est.FirstStep < best.FirstStep || + est.MaxIncrease == best.MaxIncrease && est.MaxStep == best.MaxStep && est.FirstStep == best.FirstStep && est.FirstIncrease > best.FirstIncrease { + if est.MaxIncrease > 0 { + bestPath = path + best = est + } + } + } + if len(bestPath) == 0 { + return nil, nil + } + return bestPath, best +} diff --git a/test/gofmbt/doc.go b/test/gofmbt/doc.go new file mode 100644 index 000000000..432867798 --- /dev/null +++ b/test/gofmbt/doc.go @@ -0,0 +1,161 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gofmbt implements a model-based testing library, including +// tools for +// 1. defining test models +// 2. defining what a test should cover +// 3. generating tests sequences with optimal coverage +// +// # Models +// +// A model specifies what and when can be tested. "What" is specified +// by an Action while "when" is expressed using states and +// transitions. +// +// State is implemented by user. It needs to implement String(). For +// example, if testing a music player, a State can be defined as: +// +// type PlayerState struct { +// playing bool // player is either playing or paused +// song int // the number of the song being played +// } +// +// StateChange is a function that takes a start state as an input and +// returns an end state as an output. If a StateChange function +// returns nil, the state change is unspecified at the start +// state. User defines typically many StateChange functions, each of +// which modify one or more attributes of State. For example, a state +// change that starts a paused player, but is unspecified if the +// player is already playing, can be defined as: +// +// func startPlaying (current State) State { +// s := current.(*PlayerState) +// if s.playing { +// return nil +// } +// return &PlayerState{ +// playing: true, +// song: s.song, +// } +// } +// +// Action is a string, possibly specified by separate +// format+arguments, that identifies what exactly should be done when +// the test generator suggests executing a test step with this +// action. For example, an action can be a keyword or a test step name +// possibly with parameters, or an executable line of any script +// language like "press-button 'play'". +// +// Transition is a combination of an Action and a StateChange +// function. For example, +// +// play := NewTransition(NewAction("press-button '%s'", "play"), startPlaying) +// +// Finally, Transitions are added to a Model using transition +// generator functions. They are functions that return a slice of +// Transitions that may be specified in a State. For instance: +// +// model.From(func(current State) []*Transition { +// return []*Transition{play} +// }) +// +// Note that the transition generator function can already do checks +// on the current State attributes and return only transitions that +// are specified at the state. However, as the StateChange function of +// a returned transition may return nil, not all returned transitions +// need to be defined at the state. Rather, transition generator +// functions enable having common preconditions for all transitions +// that the generator may return. Model.From() can be called multiple +// times to add multiple transition generator functions. +// +// Refer to model_test.go to find examples of defining the same model +// for a player in two different ways: first with StateChanges and +// Transitions, and then with convenience functions When/OnAction/Do. +// +// # Test generation +// +// Tests are sequences of Steps. Every Step has an Action, a start +// State, and an end State. Model.StepsFrom(State) returns all +// possible Steps whose start state is State. +// +// Path is a sequence of Steps where the end state of a Step is the +// start state of the next Step. Model.Paths(State, maxLen) returns +// all possible Paths of at most maxLen Steps where the first Step of +// every Path starts from the State. +// +// Coverer helps finding Paths that increase coverage of wanted +// elements. Elements to be covered are specified by Coverer methods: +// - CoverStates(): cover unique State.String()s: +// visit every state. +// - CoverStateActions(): unique StartState().String() + Action().String(): +// test every action in every state. +// - CoverStateCombination(n): unique State_1, ..., State_n combinations: +// test all state-paths of length n. +// - CoverActions(): unique Action.Strings()s: +// test every action. Different parameters counts as different actions. +// - CoverActionCombinations(n): unique Action_1, ..., Action_n combinations: +// test all action-paths of length n. +// - CoverActionFormats(): unique Action formats: +// test every action format, ignoring action parameters. +// +// Calling multiple Cover*() functions allows specifying multiple +// elements whose coverage counts. For example, CoverActions() and +// CoverStates() counts every new Action and every new State, but it +// does not require executing every Action and in every State like +// CoverStateActions() would do. On the other hand, if all these three +// functions are called, then the best test Path is one that gives +// greatest increase the coverage of all the three elements at the +// same time. In practice, this prioritises testing new Actions and +// new States as long as they are found, at the same time when trying +// to cover every Action in every State. +// +// Cover.BestPath(Model, State, maxLen) returns a Path, starting +// from a State in a Model, that results in largest increase in +// whatever elements are covered. The Path is nil if coverage cannot +// be increased by any Path of at most maxLen Steps. +// +// When one or more Steps in a Path have been handled, they are marked +// as covered by calling Coverer.MarkCovered(Step...). Having all +// marked, Coverer.UpdateCoverage() must be called. This can be heavy +// operation, depending on how coverage is measured and how many steps +// have been covered, and therefore it is done separately from marking +// individual Steps covered. Once updated, Coverer.Coverage() returns +// the total number of elements that have been covered by in all +// marked Steps, and Coverer.BestPath() will use new coverage as basis +// when searching for new BestPaths(). +// +// Test generation loop example: +// +// model := myModel() +// state := &MyModelState { ... } // initial state of generated test +// coverer := NewCoverer() +// coverer.CoverActions() +// coverer.CoverStates() +// coverer.CoverStateActions() +// for { +// path, stats := coverer.BestPath(model, state, 6) +// if len(path) == 0 { +// break // could not find a path that increased coverage +// } +// for _, step := range path[:stats.FirstStep+1] { +// fmt.Printf("# coverage: %d, step: %s\n", coverer.Coverage(), step) +// fmt.Printf("%s\n", step.Action()) +// coverer.MarkCovered(step) +// coverer.UpdateCoverage() +// } +// state = path[stats.FirstStep].EndState() +// } + +package gofmbt diff --git a/test/gofmbt/model.go b/test/gofmbt/model.go new file mode 100644 index 000000000..f86579222 --- /dev/null +++ b/test/gofmbt/model.go @@ -0,0 +1,67 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type Model struct { + gen []TransitionGen +} + +func NewModel() *Model { + return &Model{} +} + +func (m *Model) From(transitionGen func(State) []*Transition) { + m.gen = append(m.gen, transitionGen) +} + +func (m *Model) TransitionsFrom(s State) []*Transition { + ts := []*Transition{} + for _, gen := range m.gen { + ts = append(ts, gen(s)...) + } + return ts +} + +func (m *Model) StepsFrom(s State) []*Step { + steps := []*Step{} + for _, t := range m.TransitionsFrom(s) { + if endState := t.stateChange(s); endState != nil { + steps = append(steps, NewStep(s, t.action, endState)) + } + } + return steps +} + +func (m *Model) Paths(s State, maxLen int) []Path { + paths := []Path{} + if maxLen == 0 { + return nil + } + for _, step := range m.StepsFrom(s) { + newPath := NewPath(step) + childPaths := m.Paths(step.EndState(), maxLen-1) + if len(childPaths) == 0 { + paths = append(paths, newPath) + } else { + for _, childPath := range childPaths { + paths = append(paths, append(newPath, childPath...)) + } + } + } + if len(paths) == 0 { + return nil + } + return paths +} diff --git a/test/gofmbt/model_test.go b/test/gofmbt/model_test.go new file mode 100644 index 000000000..3161eeaf4 --- /dev/null +++ b/test/gofmbt/model_test.go @@ -0,0 +1,234 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" + "testing" +) + +type PlayerState struct { + playing bool + song int +} + +func (ps *PlayerState) String() string { + return fmt.Sprintf("{playing:%v,song:%v}", ps.playing, ps.song) +} + +// This is Rosetta's stone on modelling directly with StateChange +// functions and NewTransitions in newPlayerModelWithRawTransitions, +// and writing exactly the same model using When/OnAction/Do. +func newPlayerModelWithRawTransitions() *Model { + model := NewModel() + play := func(current State) State { + s := current.(*PlayerState) + if s.playing { + return nil + } + return &PlayerState{ + playing: true, + song: s.song, + } + } + pause := func(current State) State { + s := current.(*PlayerState) + if !s.playing { + return nil + } + return &PlayerState{ + playing: false, + song: s.song, + } + } + nextsong := func(current State) State { + s := current.(*PlayerState) + if s.song >= 3 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song + 1, + } + } + prevsong := func(current State) State { + s := current.(*PlayerState) + if s.song <= 1 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song - 1, + } + } + + model.From(func(current State) []*Transition { + return []*Transition{ + NewTransition(NewAction("play"), play), + NewTransition(NewAction("pause"), pause), + NewTransition(NewAction("nextsong"), nextsong), + NewTransition(NewAction("prevsong"), prevsong), + } + }) + + return model +} + +func newPlayerModelWithWhenOnAction() *Model { + setState := func(playing bool, song int) StateChange { + return func(_ State) State { + return &PlayerState{playing, song} + } + } + + model := NewModel() + model.From(func(start State) []*Transition { + s := start.(*PlayerState) + return When(true, + When(s.playing, + OnAction("pause").Do(setState(false, s.song))), + When(!s.playing, + OnAction("play").Do(setState(true, s.song))), + When(s.song < 3, + OnAction("nextsong").Do(setState(s.playing, s.song+1))), + When(s.song > 1, + OnAction("prevsong").Do(setState(s.playing, s.song-1))), + ) + }) + return model +} + +var playerModels map[string]*Model = map[string]*Model{ + "raw": newPlayerModelWithRawTransitions(), + "when": newPlayerModelWithWhenOnAction(), +} + +func TestCoverPlayerActions(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverActionCombinations(1) + path, stats := coverer.BestPath(model, state, 6) + if len(path) != 6 { + t.Fatalf("model %q: expected len(path)==6, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 4 { + t.Fatalf("model %q: expected reaching coverage 4, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 3 { + t.Fatalf("model %q: expected all actions covered at step 4, got %d", modelName, stats.MaxStep) + } + + covered := map[string]int{} + for _, step := range path[:stats.MaxStep+1] { + covered[step.Action().String()] += 1 + } + if len(covered) != 4 { + t.Fatalf("model %q: expected 4 different actions, got %d", modelName, len(covered)) + } + for _, expectedStep := range []string{"play", "pause", "nextsong", "prevsong"} { + if _, ok := covered[expectedStep]; !ok { + t.Fatalf("model %q: expected %q in covered, got: %v", modelName, expectedStep, covered) + } + } + } +} + +func TestCoverPlayerStates(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStates() + path, stats := coverer.BestPath(model, state, 8) + if len(path) != 8 { + t.Fatalf("model %q: expected len(path)==10, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 6 { + t.Fatalf("model %q: expected reaching 6 different states, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 4 { + t.Fatalf("model %q: expected all stats visited at step 4, got %d", modelName, stats.MaxStep) + } + if len(coverer.CoveredStrings()) != 0 { + t.Fatalf("model %q: expected nothing to be covered yet, got %v", modelName, coverer.CoveredStrings()) + } + coverer.MarkCovered(path[:stats.MaxStep+1]...) + coverer.UpdateCoverage() + if len(coverer.CoveredStrings()) != 6 { + t.Fatalf("model %q: expected 6 states covered, got %d %v", modelName, len(coverer.CoveredStrings()), coverer.CoveredStrings()) + } + } +} + +func TestStepsFrom(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + altSteps := model.StepsFrom(state) + if len(altSteps) != 2 { + t.Fatalf("model %q: expected 2 StepsFrom %s, got %d", modelName, state, len(altSteps)) + } + for _, step := range altSteps { + if step.Action().String() == "play" { + continue + } + if step.Action().String() == "nextsong" { + continue + } + t.Fatalf("model %q: unexpected action %q in state %s. Step: %s", modelName, step.Action(), state, step) + } + } +} + +func TestSeachPathsTestSteps(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStateActions() + t.Log(modelName) + i := 0 + for { + path, stats := coverer.BestPath(model, state, 6) + if len(path) == 0 { + break + } + // Verify prev step.EndState() == next step.StartState() + var prev *Step + for _, step := range path { + if prev != nil && step.StartState().String() != prev.EndState().String() { + t.Fatalf("model %q: prev end state and next start state differ: %s != %s", modelName, prev, step) + } + prev = step + } + + // Execute path up to the first step where + // coverage increases and search for the + // BestPath continuing from there. + for _, step := range path[:stats.FirstStep+1] { + i++ + t.Log("step:", step) + coverer.MarkCovered(step) + coverer.UpdateCoverage() + t.Log("coverage:", coverer.Coverage(), "strings:", coverer.CoveredStrings()) + } + state = path[stats.FirstStep].EndState().(*PlayerState) + } + if coverer.Coverage() != 14 { + t.Fatalf("model %q: expected 14 {state}--action--> combinations to be covered, got %d", modelName, coverer.Coverage()) + } + if i != 14 { + t.Fatalf("model %q: expected reach max coverage with 14 steps, needed: %d", modelName, i) + } + } +} diff --git a/test/gofmbt/state.go b/test/gofmbt/state.go new file mode 100644 index 000000000..e676ceef4 --- /dev/null +++ b/test/gofmbt/state.go @@ -0,0 +1,19 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type State interface { + String() string +} diff --git a/test/gofmbt/step.go b/test/gofmbt/step.go new file mode 100644 index 000000000..6cb127e33 --- /dev/null +++ b/test/gofmbt/step.go @@ -0,0 +1,55 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +type Step struct { + start State + action *Action + end State +} + +type Path []*Step + +func NewStep(start State, action *Action, end State) *Step { + return &Step{start, action, end} +} + +func (step *Step) String() string { + return fmt.Sprintf("[%s--%s->%s]", step.start, step.action, step.end) +} + +func (step *Step) StartState() State { + return step.start +} + +func (step *Step) Action() *Action { + return step.action +} + +func (step *Step) EndState() State { + return step.end +} + +func NewPath(steps ...*Step) Path { + path := Path{} + for _, step := range steps { + path = append(path, step) + } + return path +} diff --git a/test/gofmbt/transition.go b/test/gofmbt/transition.go new file mode 100644 index 000000000..87b8370c6 --- /dev/null +++ b/test/gofmbt/transition.go @@ -0,0 +1,47 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +type StateChange func(State) State + +type TransitionGen func(State) []*Transition + +type Transition struct { + action *Action + stateChange StateChange +} + +func NewTransition(a *Action, sc StateChange) *Transition { + return &Transition{a, sc} +} + +func (t *Transition) Action() *Action { + return t.action +} + +func (t *Transition) StateChange(s State) State { + return t.stateChange(s) +} + +func When(enabled bool, tss ...[]*Transition) []*Transition { + if !enabled { + return nil + } + ts := []*Transition{} + for _, origTs := range tss { + ts = append(ts, origTs...) + } + return ts +}