diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go new file mode 100644 index 000000000..e5d9d362e --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go @@ -0,0 +1,200 @@ +package main + +import ( + "flag" + "fmt" + "sort" + "strings" + + m "github.com/containers/nri-plugins/test/gofmbt" +) + +type PodResources struct { + cpu int // total CPU allocated for all containers in the pod + rcpu int // total reserved CPU allocated for all containers in the pod + mem int // total memory allocated for all containers in the pod +} + +type TestState struct { + cpu int // free CPU on node for non-reserved pods + rcpu int // free CPU on node for reserved pods + mem int // free memory on node + podRes map[string]*PodResources // map running pod name to resources allocated to it +} + +func (s *TestState) String() string { + pr := []string{} + pods := make([]string, 0, len(s.podRes)) + for pod := range s.podRes { + pods = append(pods, pod) + } + sort.Strings(pods) + for _, pod := range pods { + res := s.podRes[pod] + switch { + case res.rcpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dM", pod, res.cpu, res.mem)) + case res.cpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmRCPU/%dM", pod, res.rcpu, res.mem)) + default: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dmRCPU/%dM", pod, res.cpu, res.rcpu, res.mem)) + } + } + return fmt.Sprintf("[free:%dmCPU/%dmRCPU/%dM pods:[%s]]", s.cpu, s.rcpu, s.mem, strings.Join(pr, " ")) +} + +func createPod(pod string, cpu, rcpu, mem int) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + if s.cpu < cpu || s.rcpu < rcpu || s.mem < mem { + // refuse from state change if not enough resources + return nil + } + if _, ok := s.podRes[pod]; ok { + // refuse to create pod if it is already running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + newPodRes[k] = v + } + newPodRes[pod] = &PodResources{cpu, rcpu, mem} + return &TestState{ + cpu: s.cpu - cpu, + rcpu: s.rcpu - rcpu, + mem: s.mem - mem, + podRes: newPodRes, + } + } +} + +func deletePod(pod string) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + res, ok := s.podRes[pod] + if !ok { + // refuse to delete pod if it is not running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + if k != pod { + newPodRes[k] = v + } + } + return &TestState{ + cpu: s.cpu + res.cpu, + rcpu: s.rcpu + res.rcpu, + mem: s.mem + res.mem, + podRes: newPodRes, + } + } +} + +func newModel() *m.Model { + podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"} + rPodNames := []string{"rbe0", "rgu0", "rbu0"} + + model := m.NewModel() + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + return m.When(true, + m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1*0, 0, 1*0)), + m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3*0, 0, 3*0)), + m.OnAction("NAME=rbe0 CONTCOUNT=2 CPU=0 MEM=0 namespace=kube-system create besteffort").Do(createPod("rbe0", 0, 2*0, 2*0)), + m.When(s.mem > 0, + m.When(s.cpu >= 200, + m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1*200, 0, 1*1500)), + m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2*1000, 0, 2*500)), + m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2*1200, 0, 2*4500)), + m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3*2000, 0, 3*500)), + m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1*4200, 0, 1*100)), + m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1*1200, 0, 1*50)), + m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2*1900, 0, 2*300)), + ), + m.When(s.rcpu > 99, + m.OnAction("NAME=rgu0 CONTCOUNT=2 CPU=100m MEM=1000M namespace=kube-system create guaranteed").Do(createPod("rgu0", 0, 2*100, 2*1000)), + m.OnAction("NAME=rbu0 CONTCOUNT=1 CPU=100m MEM=100M CPUREQ=99m MEMREQ=99M CPULIM=100m MEMLIM=100M namespace=kube-system create burstable").Do(createPod("rbu0", 0, 1*100, 1*100)), + ), + ), + ) + }) + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + ts := []*m.Transition{} + for _, pod := range podNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + for _, pod := range rPodNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod --namespace kube-system %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + return ts + }) + + return model +} + +var ( + maxMem int + maxCpu int + maxReservedCpu int + maxTestSteps int + randomSeed int64 + randomness int + searchDepth int +) + +func main() { + flag.IntVar(&maxMem, "mem", 7500, "memory available for test pods") + flag.IntVar(&maxCpu, "cpu", 15000, "non-reserved milli-CPU available for test pods") + flag.IntVar(&maxReservedCpu, "reserved-cpu", 1000, "reserved milli-CPU availble for test pods") + flag.IntVar(&maxTestSteps, "test-steps", 3000, "number of test steps") + flag.Int64Var(&randomSeed, "random-seed", 0, "random seed for selecting best path") + flag.IntVar(&randomness, "randomness", 0, "the greater the randomness, the larger the set of paths for choosing best path. 0 means no randomness, 5 picks any path that increases coverage.") + flag.IntVar(&searchDepth, "search-depth", 4, "number of steps to look ahead when selecting best path") + + flag.Parse() + + model := newModel() + coverer := m.NewCoverer() + coverer.CoverActionCombinations(3) + + if randomSeed > 0 || randomness > 0 { + coverer.SetBestPathRandom(randomSeed, randomness) + } + + var state m.State + + state = &TestState{ + cpu: maxCpu, + rcpu: maxReservedCpu, + mem: maxMem, + } + fmt.Printf("echo === generated with: --mem=%d --cpu=%d --reserved-cpu=%d --test-steps=%d --random-seed=%d --randomness=%d --search-depth=%d\n", maxMem, maxCpu, maxReservedCpu, maxTestSteps, randomSeed, randomness, searchDepth) + testStep := 0 + for testStep < maxTestSteps { + path, covStats := coverer.BestPath(model, state, searchDepth) + if len(path) == 0 { + fmt.Printf("# did not find anything to cover\n") + break + } + for i := 0; i < covStats.MaxStep+1; i++ { + testStep++ + step := path[i] + fmt.Printf("\necho === step:%d coverage:%d state:%v\n", testStep, coverer.Coverage(), state) + fmt.Println(step.Action()) + state = step.EndState() + coverer.MarkCovered(step) + coverer.UpdateCoverage() + if testStep >= maxTestSteps { + break + } + } + } +} diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh index 6d56afefb..0eedf80bf 100755 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh @@ -10,9 +10,8 @@ Configuring test generation with environment variables: CPU= Non-reserved CPU [mCPU] available for test pods in the system. RESERVED_CPU= Reserved CPU [mCPU] available for test pods in the system. STEPS= Total number of test steps in all parallel tests. + SEARCH_DEPTH= Test generator search depth for best paths. - FMBT_IMAGE= Generate the test using fmbt from docker image IMG:TAG. - The default is fmbt-cli:latest. EOF exit 0 } @@ -28,36 +27,32 @@ MEM=${MEM:-7500} CPU=${CPU:-14050} RESERVED_CPU=${RESERVED_CPU:-1000} STEPS=${STEPS:-100} -FMBT_IMAGE=${FMBT_IMAGE:-"fmbt-cli:latest"} +SEARCH_DEPTH=${SEARCH_DEPTH:-4} mem_per_test=$(( MEM / TESTCOUNT )) cpu_per_test=$(( CPU / TESTCOUNT )) reserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT )) steps_per_test=$(( STEPS / TESTCOUNT )) -# Check fmbt Docker image -docker run "$FMBT_IMAGE" fmbt --version 2>&1 | grep ^Version: || { - echo "error: cannot run fmbt from Docker image '$FMBT_IMAGE'" - echo "You can build the image locally by running:" - echo "( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )" - exit 1 -} - cd "$(dirname "$0")" || { echo "cannot cd to the directory of $0" exit 1 } + for testnum in $(seq 1 "$TESTCOUNT"); do testid=$(( testnum - 1)) - sed -e "s/max_mem=.*/max_mem=${mem_per_test}/" \ - -e "s/max_cpu=.*/max_cpu=${cpu_per_test}/" \ - -e "s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/" \ - < fuzz.aal > tmp.fuzz.aal - sed -e "s/fuzz\.aal/tmp.fuzz.aal/" \ - -e "s/pass = steps(.*/pass = steps(${steps_per_test})/" \ - < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf OUTFILE=generated${testid}.sh + prefix_pods_with_testid="s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" echo "generating $OUTFILE..." - docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" + go run ./generate.go \ + --mem $mem_per_test \ + --cpu $cpu_per_test \ + --reserved-cpu $reserved_cpu_per_test \ + --test-steps $steps_per_test \ + --random-seed $testid \ + --randomness 2 \ + --search-depth $SEARCH_DEPTH \ + | sed -e "$prefix_pods_with_testid" \ + > "$OUTFILE" done diff --git a/test/gofmbt/action.go b/test/gofmbt/action.go new file mode 100644 index 000000000..ce68e099a --- /dev/null +++ b/test/gofmbt/action.go @@ -0,0 +1,73 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +// Action, associated with a state change, specifies what to execute +// to cause corresponding state change on the system under test. +type Action struct { + name string + format string + args []interface{} +} + +// NewAction creates a new action. +func NewAction(format string, args ...interface{}) *Action { + return &Action{ + format: format, + args: args, + name: fmt.Sprintf(format, args...), + } +} + +// String returns a string representation of an action. +func (a *Action) String() string { + return a.name +} + +// When returns a slice containing transitions if enabled is +// true. This is a convenience function for When/OnAction/Do modeling +// syntax. +func When(enabled bool, tss ...[]*Transition) []*Transition { + if !enabled { + return nil + } + ts := []*Transition{} + for _, origTs := range tss { + ts = append(ts, origTs...) + } + return ts +} + +// OnAction returns new Action. This is a convenience function for +// When/OnAction/Do modeling syntax. +func OnAction(format string, args ...interface{}) *Action { + return NewAction(format, args...) +} + +// Do returns a slice containing one transition. Do is a convenience +// function for When/OnAction/Do modeling syntax. +func (a *Action) Do(stateChanges ...StateChange) []*Transition { + stateChange := func(s State) State { + for _, sc := range stateChanges { + s = sc(s) + } + return s + } + return []*Transition{NewTransition(a, stateChange)} +} diff --git a/test/gofmbt/cover.go b/test/gofmbt/cover.go new file mode 100644 index 000000000..3a2c838ed --- /dev/null +++ b/test/gofmbt/cover.go @@ -0,0 +1,338 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "math/rand" + "strings" +) + +const ( + BestPathRandomNone = iota + BestPathRandomAmongEquallyGood + BestPathRandomAmongFastestMaxCoverageIncrease + BestPathRandomAmongMaxCoverageIncrease + BestPathRandomAmongAnyPath +) + +// CoveredInPath is a function that returns a slice of strings covered +// by a path. +type CoveredInPath func(Path) []string + +// Coverer combines what is counted as covered, how to count it, and +// helps finding Paths that increase coverage. +type Coverer struct { + coveredPath Path // Path that is currently covered. + coverCount map[string]int // Strings covered by the coveredPath. + covFuncs []CoveredInPath // Functions that return strings covered by a path. + historyLen int // Length of the history in coveredPath that needs to be considered when estimating coverage increase for new steps that extend the path. + rand *rand.Rand // Random number generator initialized with a given seed. + randomness int // Randomness level. +} + +// NewCoverer creates a new Coverer. +func NewCoverer() *Coverer { + return &Coverer{ + coverCount: map[string]int{}, + } +} + +// ActionNames returns names of actions in a path. +func ActionNames(path Path) []string { + names := []string{} + for _, t := range path { + names = append(names, t.action.name) + } + return names +} + +// ActionFormats returns formats of actions in a path. +func ActionFormats(path Path) []string { + formats := []string{} + for _, t := range path { + formats = append(formats, t.action.format) + } + return formats +} + +// StateStrings returns state names in a path. +func StateStrings(path Path) []string { + if len(path) == 0 { + return nil + } + s := []string{path[0].start.String()} + for _, step := range path { + s = append(s, step.end.String()) + } + return s +} + +// StateActionStrings returns state-action pairs in a path. +func StateActionStrings(path Path) []string { + stateActionSep := "\x00" + stateActions := make([]string, 0, len(path)) + for _, step := range path { + stateActions = append(stateActions, step.StartState().String()+stateActionSep+step.Action().String()) + } + return stateActions +} + +// CoverActions starts counting covered action names. +func (c *Coverer) CoverActions() { + c.addCovFunc(ActionNames) +} + +// CoverActionFormats starts counting covered action formats. +func (c *Coverer) CoverActionFormats() { + c.addCovFunc(ActionFormats) +} + +// CoverActionCombinations starts counting covered action name combinations of length up to combLenMax. +func (c *Coverer) CoverActionCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionNames(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +// CoverActionFormatCombinations starts counting covered action format combinations of length up to combLenMax. +func (c *Coverer) CoverActionFormatCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + actionSep := "\x00" + c.addCovFunc(func(path Path) []string { + actionCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + actionCombs = append(actionCombs, strings.Join(ActionFormats(path[first:first+combLen]), actionSep)) + } + } + return actionCombs + }) +} + +// CoverStates starts counting covered states. +func (c *Coverer) CoverStates() { + c.addCovFunc(StateStrings) +} + +// CoverStateActions starts counting covered state-action pairs. +func (c *Coverer) CoverStateActions() { + c.addCovFunc(StateActionStrings) +} + +// CoverStateCombinations starts counting covered state combinations of length up to combLenMax. +func (c *Coverer) CoverStateCombinations(combLenMax int) { + if c.historyLen < combLenMax { + c.historyLen = combLenMax + } + stateSep := "\x00" + c.addCovFunc(func(path Path) []string { + stateCombs := []string{} + for combLen := 1; combLen <= combLenMax; combLen++ { + for first := 0; first <= len(path)-combLen; first++ { + stateCombs = append(stateCombs, strings.Join(StateStrings(path[first:first+combLen]), stateSep)) + } + } + return stateCombs + }) +} + +func (c *Coverer) addCovFunc(covFunc CoveredInPath) { + c.covFuncs = append(c.covFuncs, covFunc) +} + +func (c *Coverer) covFunc(path Path) []string { + allCovered := []string{} + for _, covFunc := range c.covFuncs { + allCovered = append(allCovered, covFunc(path)...) + } + return allCovered +} + +// Coverage returns the number of unique strings covered. +func (c *Coverer) Coverage() int { + return len(c.coverCount) +} + +// CoveredStrings returns all unique strings covered. +func (c *Coverer) CoveredStrings() []string { + cs := make([]string, 0, len(c.coverCount)) + for s := range c.coverCount { + cs = append(cs, s) + } + return cs +} + +// UpdateCoverage updates the count of covered strings. +func (c *Coverer) UpdateCoverage() { + c.coverCount = map[string]int{} + for _, s := range c.covFunc(c.coveredPath) { + c.coverCount[s]++ + } +} + +// MarkCovered marks a sequence of steps as covered. The sequence is +// appended to the currently covered path. Note that covered strings +// is not updated until UpdateCoverage() is called. +func (c *Coverer) MarkCovered(step ...*Step) { + c.coveredPath = append(c.coveredPath, step...) +} + +// CoverageIncreaseStats holds statistics on estimated coverage +// increase when extending a path. +type CoverageIncreaseStats struct { + MaxStep int // Index of the step in the path extension after which max increase is reached. + MaxIncrease int // Maximum increase in coverage with the path extension. + FirstStep int // Index of the step in the path extension after which first coverage increase is reached. + FirstIncrease int // First increase in coverage with the path extension. +} + +// EstimateCoverageIncrease estimates coverage increase when extending +// currently coveredPath with a new path. +func (c *Coverer) EstimateCoverageIncrease(path Path) *CoverageIncreaseStats { + est := &CoverageIncreaseStats{FirstStep: -1, MaxStep: -1} + fullCoverCount := map[string]int{} + historyLen := c.historyLen + if historyLen > len(c.coveredPath) { + historyLen = len(c.coveredPath) + } + pathWithHistory := append(c.coveredPath[len(c.coveredPath)-historyLen:], path...) + allNewCovered := c.covFunc(pathWithHistory) + for _, s := range allNewCovered { + if c.coverCount[s] == 0 { + fullCoverCount[s]++ + } + } + est.MaxIncrease = len(fullCoverCount) + firstCoverCount := map[string]int{} + for i := historyLen; i < len(pathWithHistory); i++ { + newCovered := c.covFunc(pathWithHistory[:i+1]) + for _, s := range newCovered { + if c.coverCount[s] == 0 { + firstCoverCount[s]++ + } + } + if est.FirstStep == -1 && len(firstCoverCount) > 0 { + est.FirstStep = i - historyLen + est.FirstIncrease = len(firstCoverCount) + } + if est.MaxStep == -1 && len(firstCoverCount) == est.MaxIncrease { + est.MaxStep = i - historyLen + break + } + } + return est +} + +// SetBestPathRandom sets the seed and randomness level for selecting +// a path that increase coverage. +func (c *Coverer) SetBestPathRandom(seed int64, randomness int) { + c.rand = rand.New(rand.NewSource(seed)) + c.randomness = randomness +} + +func (c *Coverer) ShufflePaths(paths []Path) { + if c.rand == nil { + return + } + c.rand.Shuffle(len(paths), func(i, j int) { + paths[i], paths[j] = paths[j], paths[i] + }) +} + +func (c *Coverer) BestPath(m *Model, s State, maxLen int) (Path, *CoverageIncreaseStats) { + var best *CoverageIncreaseStats + bestPath := Path{} + paths := m.Paths(s, maxLen) + if len(paths) == 0 { + return nil, nil + } + if c.randomness != BestPathRandomNone { + c.ShufflePaths(paths) + } + for _, path := range paths { + est := c.EstimateCoverageIncrease(path) + // only return paths that increase coverage + if est.MaxIncrease == 0 { + continue + } + if best == nil { + bestPath, best = path, est + if c.randomness >= BestPathRandomAmongAnyPath { + break + } + continue + } + if est.MaxIncrease < best.MaxIncrease { + continue + } + if est.MaxIncrease > best.MaxIncrease { + bestPath, best = path, est + continue + } + // if we are here, est.MaxIncrease == best.MaxIncrease + if c.randomness == BestPathRandomAmongMaxCoverageIncrease { + // we are free to take any path with the same max increase, never mind about other stats + continue + } + if est.MaxStep > best.MaxStep { + continue + } + if est.MaxStep < best.MaxStep { + bestPath, best = path, est + continue + } + // if we are here, est.MaxStep == best.MaxStep + if c.randomness == BestPathRandomAmongFastestMaxCoverageIncrease { + // we are free to take any path that equally few steps to reach max increase, never mind about other stats + continue + } + if est.FirstStep > best.FirstStep { + continue + } + if est.FirstStep < best.FirstStep { + bestPath, best = path, est + continue + } + // if we are here, est.FirstStep == best.FirstStep + if est.FirstIncrease < best.FirstIncrease { + continue + } + if est.FirstIncrease > best.FirstIncrease { + bestPath, best = path, est + continue + } + // If we are here, est.FirstIncrease == + // best.FirstIncrease that is, est and best paths are + // equally good. BestPathRandomAmongEquallyGood has + // been taken care of by shuffling paths in the + // beginning. + } + if len(bestPath) == 0 { + return nil, nil + } + return bestPath, best +} diff --git a/test/gofmbt/doc.go b/test/gofmbt/doc.go new file mode 100644 index 000000000..432867798 --- /dev/null +++ b/test/gofmbt/doc.go @@ -0,0 +1,161 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gofmbt implements a model-based testing library, including +// tools for +// 1. defining test models +// 2. defining what a test should cover +// 3. generating tests sequences with optimal coverage +// +// # Models +// +// A model specifies what and when can be tested. "What" is specified +// by an Action while "when" is expressed using states and +// transitions. +// +// State is implemented by user. It needs to implement String(). For +// example, if testing a music player, a State can be defined as: +// +// type PlayerState struct { +// playing bool // player is either playing or paused +// song int // the number of the song being played +// } +// +// StateChange is a function that takes a start state as an input and +// returns an end state as an output. If a StateChange function +// returns nil, the state change is unspecified at the start +// state. User defines typically many StateChange functions, each of +// which modify one or more attributes of State. For example, a state +// change that starts a paused player, but is unspecified if the +// player is already playing, can be defined as: +// +// func startPlaying (current State) State { +// s := current.(*PlayerState) +// if s.playing { +// return nil +// } +// return &PlayerState{ +// playing: true, +// song: s.song, +// } +// } +// +// Action is a string, possibly specified by separate +// format+arguments, that identifies what exactly should be done when +// the test generator suggests executing a test step with this +// action. For example, an action can be a keyword or a test step name +// possibly with parameters, or an executable line of any script +// language like "press-button 'play'". +// +// Transition is a combination of an Action and a StateChange +// function. For example, +// +// play := NewTransition(NewAction("press-button '%s'", "play"), startPlaying) +// +// Finally, Transitions are added to a Model using transition +// generator functions. They are functions that return a slice of +// Transitions that may be specified in a State. For instance: +// +// model.From(func(current State) []*Transition { +// return []*Transition{play} +// }) +// +// Note that the transition generator function can already do checks +// on the current State attributes and return only transitions that +// are specified at the state. However, as the StateChange function of +// a returned transition may return nil, not all returned transitions +// need to be defined at the state. Rather, transition generator +// functions enable having common preconditions for all transitions +// that the generator may return. Model.From() can be called multiple +// times to add multiple transition generator functions. +// +// Refer to model_test.go to find examples of defining the same model +// for a player in two different ways: first with StateChanges and +// Transitions, and then with convenience functions When/OnAction/Do. +// +// # Test generation +// +// Tests are sequences of Steps. Every Step has an Action, a start +// State, and an end State. Model.StepsFrom(State) returns all +// possible Steps whose start state is State. +// +// Path is a sequence of Steps where the end state of a Step is the +// start state of the next Step. Model.Paths(State, maxLen) returns +// all possible Paths of at most maxLen Steps where the first Step of +// every Path starts from the State. +// +// Coverer helps finding Paths that increase coverage of wanted +// elements. Elements to be covered are specified by Coverer methods: +// - CoverStates(): cover unique State.String()s: +// visit every state. +// - CoverStateActions(): unique StartState().String() + Action().String(): +// test every action in every state. +// - CoverStateCombination(n): unique State_1, ..., State_n combinations: +// test all state-paths of length n. +// - CoverActions(): unique Action.Strings()s: +// test every action. Different parameters counts as different actions. +// - CoverActionCombinations(n): unique Action_1, ..., Action_n combinations: +// test all action-paths of length n. +// - CoverActionFormats(): unique Action formats: +// test every action format, ignoring action parameters. +// +// Calling multiple Cover*() functions allows specifying multiple +// elements whose coverage counts. For example, CoverActions() and +// CoverStates() counts every new Action and every new State, but it +// does not require executing every Action and in every State like +// CoverStateActions() would do. On the other hand, if all these three +// functions are called, then the best test Path is one that gives +// greatest increase the coverage of all the three elements at the +// same time. In practice, this prioritises testing new Actions and +// new States as long as they are found, at the same time when trying +// to cover every Action in every State. +// +// Cover.BestPath(Model, State, maxLen) returns a Path, starting +// from a State in a Model, that results in largest increase in +// whatever elements are covered. The Path is nil if coverage cannot +// be increased by any Path of at most maxLen Steps. +// +// When one or more Steps in a Path have been handled, they are marked +// as covered by calling Coverer.MarkCovered(Step...). Having all +// marked, Coverer.UpdateCoverage() must be called. This can be heavy +// operation, depending on how coverage is measured and how many steps +// have been covered, and therefore it is done separately from marking +// individual Steps covered. Once updated, Coverer.Coverage() returns +// the total number of elements that have been covered by in all +// marked Steps, and Coverer.BestPath() will use new coverage as basis +// when searching for new BestPaths(). +// +// Test generation loop example: +// +// model := myModel() +// state := &MyModelState { ... } // initial state of generated test +// coverer := NewCoverer() +// coverer.CoverActions() +// coverer.CoverStates() +// coverer.CoverStateActions() +// for { +// path, stats := coverer.BestPath(model, state, 6) +// if len(path) == 0 { +// break // could not find a path that increased coverage +// } +// for _, step := range path[:stats.FirstStep+1] { +// fmt.Printf("# coverage: %d, step: %s\n", coverer.Coverage(), step) +// fmt.Printf("%s\n", step.Action()) +// coverer.MarkCovered(step) +// coverer.UpdateCoverage() +// } +// state = path[stats.FirstStep].EndState() +// } + +package gofmbt diff --git a/test/gofmbt/model.go b/test/gofmbt/model.go new file mode 100644 index 000000000..00c5ce884 --- /dev/null +++ b/test/gofmbt/model.go @@ -0,0 +1,73 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +// Model specifies a state space. +type Model struct { + gen []TransitionGen // transition generators +} + +// NewModel creates a new model. +func NewModel() *Model { + return &Model{} +} + +// From adds a transition generator to the model. +func (m *Model) From(transitionGen func(State) []*Transition) { + m.gen = append(m.gen, transitionGen) +} + +// TransitionsFrom returns all transitions that may be taken from a given state. +func (m *Model) TransitionsFrom(s State) []*Transition { + ts := []*Transition{} + for _, gen := range m.gen { + ts = append(ts, gen(s)...) + } + return ts +} + +// Steps returns all steps that start from a given state. +func (m *Model) StepsFrom(s State) []*Step { + steps := []*Step{} + for _, t := range m.TransitionsFrom(s) { + if endState := t.stateChange(s); endState != nil { + steps = append(steps, NewStep(s, t.action, endState)) + } + } + return steps +} + +// Paths returns all paths of at most maxLen steps that start from a given state. +func (m *Model) Paths(s State, maxLen int) []Path { + paths := []Path{} + if maxLen == 0 { + return nil + } + for _, step := range m.StepsFrom(s) { + newPath := NewPath(step) + childPaths := m.Paths(step.EndState(), maxLen-1) + if len(childPaths) == 0 { + paths = append(paths, newPath) + } else { + for _, childPath := range childPaths { + paths = append(paths, append(newPath, childPath...)) + } + } + } + if len(paths) == 0 { + return nil + } + return paths +} diff --git a/test/gofmbt/model_test.go b/test/gofmbt/model_test.go new file mode 100644 index 000000000..3161eeaf4 --- /dev/null +++ b/test/gofmbt/model_test.go @@ -0,0 +1,234 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" + "testing" +) + +type PlayerState struct { + playing bool + song int +} + +func (ps *PlayerState) String() string { + return fmt.Sprintf("{playing:%v,song:%v}", ps.playing, ps.song) +} + +// This is Rosetta's stone on modelling directly with StateChange +// functions and NewTransitions in newPlayerModelWithRawTransitions, +// and writing exactly the same model using When/OnAction/Do. +func newPlayerModelWithRawTransitions() *Model { + model := NewModel() + play := func(current State) State { + s := current.(*PlayerState) + if s.playing { + return nil + } + return &PlayerState{ + playing: true, + song: s.song, + } + } + pause := func(current State) State { + s := current.(*PlayerState) + if !s.playing { + return nil + } + return &PlayerState{ + playing: false, + song: s.song, + } + } + nextsong := func(current State) State { + s := current.(*PlayerState) + if s.song >= 3 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song + 1, + } + } + prevsong := func(current State) State { + s := current.(*PlayerState) + if s.song <= 1 { + return nil + } + return &PlayerState{ + playing: s.playing, + song: s.song - 1, + } + } + + model.From(func(current State) []*Transition { + return []*Transition{ + NewTransition(NewAction("play"), play), + NewTransition(NewAction("pause"), pause), + NewTransition(NewAction("nextsong"), nextsong), + NewTransition(NewAction("prevsong"), prevsong), + } + }) + + return model +} + +func newPlayerModelWithWhenOnAction() *Model { + setState := func(playing bool, song int) StateChange { + return func(_ State) State { + return &PlayerState{playing, song} + } + } + + model := NewModel() + model.From(func(start State) []*Transition { + s := start.(*PlayerState) + return When(true, + When(s.playing, + OnAction("pause").Do(setState(false, s.song))), + When(!s.playing, + OnAction("play").Do(setState(true, s.song))), + When(s.song < 3, + OnAction("nextsong").Do(setState(s.playing, s.song+1))), + When(s.song > 1, + OnAction("prevsong").Do(setState(s.playing, s.song-1))), + ) + }) + return model +} + +var playerModels map[string]*Model = map[string]*Model{ + "raw": newPlayerModelWithRawTransitions(), + "when": newPlayerModelWithWhenOnAction(), +} + +func TestCoverPlayerActions(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverActionCombinations(1) + path, stats := coverer.BestPath(model, state, 6) + if len(path) != 6 { + t.Fatalf("model %q: expected len(path)==6, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 4 { + t.Fatalf("model %q: expected reaching coverage 4, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 3 { + t.Fatalf("model %q: expected all actions covered at step 4, got %d", modelName, stats.MaxStep) + } + + covered := map[string]int{} + for _, step := range path[:stats.MaxStep+1] { + covered[step.Action().String()] += 1 + } + if len(covered) != 4 { + t.Fatalf("model %q: expected 4 different actions, got %d", modelName, len(covered)) + } + for _, expectedStep := range []string{"play", "pause", "nextsong", "prevsong"} { + if _, ok := covered[expectedStep]; !ok { + t.Fatalf("model %q: expected %q in covered, got: %v", modelName, expectedStep, covered) + } + } + } +} + +func TestCoverPlayerStates(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStates() + path, stats := coverer.BestPath(model, state, 8) + if len(path) != 8 { + t.Fatalf("model %q: expected len(path)==10, got %d", modelName, len(path)) + } + if stats.MaxIncrease != 6 { + t.Fatalf("model %q: expected reaching 6 different states, got %d", modelName, stats.MaxIncrease) + } + if stats.MaxStep != 4 { + t.Fatalf("model %q: expected all stats visited at step 4, got %d", modelName, stats.MaxStep) + } + if len(coverer.CoveredStrings()) != 0 { + t.Fatalf("model %q: expected nothing to be covered yet, got %v", modelName, coverer.CoveredStrings()) + } + coverer.MarkCovered(path[:stats.MaxStep+1]...) + coverer.UpdateCoverage() + if len(coverer.CoveredStrings()) != 6 { + t.Fatalf("model %q: expected 6 states covered, got %d %v", modelName, len(coverer.CoveredStrings()), coverer.CoveredStrings()) + } + } +} + +func TestStepsFrom(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + altSteps := model.StepsFrom(state) + if len(altSteps) != 2 { + t.Fatalf("model %q: expected 2 StepsFrom %s, got %d", modelName, state, len(altSteps)) + } + for _, step := range altSteps { + if step.Action().String() == "play" { + continue + } + if step.Action().String() == "nextsong" { + continue + } + t.Fatalf("model %q: unexpected action %q in state %s. Step: %s", modelName, step.Action(), state, step) + } + } +} + +func TestSeachPathsTestSteps(t *testing.T) { + for modelName, model := range playerModels { + state := &PlayerState{false, 1} + coverer := NewCoverer() + coverer.CoverStateActions() + t.Log(modelName) + i := 0 + for { + path, stats := coverer.BestPath(model, state, 6) + if len(path) == 0 { + break + } + // Verify prev step.EndState() == next step.StartState() + var prev *Step + for _, step := range path { + if prev != nil && step.StartState().String() != prev.EndState().String() { + t.Fatalf("model %q: prev end state and next start state differ: %s != %s", modelName, prev, step) + } + prev = step + } + + // Execute path up to the first step where + // coverage increases and search for the + // BestPath continuing from there. + for _, step := range path[:stats.FirstStep+1] { + i++ + t.Log("step:", step) + coverer.MarkCovered(step) + coverer.UpdateCoverage() + t.Log("coverage:", coverer.Coverage(), "strings:", coverer.CoveredStrings()) + } + state = path[stats.FirstStep].EndState().(*PlayerState) + } + if coverer.Coverage() != 14 { + t.Fatalf("model %q: expected 14 {state}--action--> combinations to be covered, got %d", modelName, coverer.Coverage()) + } + if i != 14 { + t.Fatalf("model %q: expected reach max coverage with 14 steps, needed: %d", modelName, i) + } + } +} diff --git a/test/gofmbt/state.go b/test/gofmbt/state.go new file mode 100644 index 000000000..7c14fa9cf --- /dev/null +++ b/test/gofmbt/state.go @@ -0,0 +1,21 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +// State is a state of a model. This is defined by the user and solely +// depends on what is being modeled. +type State interface { + String() string +} diff --git a/test/gofmbt/step.go b/test/gofmbt/step.go new file mode 100644 index 000000000..ffcd54885 --- /dev/null +++ b/test/gofmbt/step.go @@ -0,0 +1,63 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +import ( + "fmt" +) + +// Step represents a step in a path. +type Step struct { + start State + action *Action + end State +} + +// Path represents a path in a model. +type Path []*Step + +// NewStep creates a new step. +func NewStep(start State, action *Action, end State) *Step { + return &Step{start, action, end} +} + +// String returns a string representation of a step. +func (step *Step) String() string { + return fmt.Sprintf("[%s--%s->%s]", step.start, step.action, step.end) +} + +// StartState returns the start state of a step. +func (step *Step) StartState() State { + return step.start +} + +// Action returns the action of a step. +func (step *Step) Action() *Action { + return step.action +} + +// EndState returns the end state of a step. +func (step *Step) EndState() State { + return step.end +} + +// NewPath creates a new path from steps. +func NewPath(steps ...*Step) Path { + path := Path{} + for _, step := range steps { + path = append(path, step) + } + return path +} diff --git a/test/gofmbt/transition.go b/test/gofmbt/transition.go new file mode 100644 index 000000000..e00abb988 --- /dev/null +++ b/test/gofmbt/transition.go @@ -0,0 +1,50 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gofmbt + +// StateChange is a function associated to a transition. The function +// returns new state that reflects the change in the state space. +// StateChange function must not modify the original state. If +// StateChange returns nil, the transition is not possible in the +// original state. +type StateChange func(State) State + +// TransitionGen is a function that generates transitions from a given +// state. +type TransitionGen func(State) []*Transition + +// Transition represents state changes by an action in various +// states. +type Transition struct { + action *Action + stateChange StateChange +} + +// NewTransition creates a new transition. +func NewTransition(a *Action, sc StateChange) *Transition { + return &Transition{a, sc} +} + +// Action returns the action of a transition. +func (t *Transition) Action() *Action { + return t.action +} + +// StateChange calls the state change function of a transition. It +// returns the new state after the transition or nil if the transition +// is not possible in the original state. +func (t *Transition) StateChange(s State) State { + return t.stateChange(s) +}