From 7d4cc4442e795c3a7e2a866b81e3a3fe50338041 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Sun, 12 Jan 2025 20:53:31 +0200 Subject: [PATCH] e2e: topology-aware fuzz switches from fmbt to gofmbt Drop dependency to fMBT in e2e fuzz test generation and replace it with gofmbt. Signed-off-by: Antti Kervinen --- go.mod | 1 + go.sum | 2 + .../n4c16/test06-fuzz/generate.go | 200 ++++++++++++++++++ .../n4c16/test06-fuzz/generate.sh | 33 ++- 4 files changed, 217 insertions(+), 19 deletions(-) create mode 100644 test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go diff --git a/go.mod b/go.mod index 67df521fa..c3f8b9619 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/containers/nri-plugins go 1.23.4 require ( + github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f github.com/containerd/nri v0.6.0 github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956 diff --git a/go.sum b/go.sum index 3900efd13..8bf8e72d2 100644 --- a/go.sum +++ b/go.sum @@ -608,6 +608,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f h1:AKRIaPPDqBRhpWnvxhvtdbVtkV/3XrboabuFaLyp1kw= +github.com/askervin/gofmbt v0.0.0-20250119175120-506d925f666f/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go new file mode 100644 index 000000000..dde2bc8f7 --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.go @@ -0,0 +1,200 @@ +package main + +import ( + "flag" + "fmt" + "sort" + "strings" + + m "github.com/askervin/gofmbt/gofmbt" +) + +type PodResources struct { + cpu int // total CPU allocated for all containers in the pod + rcpu int // total reserved CPU allocated for all containers in the pod + mem int // total memory allocated for all containers in the pod +} + +type TestState struct { + cpu int // free CPU on node for non-reserved pods + rcpu int // free CPU on node for reserved pods + mem int // free memory on node + podRes map[string]*PodResources // map running pod name to resources allocated to it +} + +func (s *TestState) String() string { + pr := []string{} + pods := make([]string, 0, len(s.podRes)) + for pod := range s.podRes { + pods = append(pods, pod) + } + sort.Strings(pods) + for _, pod := range pods { + res := s.podRes[pod] + switch { + case res.rcpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dM", pod, res.cpu, res.mem)) + case res.cpu == 0: + pr = append(pr, fmt.Sprintf("%s:%dmRCPU/%dM", pod, res.rcpu, res.mem)) + default: + pr = append(pr, fmt.Sprintf("%s:%dmCPU/%dmRCPU/%dM", pod, res.cpu, res.rcpu, res.mem)) + } + } + return fmt.Sprintf("[free:%dmCPU/%dmRCPU/%dM pods:[%s]]", s.cpu, s.rcpu, s.mem, strings.Join(pr, " ")) +} + +func createPod(pod string, cpu, rcpu, mem int) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + if s.cpu < cpu || s.rcpu < rcpu || s.mem < mem { + // refuse from state change if not enough resources + return nil + } + if _, ok := s.podRes[pod]; ok { + // refuse to create pod if it is already running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + newPodRes[k] = v + } + newPodRes[pod] = &PodResources{cpu, rcpu, mem} + return &TestState{ + cpu: s.cpu - cpu, + rcpu: s.rcpu - rcpu, + mem: s.mem - mem, + podRes: newPodRes, + } + } +} + +func deletePod(pod string) m.StateChange { + return func(current m.State) m.State { + s := current.(*TestState) + res, ok := s.podRes[pod] + if !ok { + // refuse to delete pod if it is not running + return nil + } + newPodRes := make(map[string]*PodResources) + for k, v := range s.podRes { + if k != pod { + newPodRes[k] = v + } + } + return &TestState{ + cpu: s.cpu + res.cpu, + rcpu: s.rcpu + res.rcpu, + mem: s.mem + res.mem, + podRes: newPodRes, + } + } +} + +func newModel() *m.Model { + podNames := []string{"gu0", "gu1", "gu2", "gu3", "gu4", "bu0", "bu1", "be0", "be1"} + rPodNames := []string{"rbe0", "rgu0", "rbu0"} + + model := m.NewModel() + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + return m.When(true, + m.OnAction("NAME=be0 CONTCOUNT=1 CPU=0 MEM=0 create besteffort").Do(createPod("be0", 1*0, 0, 1*0)), + m.OnAction("NAME=be1 CONTCOUNT=3 CPU=0 MEM=0 create besteffort").Do(createPod("be1", 3*0, 0, 3*0)), + m.OnAction("NAME=rbe0 CONTCOUNT=2 CPU=0 MEM=0 namespace=kube-system create besteffort").Do(createPod("rbe0", 0, 2*0, 2*0)), + m.When(s.mem > 0, + m.When(s.cpu >= 200, + m.OnAction("NAME=gu0 CONTCOUNT=1 CPU=200m MEM=1500M create guaranteed").Do(createPod("gu0", 1*200, 0, 1*1500)), + m.OnAction("NAME=gu1 CONTCOUNT=2 CPU=1000m MEM=500M create guaranteed").Do(createPod("gu1", 2*1000, 0, 2*500)), + m.OnAction("NAME=gu2 CONTCOUNT=2 CPU=1200m MEM=4500M create guaranteed").Do(createPod("gu2", 2*1200, 0, 2*4500)), + m.OnAction("NAME=gu3 CONTCOUNT=3 CPU=2000m MEM=500M create guaranteed").Do(createPod("gu3", 3*2000, 0, 3*500)), + m.OnAction("NAME=gu4 CONTCOUNT=1 CPU=4200m MEM=100M create guaranteed").Do(createPod("gu4", 1*4200, 0, 1*100)), + m.OnAction("NAME=bu0 CONTCOUNT=1 CPU=1200m MEM=50M CPUREQ=900m MEMREQ=49M CPULIM=1200m MEMLIM=50M create burstable").Do(createPod("bu0", 1*1200, 0, 1*50)), + m.OnAction("NAME=bu1 CONTCOUNT=2 CPU=1900m MEM=300M CPUREQ=1800m MEMREQ=299M CPULIM=1900m MEMLIM=300M create burstable").Do(createPod("bu1", 2*1900, 0, 2*300)), + ), + m.When(s.rcpu > 99, + m.OnAction("NAME=rgu0 CONTCOUNT=2 CPU=100m MEM=1000M namespace=kube-system create guaranteed").Do(createPod("rgu0", 0, 2*100, 2*1000)), + m.OnAction("NAME=rbu0 CONTCOUNT=1 CPU=100m MEM=100M CPUREQ=99m MEMREQ=99M CPULIM=100m MEMLIM=100M namespace=kube-system create burstable").Do(createPod("rbu0", 0, 1*100, 1*100)), + ), + ), + ) + }) + + model.From(func(current m.State) []*m.Transition { + s := current.(*TestState) + ts := []*m.Transition{} + for _, pod := range podNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + for _, pod := range rPodNames { + if _, ok := s.podRes[pod]; ok { + ts = append(ts, m.OnAction("NAME=%s vm-command 'kubectl delete pod --namespace kube-system %s --now'", pod, pod).Do(deletePod(pod))...) + } + } + return ts + }) + + return model +} + +var ( + maxMem int + maxCpu int + maxReservedCpu int + maxTestSteps int + randomSeed int64 + randomness int + searchDepth int +) + +func main() { + flag.IntVar(&maxMem, "mem", 7500, "memory available for test pods") + flag.IntVar(&maxCpu, "cpu", 15000, "non-reserved milli-CPU available for test pods") + flag.IntVar(&maxReservedCpu, "reserved-cpu", 1000, "reserved milli-CPU availble for test pods") + flag.IntVar(&maxTestSteps, "test-steps", 3000, "number of test steps") + flag.Int64Var(&randomSeed, "random-seed", 0, "random seed for selecting best path") + flag.IntVar(&randomness, "randomness", 0, "the greater the randomness, the larger the set of paths for choosing best path. 0 means no randomness, 5 picks any path that increases coverage.") + flag.IntVar(&searchDepth, "search-depth", 4, "number of steps to look ahead when selecting best path") + + flag.Parse() + + model := newModel() + coverer := m.NewCoverer() + coverer.CoverActionCombinations(3) + + if randomSeed > 0 || randomness > 0 { + coverer.SetBestPathRandom(randomSeed, randomness) + } + + var state m.State + + state = &TestState{ + cpu: maxCpu, + rcpu: maxReservedCpu, + mem: maxMem, + } + fmt.Printf("echo === generated with: --mem=%d --cpu=%d --reserved-cpu=%d --test-steps=%d --random-seed=%d --randomness=%d --search-depth=%d\n", maxMem, maxCpu, maxReservedCpu, maxTestSteps, randomSeed, randomness, searchDepth) + testStep := 0 + for testStep < maxTestSteps { + path, covStats := coverer.BestPath(model, state, searchDepth) + if len(path) == 0 { + fmt.Printf("# did not find anything to cover\n") + break + } + for i := 0; i < covStats.MaxStep+1; i++ { + testStep++ + step := path[i] + fmt.Printf("\necho === step:%d coverage:%d state:%v\n", testStep, coverer.Coverage(), state) + fmt.Println(step.Action()) + state = step.EndState() + coverer.MarkCovered(step) + coverer.UpdateCoverage() + if testStep >= maxTestSteps { + break + } + } + } +} diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh index 6d56afefb..0eedf80bf 100755 --- a/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test06-fuzz/generate.sh @@ -10,9 +10,8 @@ Configuring test generation with environment variables: CPU= Non-reserved CPU [mCPU] available for test pods in the system. RESERVED_CPU= Reserved CPU [mCPU] available for test pods in the system. STEPS= Total number of test steps in all parallel tests. + SEARCH_DEPTH= Test generator search depth for best paths. - FMBT_IMAGE= Generate the test using fmbt from docker image IMG:TAG. - The default is fmbt-cli:latest. EOF exit 0 } @@ -28,36 +27,32 @@ MEM=${MEM:-7500} CPU=${CPU:-14050} RESERVED_CPU=${RESERVED_CPU:-1000} STEPS=${STEPS:-100} -FMBT_IMAGE=${FMBT_IMAGE:-"fmbt-cli:latest"} +SEARCH_DEPTH=${SEARCH_DEPTH:-4} mem_per_test=$(( MEM / TESTCOUNT )) cpu_per_test=$(( CPU / TESTCOUNT )) reserved_cpu_per_test=$(( RESERVED_CPU / TESTCOUNT )) steps_per_test=$(( STEPS / TESTCOUNT )) -# Check fmbt Docker image -docker run "$FMBT_IMAGE" fmbt --version 2>&1 | grep ^Version: || { - echo "error: cannot run fmbt from Docker image '$FMBT_IMAGE'" - echo "You can build the image locally by running:" - echo "( cd /tmp && git clone --branch devel https://github.com/intel/fmbt && cd fmbt && docker build . -t $FMBT_IMAGE -f Dockerfile.fmbt-cli )" - exit 1 -} - cd "$(dirname "$0")" || { echo "cannot cd to the directory of $0" exit 1 } + for testnum in $(seq 1 "$TESTCOUNT"); do testid=$(( testnum - 1)) - sed -e "s/max_mem=.*/max_mem=${mem_per_test}/" \ - -e "s/max_cpu=.*/max_cpu=${cpu_per_test}/" \ - -e "s/max_reserved_cpu=.*/max_reserved_cpu=${reserved_cpu_per_test}/" \ - < fuzz.aal > tmp.fuzz.aal - sed -e "s/fuzz\.aal/tmp.fuzz.aal/" \ - -e "s/pass = steps(.*/pass = steps(${steps_per_test})/" \ - < fuzz.fmbt.conf > tmp.fuzz.fmbt.conf OUTFILE=generated${testid}.sh + prefix_pods_with_testid="s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" echo "generating $OUTFILE..." - docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" + go run ./generate.go \ + --mem $mem_per_test \ + --cpu $cpu_per_test \ + --reserved-cpu $reserved_cpu_per_test \ + --test-steps $steps_per_test \ + --random-seed $testid \ + --randomness 2 \ + --search-depth $SEARCH_DEPTH \ + | sed -e "$prefix_pods_with_testid" \ + > "$OUTFILE" done