Fix failing eden tests

Multiple patches in this PR: * Increased the default EVE version to the latest 12.4 * Removed workflow dependency on Smoke test suite. With reusable workflows this does not work properly in the EVE repository. The other test suites are triggered regardless of the outcome of Smoke tests. The only effect this dependency has, is that other workflows are delayed from starting until Smoke tests finish, increasing the total execution time of all tests. * Moved tests that depend on virtualization (apps must be deployed inside VMs) to a separate test suite. The primary reason is that the HW-assisted nested virtualization is performing poorly on the buildjet runners and we are getting many failures (not related to EVE code). For every other test it is therefore better to run EVE with acceleration disabled. A secondary reason is that most of these tests are from the networking test suite, which is already quite long and takes well over an hour to execute. It therefore makes sense to move some tests to a different test suite. Please note that when the Virtualization test suite is run locally, all tests are passing. However, on buildjet runners they are expected to fail until we resolve the issue with the nested virtualization. * Fixed 'eden pod modify' (wrong timeout + app state to wait for had to be modified to reflect recent EVE changes) * Fixed ctrl_cert_change test (improper use of -check-new) * App 'nodered' needs more time to get deployed * Test publish_location is now skipped and likely will be removed. This is because after the recent changes in the EVE/wwan microservice, injecting a fake GPS location data is no longer possible. * Modified EVE-upgrade tests to use newer base EVE version (> 12.0). This is because the policy_version of fscrypt was bumped from 1 to 2 and this change is not backward compatible. Meaning we cannot downgrade from 12.X to 11.Y or older. Signed-off-by: Milan Lenco <[email protected]>
lf-edge · Jun 21, 2024 · 6e38968 · 6e38968
1 parent e255da8
commit 6e38968
Show file tree

Hide file tree

Showing 22 changed files with 210 additions and 73 deletions.
diff --git a/.github/actions/run-eden-test/action.yml b/.github/actions/run-eden-test/action.yml
@@ -17,6 +17,8 @@ inputs:
     type: string
   artifact_run_id:
     type: string
+  require_virtualization:
+    type: bool
 
 runs:
   using: 'composite'
@@ -29,6 +31,7 @@ runs:
         eve_image: ${{ inputs.eve_image }}
         eve_artifact_name: ${{ inputs.eve_artifact_name }}
         artifact_run_id: ${{ inputs.artifact_run_id }}
+        require_virtualization: ${{ inputs.require_virtualization }}
     - name: Run tests
       run: EDEN_TEST_STOP=n ./eden test ./tests/workflow -s ${{ inputs.suite }} -v debug
       shell: bash

diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml
@@ -14,6 +14,8 @@ inputs:
     type: string
   artifact_run_id:
     type: string
+  require_virtualization:
+    type: bool
 
 runs:
   using: 'composite'
@@ -48,7 +50,12 @@ runs:
     - name: Configure
       run: |
         ./eden config add default
-        if lscpu | grep -oEq "vmx|svm"; then
+        require_virt="${{ inputs.require_virtualization }}"
+        if [[ "$require_virt" == "true" ]]; then
+          if ! lscpu | grep -oEq "vmx|svm"; then
+            echo "Missing required HW-assisted virtualization support"
+            exit 1
+          fi
           ./eden config set default --key=eve.accel --value=true
         else
           ./eden config set default --key=eve.accel --value=false

diff --git a/.github/workflows/eden.yml b/.github/workflows/eden.yml
@@ -15,5 +15,5 @@ jobs:
     name: Execute Eden test workflow
     uses: ./.github/workflows/test.yml
     with:
-      eve_image: "lfedge/eve:11.7.0"
+      eve_image: "lfedge/eve:12.4.0"
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -71,7 +71,7 @@ jobs:
 
   networking:
     name: Networking test suite
-    needs: [smoke, determine-runner]
+    needs: [determine-runner]
     runs-on: ${{ needs.determine-runner.outputs.runner }}
     steps:
       - name: Get code
@@ -96,7 +96,7 @@ jobs:
       matrix:
         file_system: ['ext4', 'zfs']
     name: Storage test suite
-    needs: [smoke, determine-runner]
+    needs: [determine-runner]
     runs-on: ${{ needs.determine-runner.outputs.runner }}
     steps:
       - name: Get code
@@ -115,9 +115,9 @@ jobs:
           eve_artifact_name: ${{ inputs.eve_artifact_name }}
           artifact_run_id: ${{ inputs.artifact_run_id }}
 
-  lpc-loc:
-    name: LPC LOC test suite
-    needs: [smoke, determine-runner]
+  lps-loc:
+    name: LPS LOC test suite
+    needs: [determine-runner]
     runs-on: ${{ needs.determine-runner.outputs.runner }}
     steps:
       - name: Get code
@@ -126,12 +126,12 @@ jobs:
           repository: "lf-edge/eden"
           ref: ${{ inputs.eden_version }}
           path: "./eden"
-      - name: Run LPC LOC tests
+      - name: Run LPS and LOC tests
         uses: ./eden/.github/actions/run-eden-test
         with:
           file_system: "ext4"
           tpm_enabled: true
-          suite: "lpc-loc.tests.txt"
+          suite: "lps-loc.tests.txt"
           eve_image: ${{ inputs.eve_image }}
           eve_artifact_name: ${{ inputs.eve_artifact_name }}
           artifact_run_id: ${{ inputs.artifact_run_id }}
@@ -142,7 +142,7 @@ jobs:
       matrix:
         file_system: ['ext4', 'zfs']
     name: EVE upgrade test suite
-    needs: [smoke, determine-runner]
+    needs: [determine-runner]
     runs-on: ${{ needs.determine-runner.outputs.runner }}
     steps:
       - name: Get code
@@ -163,7 +163,7 @@ jobs:
 
   user-apps:
     name: User apps test suite
-    needs: [smoke, determine-runner]
+    needs: [determine-runner]
     runs-on: ${{ needs.determine-runner.outputs.runner }}
     steps:
       - name: Get code
@@ -172,7 +172,7 @@ jobs:
           repository: "lf-edge/eden"
           ref: ${{ inputs.eden_version }}
           path: "./eden"
-      - name: Run User apps upgrade tests
+      - name: Run User apps tests
         uses: ./eden/.github/actions/run-eden-test
         with:
           file_system: "ext4"
@@ -181,3 +181,25 @@ jobs:
           eve_image: ${{ inputs.eve_image }}
           eve_artifact_name: ${{ inputs.eve_artifact_name }}
           artifact_run_id: ${{ inputs.artifact_run_id }}
+
+  virtualization:
+    name: Virtualization test suite
+    needs: [determine-runner]
+    runs-on: ${{ needs.determine-runner.outputs.runner }}
+    steps:
+      - name: Get code
+        uses: actions/[email protected]
+        with:
+          repository: "lf-edge/eden"
+          ref: ${{ inputs.eden_version }}
+          path: "./eden"
+      - name: Run Virtualization tests
+        uses: ./eden/.github/actions/run-eden-test
+        with:
+          file_system: "ext4"
+          tpm_enabled: true
+          suite: "virtualization.tests.txt"
+          eve_image: ${{ inputs.eve_image }}
+          eve_artifact_name: ${{ inputs.eve_artifact_name }}
+          artifact_run_id: ${{ inputs.artifact_run_id }}
+          require_virtualization: true
diff --git a/pkg/controller/loaders/fileLoader.go b/pkg/controller/loaders/fileLoader.go
@@ -128,7 +128,7 @@ func (loader *FileLoader) ProcessStream(process ProcessFunction, typeToProcess t
 	done := make(chan error)
 
 	if timeoutSeconds != 0 {
-		time.AfterFunc(timeoutSeconds*time.Second, func() {
+		time.AfterFunc(timeoutSeconds, func() {
 			done <- fmt.Errorf("timeout")
 		})
 	}

diff --git a/pkg/controller/loaders/redisLoader.go b/pkg/controller/loaders/redisLoader.go
@@ -15,7 +15,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-//RedisLoader implements loader from redis backend of controller
+// RedisLoader implements loader from redis backend of controller
 type RedisLoader struct {
 	lastID        string
 	addr          string
@@ -28,7 +28,7 @@ type RedisLoader struct {
 	appUUID       uuid.UUID
 }
 
-//NewRedisLoader return loader from redis
+// NewRedisLoader return loader from redis
 func NewRedisLoader(addr string, password string, databaseID int, streamGetters types.StreamGetters) *RedisLoader {
 	log.Debugf("NewRedisLoader init")
 	return &RedisLoader{
@@ -39,12 +39,12 @@ func NewRedisLoader(addr string, password string, databaseID int, streamGetters
 	}
 }
 
-//SetRemoteCache add cache layer
+// SetRemoteCache add cache layer
 func (loader *RedisLoader) SetRemoteCache(cache cachers.CacheProcessor) {
 	loader.cache = cache
 }
 
-//Clone create copy
+// Clone create copy
 func (loader *RedisLoader) Clone() Loader {
 	return &RedisLoader{
 		addr:          loader.addr,
@@ -77,12 +77,12 @@ func (loader *RedisLoader) getStream(typeToProcess types.LoaderObjectType) strin
 	}
 }
 
-//SetUUID set device UUID
+// SetUUID set device UUID
 func (loader *RedisLoader) SetUUID(devUUID uuid.UUID) {
 	loader.devUUID = devUUID
 }
 
-//SetAppUUID set app UUID
+// SetAppUUID set app UUID
 func (loader *RedisLoader) SetAppUUID(appUUID uuid.UUID) {
 	loader.appUUID = appUUID
 }
@@ -216,22 +216,22 @@ func (loader *RedisLoader) getOrCreateClient() (*redis.Client, error) {
 	return loader.client, err
 }
 
-//ProcessExisting for observe existing files
+// ProcessExisting for observe existing files
 func (loader *RedisLoader) ProcessExisting(process ProcessFunction, typeToProcess types.LoaderObjectType) error {
 	if _, err := loader.getOrCreateClient(); err != nil {
 		return err
 	}
 	return loader.repeatableConnection(process, typeToProcess, false)
 }
 
-//ProcessStream for observe new files
+// ProcessStream for observe new files
 func (loader *RedisLoader) ProcessStream(process ProcessFunction, typeToProcess types.LoaderObjectType, timeoutSeconds time.Duration) (err error) {
 	if _, err := loader.getOrCreateClient(); err != nil {
 		return err
 	}
 	done := make(chan error)
 	if timeoutSeconds != 0 {
-		time.AfterFunc(timeoutSeconds*time.Second, func() {
+		time.AfterFunc(timeoutSeconds, func() {
 			done <- fmt.Errorf("timeout")
 		})
 	}

diff --git a/pkg/controller/loaders/remoteLoader.go b/pkg/controller/loaders/remoteLoader.go
@@ -226,7 +226,7 @@ func (loader *RemoteLoader) ProcessExisting(process ProcessFunction, typeToProce
 func (loader *RemoteLoader) ProcessStream(process ProcessFunction, typeToProcess types.LoaderObjectType, timeoutSeconds time.Duration) (err error) {
 	done := make(chan error)
 	if timeoutSeconds != 0 {
-		time.AfterFunc(timeoutSeconds*time.Second, func() {
+		time.AfterFunc(timeoutSeconds, func() {
 			done <- fmt.Errorf("timeout")
 		})
 	}

diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go
@@ -53,7 +53,7 @@ const (
 	DefaultRegistryPort         = 5050
 
 	//tags, versions, repos
-	DefaultEVETag               = "11.7.0" // DefaultEVETag tag for EVE image
+	DefaultEVETag               = "12.4.0" // DefaultEVETag tag for EVE image
 	DefaultAdamTag              = "0.0.43"
 	DefaultRedisTag             = "7"
 	DefaultRegistryTag          = "2.7"

diff --git a/pkg/openevec/pod.go b/pkg/openevec/pod.go
@@ -559,8 +559,8 @@ func (openEVEC *OpenEVEC) PodModify(appName string, podNetworks, portPublish, ac
 			if needPurge {
 				processingFunction := func(im *info.ZInfoMsg) bool {
 					if im.Ztype == info.ZInfoTypes_ZiApp {
-						// waiting for purging state
-						if im.GetAinfo().State == info.ZSwState_PURGING {
+						// waiting for halting state
+						if im.GetAinfo().State == info.ZSwState_HALTING {
 							return true
 						}
 					}

diff --git a/tests/eclient/testdata/air-gapped-switch.txt b/tests/eclient/testdata/air-gapped-switch.txt
@@ -14,6 +14,12 @@
 [!exec:ssh] stop
 [!exec:chmod] stop
 
+# Application needs to be privileged (or run inside a VM) to be allowed to assign IP address
+# to an interface.
+exec -t 2m bash check_vm_support.sh
+source .env
+[!env:with_hw_virt] skip 'Missing HW-assisted virtualization capability'
+
 exec chmod 600 {{EdenConfig "eden.tests"}}/eclient/image/cert/id_rsa
 
 # Starting of reboot detector with 1 reboot limit
@@ -68,6 +74,21 @@ eden network ls
 ! stdout '^direct\s'
 ! stdout '^indirect\s'
 
+-- check_vm_support.sh --
+EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
+
+:>.env
+while true;
+do
+    virt=$($EDEN info --out InfoContent.dinfo.Capabilities.HWAssistedVirtualization | tail -n 1)
+    if [ -z "$virt" ]; then
+        sleep 3
+        continue
+    fi
+    [ "$virt" == "true" ] && echo "with_hw_virt=true" >>.env
+    break
+done
+
 -- wait_ssh.sh --
 EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
 for p in $*

diff --git a/tests/eclient/testdata/app_nonat.txt b/tests/eclient/testdata/app_nonat.txt
@@ -9,6 +9,12 @@
 [!exec:ssh] stop
 [!exec:chmod] stop
 
+# Changing Reverse-Path filtering to the Loose mode requires privileges which native
+# containers do not have.
+exec -t 2m bash check_vm_support.sh
+source .env
+[!env:with_hw_virt] skip 'Missing HW-assisted virtualization capability'
+
 exec chmod 600 {{EdenConfig "eden.tests"}}/eclient/image/cert/id_rsa
 
 # Starting of reboot detector with 1 reboot limit
@@ -65,6 +71,21 @@ eden network ls
 ! stdout '^direct\s'
 ! stdout '^indirect\s'
 
+-- check_vm_support.sh --
+EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
+
+:>.env
+while true;
+do
+    virt=$($EDEN info --out InfoContent.dinfo.Capabilities.HWAssistedVirtualization | tail -n 1)
+    if [ -z "$virt" ]; then
+        sleep 3
+        continue
+    fi
+    [ "$virt" == "true" ] && echo "with_hw_virt=true" >>.env
+    break
+done
+
 -- wait_ssh.sh --
 EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
 

diff --git a/tests/eclient/testdata/ctrl_cert_change.txt b/tests/eclient/testdata/ctrl_cert_change.txt
@@ -39,8 +39,7 @@ exec -t 2m bash check_sign_cert.sh
 test eden.reboot.test -test.v -timewait=20m -reboot=1 -count=1 &
 
 # wait for RUNNING state after reboot
-test eden.app.test -test.v -timewait 20m -check-new RUNNING eclient1
-test eden.app.test -test.v -timewait 20m -check-new RUNNING eclient2
+test eden.app.test -test.v -timewait 20m -check-new RUNNING eclient1 eclient2
 
 eden pod deploy -n eclient3 --memory=512MB --networks=n1 {{template "eclient_image"}} --metadata={{$userdata}}
 

diff --git a/tests/eclient/testdata/maridb.txt b/tests/eclient/testdata/maridb.txt
@@ -11,6 +11,12 @@
 [!exec:ssh] stop
 [!exec:chmod] stop
 
+# MariaDB crashes when run as a native container.
+# Most likely it expects some elevated privileges which bare containers do not have.
+exec -t 2m bash check_vm_support.sh
+source .env
+[!env:with_hw_virt] skip 'Missing HW-assisted virtualization capability'
+
 exec chmod 600 {{EdenConfig "eden.tests"}}/eclient/image/cert/id_rsa
 
 # Starting of reboot detector with a 1 reboot limit
@@ -44,6 +50,21 @@ eden pod delete {{$server}}
 
 test eden.app.test -test.v -timewait 10m - eclient {{$server}}
 
+-- check_vm_support.sh --
+EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
+
+:>.env
+while true;
+do
+    virt=$($EDEN info --out InfoContent.dinfo.Capabilities.HWAssistedVirtualization | tail -n 1)
+    if [ -z "$virt" ]; then
+        sleep 3
+        continue
+    fi
+    [ "$virt" == "true" ] && echo "with_hw_virt=true" >>.env
+    break
+done
+
 -- wait_ssh.sh --
 EDEN={{EdenConfig "eden.root"}}/{{EdenConfig "eden.bin-dist"}}/{{EdenConfig "eden.eden-bin"}}
 for i in `seq 20`

diff --git a/tests/eclient/testdata/nodered.txt b/tests/eclient/testdata/nodered.txt
@@ -12,7 +12,7 @@
 
 eden pod deploy -n nodered docker://nodered/node-red:2.1.3 -p {{template "port"}}:1880
 
-test eden.app.test -test.v -timewait 10m RUNNING nodered
+test eden.app.test -test.v -timewait 25m RUNNING nodered
 
 exec -t 10m bash wait_curl.sh