tests: benchmarks: Collect coverage data from multicore benchmarks

Add coverage calculation test case for each multicore idle* test Signed-off-by: Bartosz Miller <[email protected]>
nrfconnect · Jan 17, 2025 · 4ac078c · 4ac078c
1 parent f225bde
commit 4ac078c
Show file tree

Hide file tree

Showing 33 changed files with 509 additions and 27 deletions.
diff --git a/tests/benchmarks/multicore/idle_adc/Kconfig.sysbuild b/tests/benchmarks/multicore/idle_adc/Kconfig.sysbuild
@@ -7,4 +7,5 @@
 source "${ZEPHYR_BASE}/share/sysbuild/Kconfig"
 
 config REMOTE_BOARD
-	string "The board used for remote target"
+	string
+	default "$(BOARD)/nrf54h20/cpurad" if SOC_NRF54H20_CPUAPP
diff --git a/tests/benchmarks/multicore/idle_adc/coverage.conf b/tests/benchmarks/multicore/idle_adc/coverage.conf
@@ -0,0 +1,20 @@
+CONFIG_ADC=y
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_adc/src/main.c b/tests/benchmarks/multicore/idle_adc/src/main.c
@@ -28,6 +28,7 @@ DT_FOREACH_CHILD_SEP(ADC_NODE, ADC_CHANNEL_CFG_DT, (,))};
 int main(void)
 {
 	int err;
+	int test_repetitions = 3;
 	uint16_t channel_reading[5];
 	int16_t sample_value;
 
@@ -51,7 +52,13 @@ int main(void)
 	err = adc_setup();
 	__ASSERT_NO_MSG(err == 0);
 
-	while (1) {
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		gpio_pin_set_dt(&gpio, 1);
 		err = adc_read(adc, &sequence);
 		sample_value = channel_reading[0];
@@ -63,5 +70,7 @@ int main(void)
 		__ASSERT_NO_MSG(sample_value == ADC_LOW_LEVEL);
 		k_sleep(K_SECONDS(1));
 	}
+
+	printk("Coverage analysis start\n");
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_adc/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf b/tests/benchmarks/multicore/idle_adc/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
diff --git a/tests/benchmarks/multicore/idle_adc/testcase.yaml b/tests/benchmarks/multicore/idle_adc/testcase.yaml
@@ -6,17 +6,37 @@ common:
     - ci_tests_benchmarks_multicore
     - adc
     - ppk_power_measure
+
 tests:
   benchmarks.multicore.idle_adc.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
     extra_args:
       - FILE_SUFFIX=s2ram
-      - SB_CONF_FILE=sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
     harness_config:
       fixture: gpio_loopback
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_adc"
+
+  benchmarks.multicore.idle_adc.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
+
+
diff --git a/tests/benchmarks/multicore/idle_exmif/coverage.conf b/tests/benchmarks/multicore/idle_exmif/coverage.conf
@@ -0,0 +1,19 @@
+CONFIG_MSPI=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_exmif/src/main.c b/tests/benchmarks/multicore/idle_exmif/src/main.c
@@ -160,6 +160,7 @@ static void configure_exmif_for_test(const struct device *controller, struct msp
 int main(void)
 {
 	int ret;
+	int test_repetitions = 3;
 
 	printk("Multicore idle exmif test on %s\n", CONFIG_BOARD_TARGET);
 	k_msleep(10);
@@ -172,9 +173,15 @@ int main(void)
 	__ASSERT_NO_MSG(ret == 0);
 
 	configure_exmif_for_test(controller, &dev_id);
-
 	k_msleep(500);
-	while (1) {
+
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		printk("Wake up\n");
 		for (int counter = 0; counter < NUMBER_OF_TEST_READS; counter++) {
 			read_jedec_id(controller, &dev_id);
@@ -186,5 +193,6 @@ int main(void)
 		gpio_pin_set_dt(&led, 1);
 	}
 
+	printk("Coverage analysis start\n");
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_exmif/testcase.yaml b/tests/benchmarks/multicore/idle_exmif/testcase.yaml
@@ -1,7 +1,6 @@
 common:
   sysbuild: true
   depends_on: gpio
-  harness: pytest
   tags:
     - ci_build
     - ci_tests_benchmarks_multicore
@@ -10,11 +9,30 @@ common:
 
 tests:
   benchmarks.multicore.idle_exmif.nrf54h20dk_cpuapp_cpurad:
+    filter: not CONFIG_COVERAGE
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
+    harness: pytest
     harness_config:
       fixture: ppk_power_measure
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_exmif_and_s2ram"
+
+  benchmarks.multicore.idle_exmif.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
diff --git a/tests/benchmarks/multicore/idle_gpio/coverage.conf b/tests/benchmarks/multicore/idle_gpio/coverage.conf
@@ -0,0 +1,19 @@
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_ipc/coverage.conf b/tests/benchmarks/multicore/idle_ipc/coverage.conf
@@ -0,0 +1,21 @@
+CONFIG_HEAP_MEM_POOL_SIZE=4096
+CONFIG_MBOX=y
+CONFIG_IPC_SERVICE=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_ipc/src/main.c b/tests/benchmarks/multicore/idle_ipc/src/main.c
@@ -72,6 +72,7 @@ int main(void)
 	int ret;
 	unsigned long last_cnt = 0;
 	unsigned long delta = 0;
+	int test_repetitions = 3;
 
 	p_payload = (struct payload *)k_malloc(CONFIG_APP_IPC_SERVICE_MESSAGE_LEN);
 	if (!p_payload) {
@@ -112,8 +113,13 @@ int main(void)
 	}
 	k_msleep(CONFIG_TEST_START_DELAY_MS);
 
-	while (true) {
-
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		printk("Hello\n");
 		printk("Send data over IPC\n");
 
@@ -139,6 +145,7 @@ int main(void)
 		printk("Go to sleep (s2ram)\n");
 		k_msleep(CONFIG_SLEEP_TIME_MS);
 	}
-
+
+	printk("Coverage analysis start\n");
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_ipc/testcase.yaml b/tests/benchmarks/multicore/idle_ipc/testcase.yaml
@@ -7,6 +7,7 @@ common:
     - ppk_power_measure
 tests:
   benchmarks.multicore.idle_ipc.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
@@ -18,6 +19,7 @@ tests:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_ipc"
 
   benchmarks.multicore.idle_ipc.shifted.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
@@ -29,3 +31,20 @@ tests:
       fixture: ppk_power_measure
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_ipc"
+
+  benchmarks.multicore.idle_ipc.shifted.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
diff --git a/tests/benchmarks/multicore/idle_pwm_led/coverage.conf b/tests/benchmarks/multicore/idle_pwm_led/coverage.conf
@@ -0,0 +1,20 @@
+CONFIG_PWM=y
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_pwm_led/src/main.c b/tests/benchmarks/multicore/idle_pwm_led/src/main.c
@@ -29,6 +29,7 @@ int main(void)
 	uint32_t pulse_max;
 	int32_t pulse_step;
 	uint32_t current_pulse_width;
+	int test_repetitions = 3;
 
 	if (!gpio_is_ready_dt(&led)) {
 		LOG_ERR("GPIO Device not ready");
@@ -75,7 +76,13 @@ int main(void)
 		pm_device_runtime_enable(pwm_led.dev);
 #endif
 
-	while (1) {
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		LOG_INF("Multicore idle_pwm_led test iteration %u", cnt++);
 
 #if defined(CONFIG_PM_DEVICE_RUNTIME)
@@ -120,5 +127,6 @@ int main(void)
 		gpio_pin_set_dt(&led, 1);
 	}
 
+	printk("Coverage analysis start\n");
 	return 0;
 }