tests: benchmarks: Collect coverage data from multicore benchmarks

Add coverage calculation test case for each multicore idle* test Signed-off-by: Bartosz Miller <[email protected]>
nrfconnect · Jan 20, 2025 · 4065e8a · 4065e8a
1 parent 5e12525
commit 4065e8a
Show file tree

Hide file tree

Showing 38 changed files with 579 additions and 30 deletions.
diff --git a/tests/benchmarks/multicore/idle_adc/Kconfig.sysbuild b/tests/benchmarks/multicore/idle_adc/Kconfig.sysbuild
@@ -7,4 +7,5 @@
 source "${ZEPHYR_BASE}/share/sysbuild/Kconfig"
 
 config REMOTE_BOARD
-	string "The board used for remote target"
+	string
+	default "$(BOARD)/nrf54h20/cpurad" if SOC_NRF54H20_CPUAPP
diff --git a/tests/benchmarks/multicore/idle_adc/coverage.conf b/tests/benchmarks/multicore/idle_adc/coverage.conf
@@ -0,0 +1,20 @@
+CONFIG_ADC=y
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_adc/src/main.c b/tests/benchmarks/multicore/idle_adc/src/main.c
@@ -30,6 +30,7 @@ int main(void)
 	int err;
 	uint16_t channel_reading[5];
 	int16_t sample_value;
+	int test_repetitions = 3;
 
 	/* Options for the sequence sampling. */
 	const struct adc_sequence_options options = {
@@ -51,7 +52,13 @@ int main(void)
 	err = adc_setup();
 	__ASSERT_NO_MSG(err == 0);
 
-	while (1) {
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		gpio_pin_set_dt(&gpio, 1);
 		err = adc_read(adc, &sequence);
 		sample_value = channel_reading[0];
@@ -63,5 +70,8 @@ int main(void)
 		__ASSERT_NO_MSG(sample_value == ADC_LOW_LEVEL);
 		k_sleep(K_SECONDS(1));
 	}
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis start\n");
+#endif
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_adc/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf b/tests/benchmarks/multicore/idle_adc/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
diff --git a/tests/benchmarks/multicore/idle_adc/testcase.yaml b/tests/benchmarks/multicore/idle_adc/testcase.yaml
@@ -6,17 +6,36 @@ common:
     - ci_tests_benchmarks_multicore
     - adc
     - ppk_power_measure
+
 tests:
   benchmarks.multicore.idle_adc.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
     extra_args:
       - FILE_SUFFIX=s2ram
-      - SB_CONF_FILE=sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
     harness_config:
       fixture: gpio_loopback
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_adc"
+
+  benchmarks.multicore.idle_adc.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      fixture: gpio_loopback
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
diff --git a/tests/benchmarks/multicore/idle_comp/Kconfig.sysbuild b/tests/benchmarks/multicore/idle_comp/Kconfig.sysbuild
@@ -7,4 +7,5 @@
 source "${ZEPHYR_BASE}/share/sysbuild/Kconfig"
 
 config REMOTE_BOARD
-	string "The board used for remote target"
+	string
+	default "$(BOARD)/nrf54h20/cpurad" if SOC_NRF54H20_CPUAPP
diff --git a/tests/benchmarks/multicore/idle_comp/coverage.conf b/tests/benchmarks/multicore/idle_comp/coverage.conf
@@ -0,0 +1,20 @@
+CONFIG_COMPARATOR=y
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_comp/src/main.c b/tests/benchmarks/multicore/idle_comp/src/main.c
@@ -22,6 +22,7 @@ int main(void)
 {
 
 	int rc;
+	int test_repetitions = 3;
 
 	gpio_pin_configure_dt(&test_pin, GPIO_OUTPUT_INACTIVE);
 
@@ -34,7 +35,13 @@ int main(void)
 	rc = comparator_set_trigger(test_dev, COMPARATOR_TRIGGER_BOTH_EDGES);
 	__ASSERT_NO_MSG(rc == 0);
 
-	while (1) {
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		counter = 0;
 		k_msleep(200);
 		gpio_pin_set_dt(&test_pin, 1);
@@ -49,5 +56,9 @@ int main(void)
 		pm_device_runtime_get(test_dev);
 		k_msleep(1);
 	}
+
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis start\n");
+#endif
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_comp/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf b/tests/benchmarks/multicore/idle_comp/sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
diff --git a/tests/benchmarks/multicore/idle_comp/testcase.yaml b/tests/benchmarks/multicore/idle_comp/testcase.yaml
@@ -6,31 +6,50 @@ common:
     - ppk_power_measure
 tests:
   benchmarks.multicore.idle_comp.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
     extra_args:
       - FILE_SUFFIX=s2ram
-      - SB_CONF_FILE=sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
       - EXTRA_DTC_OVERLAY_FILE="boards/comp.overlay"
     harness_config:
       fixture: gpio_loopback
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_comp"
 
   benchmarks.multicore.idle_lpcomp.nrf54h20dk_cpuapp_cpurad.s2ram:
+    filter: not CONFIG_COVERAGE
     harness: pytest
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
     extra_args:
       - FILE_SUFFIX=s2ram
-      - SB_CONF_FILE=sysbuild/nrf54h20dk_nrf54h20_cpurad.conf
       - EXTRA_DTC_OVERLAY_FILE="boards/lpcomp.overlay"
     harness_config:
       fixture: gpio_loopback
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_power_consumption_lpcomp"
+
+  benchmarks.multicore.idle_lpcomp.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - EXTRA_DTC_OVERLAY_FILE="boards/lpcomp.overlay"
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      fixture: gpio_loopback
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
diff --git a/tests/benchmarks/multicore/idle_exmif/coverage.conf b/tests/benchmarks/multicore/idle_exmif/coverage.conf
@@ -0,0 +1,19 @@
+CONFIG_MSPI=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_exmif/src/main.c b/tests/benchmarks/multicore/idle_exmif/src/main.c
@@ -160,6 +160,7 @@ static void configure_exmif_for_test(const struct device *controller, struct msp
 int main(void)
 {
 	int ret;
+	int test_repetitions = 3;
 
 	printk("Multicore idle exmif test on %s\n", CONFIG_BOARD_TARGET);
 	k_msleep(10);
@@ -172,9 +173,15 @@ int main(void)
 	__ASSERT_NO_MSG(ret == 0);
 
 	configure_exmif_for_test(controller, &dev_id);
-
 	k_msleep(500);
-	while (1) {
+
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		printk("Wake up\n");
 		for (int counter = 0; counter < NUMBER_OF_TEST_READS; counter++) {
 			read_jedec_id(controller, &dev_id);
@@ -186,5 +193,8 @@ int main(void)
 		gpio_pin_set_dt(&led, 1);
 	}
 
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis start\n");
+#endif
 	return 0;
 }
diff --git a/tests/benchmarks/multicore/idle_exmif/testcase.yaml b/tests/benchmarks/multicore/idle_exmif/testcase.yaml
@@ -1,7 +1,6 @@
 common:
   sysbuild: true
   depends_on: gpio
-  harness: pytest
   tags:
     - ci_build
     - ci_tests_benchmarks_multicore
@@ -10,11 +9,30 @@ common:
 
 tests:
   benchmarks.multicore.idle_exmif.nrf54h20dk_cpuapp_cpurad:
+    filter: not CONFIG_COVERAGE
     platform_allow:
       - nrf54h20dk/nrf54h20/cpuapp
     integration_platforms:
       - nrf54h20dk/nrf54h20/cpuapp
+    harness: pytest
     harness_config:
       fixture: ppk_power_measure
       pytest_root:
         - "${CUSTOM_ROOT_TEST_DIR}/test_measure_power_consumption.py::test_measure_and_data_dump_exmif_and_s2ram"
+
+  benchmarks.multicore.idle_exmif.nrf54h20dk_cpuapp_cpurad.coverage:
+    filter: CONFIG_COVERAGE
+    platform_allow:
+      - nrf54h20dk/nrf54h20/cpuapp
+    integration_platforms:
+      - nrf54h20dk/nrf54h20/cpuapp
+    extra_args:
+      - CONF_FILE=coverage.conf
+      - SHIELD=coverage_support
+    harness: console
+    harness_config:
+      type: multi_line
+      ordered: true
+      regex:
+        - ".*Coverage analysis enabled.*"
+        - ".*Coverage analysis start.*"
diff --git a/tests/benchmarks/multicore/idle_gpio/coverage.conf b/tests/benchmarks/multicore/idle_gpio/coverage.conf
@@ -0,0 +1,19 @@
+CONFIG_GPIO=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_ipc/coverage.conf b/tests/benchmarks/multicore/idle_ipc/coverage.conf
@@ -0,0 +1,21 @@
+CONFIG_HEAP_MEM_POOL_SIZE=4096
+CONFIG_MBOX=y
+CONFIG_IPC_SERVICE=y
+
+CONFIG_PM=y
+CONFIG_PM_S2RAM=n
+CONFIG_PM_S2RAM_CUSTOM_MARKING=n
+CONFIG_PM_DEVICE=y
+CONFIG_PM_DEVICE_RUNTIME=y
+CONFIG_POWEROFF=y
+
+CONFIG_BOOT_BANNER=n
+CONFIG_ASSERT=n
+CONFIG_CLOCK_CONTROL=n
+
+CONFIG_PRINTK=y
+CONFIG_LOG=n
+CONFIG_CONSOLE=y
+CONFIG_UART_CONSOLE=y
+
+CONFIG_LOG_BUFFER_SIZE=16384
diff --git a/tests/benchmarks/multicore/idle_ipc/src/main.c b/tests/benchmarks/multicore/idle_ipc/src/main.c
@@ -72,6 +72,7 @@ int main(void)
 	int ret;
 	unsigned long last_cnt = 0;
 	unsigned long delta = 0;
+	int test_repetitions = 3;
 
 	p_payload = (struct payload *)k_malloc(CONFIG_APP_IPC_SERVICE_MESSAGE_LEN);
 	if (!p_payload) {
@@ -112,8 +113,13 @@ int main(void)
 	}
 	k_msleep(CONFIG_TEST_START_DELAY_MS);
 
-	while (true) {
-
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis enabled\n");
+	while (test_repetitions--)
+#else
+	while (test_repetitions)
+#endif
+	{
 		printk("Hello\n");
 		printk("Send data over IPC\n");
 
@@ -140,5 +146,8 @@ int main(void)
 		k_msleep(CONFIG_SLEEP_TIME_MS);
 	}
 
+#if defined(CONFIG_COVERAGE)
+	printk("Coverage analysis start\n");
+#endif
 	return 0;
 }