From 10edb1cb9d0a5d65cc22e87c8b7790b495e32936 Mon Sep 17 00:00:00 2001 From: Lars Asplund Date: Wed, 1 Jan 2025 23:01:09 +0100 Subject: [PATCH] Added support for dynamic seed generation. --- docs/run/img/get_seed_and_runner_cfg.html | 2 + docs/run/img/get_seed_wo_runner_cfg.html | 5 + docs/run/img/get_uniform_seed.html | 5 + docs/run/img/list.html | 8 +- docs/run/img/seed_option.html | 1 + docs/run/img/tb_magic_paths_stdout.html | 9 +- docs/run/img/tb_minimal_stdout.html | 8 +- docs/run/img/tb_running_test_case_stdout.html | 20 ++-- docs/run/img/tb_stop_level_stdout.html | 19 ++-- docs/run/img/tb_stopping_failure_stdout.html | 43 +++---- .../tb_with_lower_level_control_stdout.html | 14 +-- docs/run/img/tb_with_test_cases_stdout.html | 14 +-- docs/run/img/tb_with_watchdog_stdout.html | 34 +++--- docs/run/src/run.py | 20 ++++ docs/run/src/tb_seed.vhd | 59 ++++++++++ docs/run/user_guide.rst | 58 +++++++++- examples/vhdl/run/run.py | 14 +-- tests/unit/test_test_bench.py | 2 +- tests/unit/test_test_suites.py | 2 + vunit/sim_if/factory.py | 3 +- vunit/test/bench.py | 9 +- vunit/test/bench_list.py | 4 +- vunit/test/suites.py | 24 +++- vunit/ui/__init__.py | 2 +- vunit/vhdl/run/src/run.vhd | 106 +++++++++++++++++- vunit/vhdl/run/src/run_api.vhd | 13 +++ vunit/vhdl/run/src/run_types.vhd | 4 + vunit/vhdl/run/src/runner_pkg.vhd | 17 ++- vunit/vhdl/run/test/run_tests.vhd | 98 +++++++++++++++- vunit/vhdl/string_ops/src/string_ops.vhd | 37 ++++++ vunit/vhdl/string_ops/test/tb_string_ops.vhd | 9 ++ vunit/vunit_cli.py | 6 + 32 files changed, 559 insertions(+), 110 deletions(-) create mode 100644 docs/run/img/get_seed_and_runner_cfg.html create mode 100644 docs/run/img/get_seed_wo_runner_cfg.html create mode 100644 docs/run/img/get_uniform_seed.html create mode 100644 docs/run/img/seed_option.html create mode 100644 docs/run/src/tb_seed.vhd diff --git a/docs/run/img/get_seed_and_runner_cfg.html b/docs/run/img/get_seed_and_runner_cfg.html new file mode 100644 index 000000000..63c2e78cd --- /dev/null +++ b/docs/run/img/get_seed_and_runner_cfg.html @@ -0,0 +1,2 @@ +
constant global_seed : string := get_seed(runner_cfg, "Optional salt");
+
diff --git a/docs/run/img/get_seed_wo_runner_cfg.html b/docs/run/img/get_seed_wo_runner_cfg.html new file mode 100644 index 000000000..5c56cc549 --- /dev/null +++ b/docs/run/img/get_seed_wo_runner_cfg.html @@ -0,0 +1,5 @@ +
randomizing_process : process is
+  variable local_seed : string_seed_t; -- string_seed_t = string(1 to 16)
+begin
+  get_seed(local_seed, salt => randomizing_process'path_name);
+
diff --git a/docs/run/img/get_uniform_seed.html b/docs/run/img/get_uniform_seed.html new file mode 100644 index 000000000..c4842b882 --- /dev/null +++ b/docs/run/img/get_uniform_seed.html @@ -0,0 +1,5 @@ +
get_uniform_seed(seed1, seed2, "Optional salt");
+
+uniform(seed1, seed2, a_random_value);
+uniform(seed1, seed2, another_random_value);
+
diff --git a/docs/run/img/list.html b/docs/run/img/list.html index 9b98fcbae..bed81644a 100644 --- a/docs/run/img/list.html +++ b/docs/run/img/list.html @@ -1,4 +1,8 @@ 
> python run.py --list
+lib.tb_fail_on_warning.Test that fails on an assert
+lib.tb_fail_on_warning.Test that crashes on boundary problems
+lib.tb_fail_on_warning.Test that fails on VUnit check procedure
+lib.tb_fail_on_warning.Test that a warning passes
 lib.tb_magic_paths.all
 lib.tb_minimal.all
 lib.tb_running_test_case.Test scenario A
@@ -6,11 +10,13 @@
 lib.tb_running_test_case.Test something else
 lib.tb_run_all_in_same_sim.Test to_string for integer again
 lib.tb_run_all_in_same_sim.Test to_string for boolean again
+lib.tb_seed.all
 lib.tb_standalone.Test that fails on VUnit check procedure
 lib.tb_standalone.Test to_string for boolean
 lib.tb_stopping_failure.Test that fails on an assert
 lib.tb_stopping_failure.Test that crashes on boundary problems
 lib.tb_stopping_failure.Test that fails on VUnit check procedure
+lib.tb_stopping_failure.Test that a warning passes
 lib.tb_stop_level.Test that fails multiple times but doesn't stop
 lib.tb_with_lower_level_control.Test something
 lib.tb_with_lower_level_control.Test something else
@@ -19,5 +25,5 @@
 lib.tb_with_watchdog.Test that stalls
 lib.tb_with_watchdog.Test to_string for boolean
 lib.tb_with_watchdog.Test that needs longer timeout
-Listed 20 tests
+Listed 26 tests
diff --git a/docs/run/img/seed_option.html b/docs/run/img/seed_option.html new file mode 100644 index 000000000..9a02b5673 --- /dev/null +++ b/docs/run/img/seed_option.html @@ -0,0 +1 @@ +
> python run.py "lib.tb_seed.Test that fails" --seed fb19f3cca859d69c
diff --git a/docs/run/img/tb_magic_paths_stdout.html b/docs/run/img/tb_magic_paths_stdout.html index ae539385a..68af5fe36 100644 --- a/docs/run/img/tb_magic_paths_stdout.html +++ b/docs/run/img/tb_magic_paths_stdout.html @@ -1,18 +1,19 @@ 
> python run.py -v
 Starting lib.tb_magic_paths.all
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_magic_paths.all_243b3c717ce1d4e82490245d1b7e8fe8797f5e94\output.txt
+Seed for lib.tb_magic_paths.all: 3680f2e2321cdac3
                0 fs - default              -    INFO - Directory containing testbench: C:/github/vunit/docs/run/src/
                0 fs - default              -    INFO - Test output directory: C:/github/vunit/docs/run/src/vunit_out/test_output/lib.tb_magic_paths.all_243b3c717ce1d4e82490245d1b7e8fe8797f5e94/
 simulation stopped @0ms with status 0
-pass (P=1 S=0 F=0 T=1) lib.tb_magic_paths.all (0.5 seconds)
+pass (P=1 S=0 F=0 T=1) lib.tb_magic_paths.all (0.6 s)
 
 ==== Summary ==================================
-pass lib.tb_magic_paths.all (0.5 seconds)
+pass lib.tb_magic_paths.all (0.6 s)
 ===============================================
 pass 1 of 1
 ===============================================
-Total time was 0.5 seconds
-Elapsed time was 0.5 seconds
+Total time was 0.6 s
+Elapsed time was 0.6 s
 ===============================================
 All passed!
diff --git a/docs/run/img/tb_minimal_stdout.html b/docs/run/img/tb_minimal_stdout.html index 36c933ffa..33b883caf 100644 --- a/docs/run/img/tb_minimal_stdout.html +++ b/docs/run/img/tb_minimal_stdout.html @@ -1,15 +1,15 @@ 
> python run.py
 Starting lib.tb_minimal.all
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_minimal.all_42aa262c7c96c708ab3f3960f033f2328c642136\output.txt
-pass (P=1 S=0 F=0 T=1) lib.tb_minimal.all (0.5 seconds)
+pass (P=1 S=0 F=0 T=1) lib.tb_minimal.all (0.6 s)
 
 ==== Summary ==============================
-pass lib.tb_minimal.all (0.5 seconds)
+pass lib.tb_minimal.all (0.6 s)
 ===========================================
 pass 1 of 1
 ===========================================
-Total time was 0.5 seconds
-Elapsed time was 0.5 seconds
+Total time was 0.6 s
+Elapsed time was 0.6 s
 ===========================================
 All passed!
diff --git a/docs/run/img/tb_running_test_case_stdout.html b/docs/run/img/tb_running_test_case_stdout.html index 11b105f58..3ed29dc13 100644 --- a/docs/run/img/tb_running_test_case_stdout.html +++ b/docs/run/img/tb_running_test_case_stdout.html @@ -1,25 +1,25 @@ 
> python run.py
 Starting lib.tb_running_test_case.Test scenario A
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_running_test_case.Test_scenario_A_b118fcdf82c6ba5772e038ce7455962692f50c2a\output.txt
-pass (P=1 S=0 F=0 T=3) lib.tb_running_test_case.Test scenario A (0.5 seconds)
+pass (P=1 S=0 F=0 T=3) lib.tb_running_test_case.Test scenario A (0.6 s)
 
-Starting lib.tb_running_test_case.Test scenario B
+(22:55:49) Starting lib.tb_running_test_case.Test scenario B
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_running_test_case.Test_scenario_B_8cd407ec92589901f9b4594c0f27835478242f2c\output.txt
-pass (P=2 S=0 F=0 T=3) lib.tb_running_test_case.Test scenario B (0.5 seconds)
+pass (P=2 S=0 F=0 T=3) lib.tb_running_test_case.Test scenario B (0.6 s)
 
-Starting lib.tb_running_test_case.Test something else
+(22:55:50) Starting lib.tb_running_test_case.Test something else
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_running_test_case.Test_something_else_27dcc1aa8d44993b6b2d0b0a017fa6001b4c2aa7\output.txt
-pass (P=3 S=0 F=0 T=3) lib.tb_running_test_case.Test something else (0.5 seconds)
+pass (P=3 S=0 F=0 T=3) lib.tb_running_test_case.Test something else (0.6 s)
 
 ==== Summary ========================================================
-pass lib.tb_running_test_case.Test scenario A     (0.5 seconds)
-pass lib.tb_running_test_case.Test scenario B     (0.5 seconds)
-pass lib.tb_running_test_case.Test something else (0.5 seconds)
+pass lib.tb_running_test_case.Test scenario A     (0.6 s)
+pass lib.tb_running_test_case.Test scenario B     (0.6 s)
+pass lib.tb_running_test_case.Test something else (0.6 s)
 =====================================================================
 pass 3 of 3
 =====================================================================
-Total time was 1.6 seconds
-Elapsed time was 1.6 seconds
+Total time was 1.8 s
+Elapsed time was 1.8 s
 =====================================================================
 All passed!
diff --git a/docs/run/img/tb_stop_level_stdout.html b/docs/run/img/tb_stop_level_stdout.html index 6bd1ac5d1..6d8548bee 100644 --- a/docs/run/img/tb_stop_level_stdout.html +++ b/docs/run/img/tb_stop_level_stdout.html @@ -1,26 +1,23 @@ 
> python run.py
 Starting lib.tb_stop_level.Test that fails multiple times but doesn't stop
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stop_level.Test_that_fails_multiple_times_but_doesn't_stop_d08f48d859442d0bc71e2bcdd8b429119f7cc17c\output.txt
+Seed for lib.tb_stop_level.Test that fails multiple times but doesn't stop: 7a0e979de335b966
                0 fs - check                -   ERROR - Equality check failed - Got 17. Expected 18.
                0 fs - check                -   ERROR - Equality check failed - Got 17. Expected 19.
 FAILURE - Logger check has 2 errors
-C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:84:7:@0ms:(report failure): Final log check failed
-C:\ghdl\bin\ghdl.exe:error: report failed
-in process .tb_stop_level(tb).test_runner
-  from: vunit_lib.logger_pkg.final_log_check at logger_pkg-body.vhd:1249
-  from: vunit_lib.run_pkg.test_runner_cleanup at run.vhd:114
-  from: process lib.tb_stop_level(tb).test_runner at tb_stop_level.vhd:29
-C:\ghdl\bin\ghdl.exe:error: simulation failed
-fail (P=0 S=0 F=1 T=1) lib.tb_stop_level.Test that fails multiple times but doesn't stop (0.5 seconds)
+C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@0ms:(report failure): Final log check failed
+ghdl:error: report failed
+ghdl:error: simulation failed
+fail (P=0 S=0 F=1 T=1) lib.tb_stop_level.Test that fails multiple times but doesn't stop (0.6 s)
 
 ==== Summary =============================================================================
-fail lib.tb_stop_level.Test that fails multiple times but doesn't stop (0.5 seconds)
+fail lib.tb_stop_level.Test that fails multiple times but doesn't stop (0.6 s)
 ==========================================================================================
 pass 0 of 1
 fail 1 of 1
 ==========================================================================================
-Total time was 0.5 seconds
-Elapsed time was 0.5 seconds
+Total time was 0.6 s
+Elapsed time was 0.6 s
 ==========================================================================================
 Some failed!
diff --git a/docs/run/img/tb_stopping_failure_stdout.html b/docs/run/img/tb_stopping_failure_stdout.html index 63d7964a7..2bf20d824 100644 --- a/docs/run/img/tb_stopping_failure_stdout.html +++ b/docs/run/img/tb_stopping_failure_stdout.html @@ -1,40 +1,43 @@ 
> python run.py
 Starting lib.tb_stopping_failure.Test that fails on an assert
-Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
-C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
+Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_an_assert_f53b930e2c7649bc33253af52f8ea89a9c05f07b\output.txt
+Seed for lib.tb_stopping_failure.Test that fails on an assert: a866d0986a663e6b
+C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:24:9:@0ms:(assertion error): Assertion violation
 ghdl:error: assertion failed
 ghdl:error: simulation failed
-fail (P=0 S=0 F=1 T=4) lib.tb_stopping_failure.Test that fails on an assert (0.5 s)
+fail (P=0 S=0 F=1 T=4) lib.tb_stopping_failure.Test that fails on an assert (0.6 s)
 
-(11:35:31) Starting lib.tb_stopping_failure.Test that crashes on boundary problems
-Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
-ghdl:error: index (314) out of bounds (1 to 17) at C:\repos\vunit\docs\run\src\tb_stopping_failure.vhd:26
+(22:55:55) Starting lib.tb_stopping_failure.Test that crashes on boundary problems
+Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_crashes_on_boundary_problems_b53105615efefaa16d0cf9ee1bad37b5d3369e95\output.txt
+Seed for lib.tb_stopping_failure.Test that crashes on boundary problems: cbcb543764149e78
+ghdl:error: index (340) out of bounds (1 to 17) at C:\github\vunit\docs\run\src\tb_stopping_failure.vhd:26
 ghdl:error: simulation failed
-fail (P=0 S=0 F=2 T=4) lib.tb_stopping_failure.Test that crashes on boundary problems (0.5 s)
+fail (P=0 S=0 F=2 T=4) lib.tb_stopping_failure.Test that crashes on boundary problems (0.6 s)
 
-(11:35:31) Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
-Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
+(22:55:55) Starting lib.tb_stopping_failure.Test that fails on VUnit check procedure
+Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_fails_on_VUnit_check_procedure_717a6f8ff044e3d5fa7d7d3ec5a32971d74864dd\output.txt
+Seed for lib.tb_stopping_failure.Test that fails on VUnit check procedure: 6581894e9ea614b1
                0 fs - check                -   ERROR - Equality check failed - Got 17. Expected 18.
-C:\repos\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@0ms:(report failure): Stop simulation on log level error
+C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@0ms:(report failure): Stop simulation on log level error
 ghdl:error: report failed
 ghdl:error: simulation failed
-fail (P=0 S=0 F=3 T=4) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)
+fail (P=0 S=0 F=3 T=4) lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.6 s)
 
-(11:35:32) Starting lib.tb_stopping_failure.Test that a warning passes
-Output file: C:\repos\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_a_warning_passes_7db91f3b27aea5f89e74e39ea51ce6d61558674e\output.txt
-pass (P=1 S=0 F=3 T=4) lib.tb_stopping_failure.Test that a warning passes (0.4 s)
+(22:55:56) Starting lib.tb_stopping_failure.Test that a warning passes
+Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_stopping_failure.Test_that_a_warning_passes_7db91f3b27aea5f89e74e39ea51ce6d61558674e\output.txt
+pass (P=1 S=0 F=3 T=4) lib.tb_stopping_failure.Test that a warning passes (0.6 s)
 
 ==== Summary ============================================================================
-pass lib.tb_stopping_failure.Test that a warning passes               (0.4 s)
-fail lib.tb_stopping_failure.Test that fails on an assert             (0.5 s)
-fail lib.tb_stopping_failure.Test that crashes on boundary problems   (0.5 s)
-fail lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.5 s)
+pass lib.tb_stopping_failure.Test that a warning passes               (0.6 s)
+fail lib.tb_stopping_failure.Test that fails on an assert             (0.6 s)
+fail lib.tb_stopping_failure.Test that crashes on boundary problems   (0.6 s)
+fail lib.tb_stopping_failure.Test that fails on VUnit check procedure (0.6 s)
 =========================================================================================
 pass 1 of 4
 fail 3 of 4
 =========================================================================================
-Total time was 1.8 s
-Elapsed time was 1.8 s
+Total time was 2.4 s
+Elapsed time was 2.4 s
 =========================================================================================
 Some failed!
diff --git a/docs/run/img/tb_with_lower_level_control_stdout.html b/docs/run/img/tb_with_lower_level_control_stdout.html index 2593def3b..1225640e3 100644 --- a/docs/run/img/tb_with_lower_level_control_stdout.html +++ b/docs/run/img/tb_with_lower_level_control_stdout.html @@ -1,20 +1,20 @@ 
> python run.py
 Starting lib.tb_with_lower_level_control.Test something
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_lower_level_control.Test_something_a280708dc2f527dff5e45e7a7d2b48df39330b4f\output.txt
-pass (P=1 S=0 F=0 T=2) lib.tb_with_lower_level_control.Test something (0.5 seconds)
+pass (P=1 S=0 F=0 T=2) lib.tb_with_lower_level_control.Test something (0.6 s)
 
-Starting lib.tb_with_lower_level_control.Test something else
+(22:55:51) Starting lib.tb_with_lower_level_control.Test something else
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_lower_level_control.Test_something_else_e47dc199cab8c612d9a0f46b8be7d141576fc970\output.txt
-pass (P=2 S=0 F=0 T=2) lib.tb_with_lower_level_control.Test something else (0.5 seconds)
+pass (P=2 S=0 F=0 T=2) lib.tb_with_lower_level_control.Test something else (0.6 s)
 
 ==== Summary ===============================================================
-pass lib.tb_with_lower_level_control.Test something      (0.5 seconds)
-pass lib.tb_with_lower_level_control.Test something else (0.5 seconds)
+pass lib.tb_with_lower_level_control.Test something      (0.6 s)
+pass lib.tb_with_lower_level_control.Test something else (0.6 s)
 ============================================================================
 pass 2 of 2
 ============================================================================
-Total time was 1.1 seconds
-Elapsed time was 1.1 seconds
+Total time was 1.2 s
+Elapsed time was 1.2 s
 ============================================================================
 All passed!
diff --git a/docs/run/img/tb_with_test_cases_stdout.html b/docs/run/img/tb_with_test_cases_stdout.html index 8fc2ae2f8..ea9f2d12c 100644 --- a/docs/run/img/tb_with_test_cases_stdout.html +++ b/docs/run/img/tb_with_test_cases_stdout.html @@ -1,20 +1,20 @@ 
> python run.py
 Starting lib.tb_with_test_cases.Test to_string for integer
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_test_cases.Test_to_string_for_integer_f5d39e15e865eddcda2b57f65dddc2428c994af4\output.txt
-pass (P=1 S=0 F=0 T=2) lib.tb_with_test_cases.Test to_string for integer (0.6 seconds)
+pass (P=1 S=0 F=0 T=2) lib.tb_with_test_cases.Test to_string for integer (0.6 s)
 
-Starting lib.tb_with_test_cases.Test to_string for boolean
+(22:55:48) Starting lib.tb_with_test_cases.Test to_string for boolean
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_test_cases.Test_to_string_for_boolean_38c3f897030cff968430d763a9bbc23202de1a7b\output.txt
-pass (P=2 S=0 F=0 T=2) lib.tb_with_test_cases.Test to_string for boolean (0.5 seconds)
+pass (P=2 S=0 F=0 T=2) lib.tb_with_test_cases.Test to_string for boolean (0.6 s)
 
 ==== Summary =============================================================
-pass lib.tb_with_test_cases.Test to_string for integer (0.6 seconds)
-pass lib.tb_with_test_cases.Test to_string for boolean (0.5 seconds)
+pass lib.tb_with_test_cases.Test to_string for integer (0.6 s)
+pass lib.tb_with_test_cases.Test to_string for boolean (0.6 s)
 ==========================================================================
 pass 2 of 2
 ==========================================================================
-Total time was 1.1 seconds
-Elapsed time was 1.1 seconds
+Total time was 1.2 s
+Elapsed time was 1.2 s
 ==========================================================================
 All passed!
diff --git a/docs/run/img/tb_with_watchdog_stdout.html b/docs/run/img/tb_with_watchdog_stdout.html index b3352ae40..1bf9c79d1 100644 --- a/docs/run/img/tb_with_watchdog_stdout.html +++ b/docs/run/img/tb_with_watchdog_stdout.html @@ -1,37 +1,31 @@ 
> python run.py
 Starting lib.tb_with_watchdog.Test that stalls
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_watchdog.Test_that_stalls_7f50c5908f9e9f9df5075e065f984eef1c2f7b2b\output.txt
+Seed for lib.tb_with_watchdog.Test that stalls: abf6f593955b8927
   10000000000000 fs - runner               -   ERROR - Test runner timeout after 10000000000000 fs.
-C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:84:7:@10ms:(report failure): Stop simulation on log level error
-C:\ghdl\bin\ghdl.exe:error: report failed
-in process .tb_with_watchdog(tb).P0
-  from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:736
-  from: vunit_lib.logger_pkg.decrease_stop_count at logger_pkg-body.vhd:731
-  from: vunit_lib.logger_pkg.log at logger_pkg-body.vhd:910
-  from: vunit_lib.logger_pkg.error at logger_pkg-body.vhd:969
-  from: vunit_lib.run_pkg.test_runner_watchdog at run.vhd:308
-  from: process lib.tb_with_watchdog(tb).P0 at tb_with_watchdog.vhd:37
-C:\ghdl\bin\ghdl.exe:error: simulation failed
-fail (P=0 S=0 F=1 T=3) lib.tb_with_watchdog.Test that stalls (0.5 seconds)
+C:\github\vunit\vunit\vhdl\core\src\core_pkg.vhd:85:7:@10ms:(report failure): Stop simulation on log level error
+ghdl:error: report failed
+ghdl:error: simulation failed
+fail (P=0 S=0 F=1 T=3) lib.tb_with_watchdog.Test that stalls (0.6 s)
 
-Starting lib.tb_with_watchdog.Test to_string for boolean
+(22:55:53) Starting lib.tb_with_watchdog.Test to_string for boolean
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_watchdog.Test_to_string_for_boolean_f167e524924d51144c5a6913c63e9fa5c6c7988c\output.txt
-pass (P=1 S=0 F=1 T=3) lib.tb_with_watchdog.Test to_string for boolean (0.5 seconds)
+pass (P=1 S=0 F=1 T=3) lib.tb_with_watchdog.Test to_string for boolean (0.6 s)
 
-Starting lib.tb_with_watchdog.Test that needs longer timeout
+(22:55:53) Starting lib.tb_with_watchdog.Test that needs longer timeout
 Output file: C:\github\vunit\docs\run\src\vunit_out\test_output\lib.tb_with_watchdog.Test_that_needs_longer_timeout_5494104827c61d0022a75acbab4c0c6de9e29643\output.txt
-pass (P=2 S=0 F=1 T=3) lib.tb_with_watchdog.Test that needs longer timeout (0.5 seconds)
+pass (P=2 S=0 F=1 T=3) lib.tb_with_watchdog.Test that needs longer timeout (0.6 s)
 
 ==== Summary ===============================================================
-pass lib.tb_with_watchdog.Test to_string for boolean     (0.5 seconds)
-pass lib.tb_with_watchdog.Test that needs longer timeout (0.5 seconds)
-fail lib.tb_with_watchdog.Test that stalls               (0.5 seconds)
+pass lib.tb_with_watchdog.Test to_string for boolean     (0.6 s)
+pass lib.tb_with_watchdog.Test that needs longer timeout (0.6 s)
+fail lib.tb_with_watchdog.Test that stalls               (0.6 s)
 ============================================================================
 pass 2 of 3
 fail 1 of 3
 ============================================================================
-Total time was 1.6 seconds
-Elapsed time was 1.6 seconds
+Total time was 1.8 s
+Elapsed time was 1.8 s
 ============================================================================
 Some failed!
diff --git a/docs/run/src/run.py b/docs/run/src/run.py index a44edc9d5..c0b732f35 100644 --- a/docs/run/src/run.py +++ b/docs/run/src/run.py @@ -126,9 +126,24 @@ def extract_snippets(): snippet, ) + for snippet in [ + "get_seed_and_runner_cfg", + "get_seed_wo_runner_cfg", + "get_uniform_seed", + ]: + highlight_code( + root / "tb_seed.vhd", + root / ".." / "img" / f"{snippet}.html", + snippet, + ) + extract_snippets() +seed_option_path = root / "seed_option.txt" +seed_option_path.write_text('> python run.py "lib.tb_seed.Test that fails" --seed fb19f3cca859d69c') +highlight_log(seed_option_path, root / ".." / "img" / "seed_option.html") + def post_run(log_registry): def _post_run(results): @@ -142,6 +157,8 @@ def _post_run(results): cli = VUnitCLI() args = cli.parse_args() +print(args.test_patterns) + if args.compile or args.list: test_patterns = ["*"] elif args.test_patterns[0] != "*": @@ -155,6 +172,7 @@ def _post_run(results): "lib.tb_with_watchdog*", "lib.tb_stopping_failure*", "lib.tb_stop_level*", + "lib.tb_seed*", "lib.tb_magic_paths*", ] @@ -181,6 +199,8 @@ def _post_run(results): tb_with_lower_level_control = lib.test_bench("tb_with_lower_level_control") tb_with_lower_level_control.scan_tests_from_file(root / "test_control.vhd") + lib.test_bench("tb_seed").test("Test that fails").set_sim_option("seed", "fb19f3cca859d69c") + log_registry = LogRegistry() vu.set_compile_option("rivierapro.vcom_flags", ["-dbg"]) diff --git a/docs/run/src/tb_seed.vhd b/docs/run/src/tb_seed.vhd new file mode 100644 index 000000000..41a1c4d0a --- /dev/null +++ b/docs/run/src/tb_seed.vhd @@ -0,0 +1,59 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this file, +-- You can obtain one at http://mozilla.org/MPL/2.0/. +-- +-- Copyright (c) 2014-2025, Lars Asplund lars.anders.asplund@gmail.com + +library vunit_lib; +context vunit_lib.vunit_context; + +library ieee; +use ieee.math_real.uniform; + +entity tb_seed is + generic (runner_cfg : string); +end entity; + +architecture tb of tb_seed is + -- start_snippet get_seed_and_runner_cfg + constant global_seed : string := get_seed(runner_cfg, "Optional salt"); + -- end_snippet get_seed_and_runner_cfg + + -- Placeholder for a non-standard RNG + procedure init_rng(seed : string) is + begin + info("RNG seeded with " & seed); + end; +begin + test_runner : process + variable seed1, seed2 : positive; + variable a_random_value, another_random_value : real; + begin + test_runner_setup(runner, runner_cfg); + + while test_suite loop + if run("Test that passes") then + -- start_snippet get_uniform_seed + get_uniform_seed(seed1, seed2, "Optional salt"); + + uniform(seed1, seed2, a_random_value); + uniform(seed1, seed2, another_random_value); + -- end_snippet get_uniform_seed + + elsif run("Test that fails") then + error("Something bad happened"); + end if; + end loop; + + test_runner_cleanup(runner); + end process; + + -- start_snippet get_seed_wo_runner_cfg + randomizing_process : process is + variable local_seed : string_seed_t; -- string_seed_t = string(1 to 16) + begin + get_seed(local_seed, salt => randomizing_process'path_name); + -- end_snippet get_seed_wo_runner_cfg + wait; + end process; +end architecture; diff --git a/docs/run/user_guide.rst b/docs/run/user_guide.rst index be74a1979..0ac6daabd 100644 --- a/docs/run/user_guide.rst +++ b/docs/run/user_guide.rst @@ -174,7 +174,7 @@ The ``run_all_in_same_sim`` attribute can also be set from the run script, see : .. important:: When setting ``run_all_in_same_sim`` from the run script, the setting must be identical for all configurations of the testbench. - + The VUnit Watchdog ------------------ @@ -238,6 +238,62 @@ failure by setting the ``fail_on_warning`` flag. .. raw:: html :file: img/tb_stop_level_stdout.html +Seeds for Random Number Generation +---------------------------------- + +The Python test runner automatically generates a 64-bit base seed, which serves as the foundation for deriving new seeds +to initialize one or more random number generators (RNGs) within the simulation. This base seed is calculated using the +system time and a thread identifier, ensuring that it varies between simulations. This variability enhances test +coverage since randomized test will cover different areas in each simulation. By running the same randomized test in +several parallel threads but with different base seeds we can also shorten the execution time while maintaining the same +level of test coverage. + +.. note:: + VHDL-2019 introduces an interface for obtaining the system time. However, as of this writing, support for this + feature is limited, and where available, it only offers second-level resolution. If used for base seed generation, + simulations in parallel threads would receive the same base seed when started simultaneously. + + To address this, the Python test runner uses system time with microsecond resolution combined with an additional + thread identifier as the source of entropy. This approach ensures the uniqueness of base seeds across parallel threads. + +The base seed is provided via the ``runner_cfg`` generic and new derived seeds can be obtained by calling the +``get_seed`` function with ``runner_cfg`` and a salt string. The salt string is hashed together with the base seed to +ensure the uniqueness of each derived seed (with very high probability). The ``salt`` parameter can be omitted if only a +single seed is needed: + +.. raw:: html + :file: img/get_seed_and_runner_cfg.html + +A seed can also be obtained without referencing ``runner_cfg``, which is particularly useful when the seed is created +within a process that is not part of the top-level testbench. However, this is only possible after ``test_runner_setup`` +has been executed. To prevent race conditions, the ``get_seed`` procedure will block execution until +``test_runner_setup`` has completed. + +.. raw:: html + :file: img/get_seed_wo_runner_cfg.html + +In the previous examples, the ``get_seed`` subprograms returned seeds as strings. However, ``get_seed`` subprograms are +also available for ``integer`` seeds and 64-bit seeds of ``unsigned`` and ``signed`` type. Additionally, the +``get_uniform_seed`` subprogram is provided to support the standard ``uniform`` procedure in ``ieee.math_real``. The +``uniform`` procedure requires two ``positive`` seeds, ``seed1`` and ``seed2``, each with its own specific legal range +that is smaller than the full ``positive`` range. + +.. raw:: html + :file: img/get_uniform_seed.html + +Reproducibility +~~~~~~~~~~~~~~~ + +The seed used for a test is logged in the test output file (``/output.txt``) and, in the event of a test failure, it is also displayed on the console: + +.. raw:: html + :file: img/tb_seed_stdout.html + +To reproduce the failing test setup and verify a bug fix, the failing seed can be specified using the ``--seed`` option: + +.. raw:: html + :file: img/seed_option.html + Running A VUnit Testbench Standalone ------------------------------------ diff --git a/examples/vhdl/run/run.py b/examples/vhdl/run/run.py index 63ab78b5b..3c91b823c 100644 --- a/examples/vhdl/run/run.py +++ b/examples/vhdl/run/run.py @@ -16,13 +16,13 @@ from pathlib import Path from vunit import VUnit -ROOT = Path(__file__).parent +root = Path(__file__).parent -VU = VUnit.from_argv() -VU.add_vhdl_builtins() +vu = VUnit.from_argv() +vu.add_vhdl_builtins() -LIB = VU.add_library("lib") -LIB.add_source_files(ROOT / "*.vhd") -LIB.entity("tb_with_lower_level_control").scan_tests_from_file(ROOT / "test_control.vhd") +lib = vu.add_library("lib") +lib.add_source_files(root / "*.vhd") +lib.test_bench("tb_with_lower_level_control").scan_tests_from_file(root / "test_control.vhd") -VU.main() +vu.main() diff --git a/tests/unit/test_test_bench.py b/tests/unit/test_test_bench.py index 767faf58a..91655b940 100644 --- a/tests/unit/test_test_bench.py +++ b/tests/unit/test_test_bench.py @@ -909,7 +909,7 @@ def create_tests(test_bench): """ Helper method to reduce boiler plate """ - return test_bench.create_tests("simulator_if", elaborate_only=False) + return test_bench.create_tests("simulator_if", seed="seed", elaborate_only=False) def get_config_of(tests, test_name): diff --git a/tests/unit/test_test_suites.py b/tests/unit/test_test_suites.py index ab21cc354..a96ffb906 100644 --- a/tests/unit/test_test_suites.py +++ b/tests/unit/test_test_suites.py @@ -105,6 +105,7 @@ def _read_test_results(self, expected, contents): elaborate_only=False, test_suite_name=None, test_cases=expected, + seed="seed" ) results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access self.assertEqual(results, expected) @@ -179,6 +180,7 @@ def func(): elaborate_only=False, test_suite_name=None, test_cases=expected, + seed="seed" ) results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access diff --git a/vunit/sim_if/factory.py b/vunit/sim_if/factory.py index 75a56172d..29c7f8cff 100644 --- a/vunit/sim_if/factory.py +++ b/vunit/sim_if/factory.py @@ -15,7 +15,7 @@ from .modelsim import ModelSimInterface from .nvc import NVCInterface from .rivierapro import RivieraProInterface -from . import BooleanOption, ListOfStringOption, VHDLAssertLevelOption +from . import BooleanOption, ListOfStringOption, VHDLAssertLevelOption, StringOption class SimulatorFactory(object): @@ -62,6 +62,7 @@ def _extract_sim_options(self): BooleanOption("disable_ieee_warnings"), BooleanOption("enable_coverage"), ListOfStringOption("pli"), + StringOption("seed"), ] ) diff --git a/vunit/test/bench.py b/vunit/test/bench.py index 95f208d5e..9110f71cd 100644 --- a/vunit/test/bench.py +++ b/vunit/test/bench.py @@ -74,7 +74,7 @@ def get_default_config(self): raise RuntimeError(f"Test bench {self.library_name!s}.{self.name!s} has individually configured tests") return self._configs[DEFAULT_NAME] - def create_tests(self, simulator_if, elaborate_only, test_list=None): + def create_tests(self, simulator_if, seed, elaborate_only, test_list=None): """ Create all test cases from this test bench """ @@ -86,7 +86,7 @@ def create_tests(self, simulator_if, elaborate_only, test_list=None): if self._individual_tests: for test_case in self._test_cases: - test_case.create_tests(simulator_if, elaborate_only, test_list) + test_case.create_tests(simulator_if, seed, elaborate_only, test_list) elif self._implicit_test: for config in self._get_configurations_to_run(): test_list.add_test( @@ -94,6 +94,7 @@ def create_tests(self, simulator_if, elaborate_only, test_list=None): test=self._implicit_test, config=config, simulator_if=simulator_if, + seed=seed, elaborate_only=elaborate_only, ) ) @@ -104,6 +105,7 @@ def create_tests(self, simulator_if, elaborate_only, test_list=None): tests=[test.test for test in self._test_cases], config=config, simulator_if=simulator_if, + seed=seed, elaborate_only=elaborate_only, ) ) @@ -412,7 +414,7 @@ def _get_configurations_to_run(self): del configs[DEFAULT_NAME] return configs.values() - def create_tests(self, simulator_if, elaborate_only, test_list=None): + def create_tests(self, simulator_if, seed, elaborate_only, test_list=None): """ Create all tests from this test case which may be several depending on the number of configurations """ @@ -422,6 +424,7 @@ def create_tests(self, simulator_if, elaborate_only, test_list=None): test=self._test, config=config, simulator_if=simulator_if, + seed=seed, elaborate_only=elaborate_only, ) ) diff --git a/vunit/test/bench_list.py b/vunit/test/bench_list.py index e80bdb277..248456c92 100644 --- a/vunit/test/bench_list.py +++ b/vunit/test/bench_list.py @@ -60,13 +60,13 @@ def get_test_benches(self): result.append(test_bench) return result - def create_tests(self, simulator_if, elaborate_only): + def create_tests(self, simulator_if, seed, elaborate_only): """ Create all test cases from the test benches """ test_list = TestList() for test_bench in self.get_test_benches(): - test_bench.create_tests(simulator_if, elaborate_only, test_list) + test_bench.create_tests(simulator_if, seed, elaborate_only, test_list) return test_list def warn_when_empty(self): diff --git a/vunit/test/suites.py b/vunit/test/suites.py index c3d12dbe4..ec7bd051a 100644 --- a/vunit/test/suites.py +++ b/vunit/test/suites.py @@ -9,6 +9,9 @@ """ from pathlib import Path +from time import time +from hashlib import blake2b +from threading import get_ident from .. import ostools from .report import PASSED, SKIPPED, FAILED @@ -18,7 +21,7 @@ class IndependentSimTestCase(object): A test case to be run in an independent simulation """ - def __init__(self, test, config, simulator_if, elaborate_only=False): + def __init__(self, test, config, simulator_if, seed, *, elaborate_only=False): self._name = f"{config.library_name!s}.{config.design_unit_name!s}" if not config.is_default: @@ -40,6 +43,7 @@ def __init__(self, test, config, simulator_if, elaborate_only=False): elaborate_only=elaborate_only, test_suite_name=self._name, test_cases=[test.name], + seed=seed, ) @property @@ -78,7 +82,7 @@ class SameSimTestSuite(object): A test suite where multiple test cases are run within the same simulation """ - def __init__(self, tests, config, simulator_if, elaborate_only=False): + def __init__(self, tests, config, simulator_if, seed, *, elaborate_only=False): self._name = f"{config.library_name!s}.{config.design_unit_name!s}" if not config.is_default: @@ -93,6 +97,7 @@ def __init__(self, tests, config, simulator_if, elaborate_only=False): elaborate_only=elaborate_only, test_suite_name=self._name, test_cases=[test.name for test in tests], + seed=seed, ) @property @@ -149,12 +154,13 @@ class TestRun(object): A single simulation run yielding the results for one or several test cases """ - def __init__(self, *, simulator_if, config, elaborate_only, test_suite_name, test_cases): + def __init__(self, *, simulator_if, config, elaborate_only, test_suite_name, test_cases, seed): # pylint: disable=too-many-arguments self._simulator_if = simulator_if self._config = config self._elaborate_only = elaborate_only self._test_suite_name = test_suite_name self._test_cases = test_cases + self._args_seed = seed def set_test_cases(self, test_cases): self._test_cases = test_cases @@ -218,6 +224,17 @@ def _simulate(self, output_path): config = self._config.copy() + if self._args_seed: + seed = self._args_seed + elif "seed" in config.sim_options: + seed = config.sim_options["seed"] + else: + now_us = str(int(time() * 1e6)).encode() + thread_id = get_ident().to_bytes(8, byteorder="little") + seed = blake2b(now_us, digest_size=8, salt=thread_id).hexdigest() + + print(f"Seed for {self._test_suite_name}: {seed}") + if "output_path" in config.generic_names and "output_path" not in config.generics: config.generics["output_path"] = str(output_path.replace("\\", "/")) + "/" @@ -229,6 +246,7 @@ def _simulate(self, output_path): "output path": output_path.replace("\\", "/") + "/", "active python runner": True, "tb path": config.tb_path.replace("\\", "/") + "/", + "seed": seed, } # @TODO Warn if runner cfg already set? diff --git a/vunit/ui/__init__.py b/vunit/ui/__init__.py index 5bda79e59..dd9975e78 100644 --- a/vunit/ui/__init__.py +++ b/vunit/ui/__init__.py @@ -728,7 +728,7 @@ def _create_tests(self, simulator_if: Union[None, SimulatorInterface]): Create the test cases """ self._test_bench_list.warn_when_empty() - test_list = self._test_bench_list.create_tests(simulator_if, self._args.elaborate) + test_list = self._test_bench_list.create_tests(simulator_if, self._args.seed, self._args.elaborate) test_list.keep_matches(self._test_filter) return test_list diff --git a/vunit/vhdl/run/src/run.vhd b/vunit/vhdl/run/src/run.vhd index 1ac149dbf..05369cbdf 100644 --- a/vunit/vhdl/run/src/run.vhd +++ b/vunit/vhdl/run/src/run.vhd @@ -6,7 +6,6 @@ -- -- Copyright (c) 2014-2025, Lars Asplund lars.anders.asplund@gmail.com -use work.logger_pkg.all; use work.log_levels_pkg.all; use work.log_handler_pkg.all; use work.ansi_pkg.enable_colors; @@ -16,7 +15,6 @@ use work.path.all; use work.core_pkg; use std.textio.all; use work.event_common_pkg.all; -use work.event_private_pkg.all; use work.checker_pkg.all; use work.dict_pkg.all; @@ -34,6 +32,9 @@ package body run_pkg is (active_python_runner(runner_cfg) and not has_key(runner_cfg, "fake active python runner"))); if has_active_python_runner(runner_state) then + if has_key(runner_cfg, "seed") then + set_base_seed(runner_state, get(runner_cfg, "seed")); + end if; core_pkg.setup(output_path(runner_cfg) & "vunit_results"); hide(runner_trace_logger, display_handler, info); end if; @@ -139,6 +140,107 @@ package body run_pkg is return get_num_of_test_cases(runner_state); end; + function fnv_1a_64(str : string) return unsigned is + variable hash : unsigned(63 downto 0) := x"cbf29ce484222325"; + variable product : unsigned(104 downto 0); + begin + for idx in str'range loop + hash(7 downto 0) := hash(7 downto 0) xor to_unsigned(character'pos(str(idx)), 8); + + -- This is a performance optimized version of product := hash * fnv_prime + -- where fnv_prime = 0x00000100000001b3. + product := to_unsigned(to_integer(hash(15 downto 0)) * 435, 105) + + (to_unsigned(to_integer(hash(31 downto 16)) * 435, 105) sll 16) + + (to_unsigned(to_integer(hash(47 downto 32)) * 435, 105) sll 32) + + (to_unsigned(to_integer(hash(63 downto 48)) * 435, 105) sll 48) + + (hash sll 40); + + hash := product(63 downto 0); + end loop; + + return hash; + end; + + impure function get_seed(runner_cfg : string; salt : string := "") return unsigned is + begin + if has_key(runner_cfg, "seed") then + return fnv_1a_64(get(runner_cfg, "seed") & salt); + end if; + + core_pkg.core_failure("Seed not found in runner_cfg = " & runner_cfg); + return ""; + end; + + impure function get_seed(runner_cfg : string; salt : string := "") return signed is + constant seed : unsigned := get_seed(runner_cfg, salt); + begin + return signed(seed); + end; + + impure function get_seed(runner_cfg : string; salt : string := "") return integer is + constant seed : unsigned := get_seed(runner_cfg, salt); + begin + return to_integer(signed(seed(31 downto 0))); + end; + + impure function get_seed(runner_cfg : string; salt : string := "") return string is + variable seed : unsigned(63 downto 0); + variable seed_image : string(1 to 19); + begin + seed := get_seed(runner_cfg, salt); + seed_image := hex_image(std_logic_vector(seed)); + + return seed_image(3 to 18); + end; + + procedure wait_until_test_runner_is_setup is + begin + if get_phase(runner_state) <= test_runner_setup then + wait on runner until get_phase(runner_state) > test_runner_setup; + end if; + end; + + procedure get_seed(seed : out unsigned; salt : string := "") is + begin + wait_until_test_runner_is_setup; + + seed := fnv_1a_64(get_base_seed(runner_state) & salt); + end; + + procedure get_seed(seed : out signed; salt : string := "") is + variable seed_unsigned : unsigned(63 downto 0); + begin + get_seed(seed_unsigned, salt); + seed := signed(seed_unsigned); + end; + + procedure get_seed(seed : out integer; salt : string := "") is + variable seed_unsigned : unsigned(63 downto 0); + begin + get_seed(seed_unsigned, salt); + seed := to_integer(signed(seed_unsigned(31 downto 0))); + end; + + procedure get_seed(seed : out string; salt : string := "") is + variable unsigned_seed : unsigned(63 downto 0); + variable unsigned_seed_image : string(1 to 19); + begin + get_seed(unsigned_seed, salt); + unsigned_seed_image := hex_image(std_logic_vector(unsigned_seed)); + seed := unsigned_seed_image(3 to 18); + end; + + procedure get_uniform_seed(seed1, seed2 : out positive; salt : string := "") is + variable hash : unsigned(63 downto 0); + begin + wait_until_test_runner_is_setup; + + hash := fnv_1a_64(get_base_seed(runner_state) & salt); + -- The math_real.uniform procedure requires 1 <= seed11 <= 2147483562; 1 <= seed2 <= 2147483398. + seed1 := 1 + (to_integer(hash(62 downto 32)) mod 2147483562); + seed2 := 1 + (to_integer(hash(30 downto 0)) mod 2147483398); + end; + impure function enabled( constant name : string) return boolean is diff --git a/vunit/vhdl/run/src/run_api.vhd b/vunit/vhdl/run/src/run_api.vhd index 06baa3939..a758a8554 100644 --- a/vunit/vhdl/run/src/run_api.vhd +++ b/vunit/vhdl/run/src/run_api.vhd @@ -14,6 +14,7 @@ use work.event_private_pkg.all; library ieee; use ieee.std_logic_1164.all; +use ieee.numeric_std.all; package run_pkg is signal runner : runner_sync_t := new_basic_event(runner_phase_event) & new_basic_event(runner_timeout_update_event) & @@ -32,6 +33,18 @@ package run_pkg is impure function num_of_enabled_test_cases return integer; + impure function get_seed(runner_cfg : string; salt : string := "") return string; + impure function get_seed(runner_cfg : string; salt : string := "") return unsigned; + impure function get_seed(runner_cfg : string; salt : string := "") return signed; + impure function get_seed(runner_cfg : string; salt : string := "") return integer; + + procedure get_seed(seed : out string; salt : string := ""); + procedure get_seed(seed : out unsigned; salt : string := ""); + procedure get_seed(seed : out signed; salt : string := ""); + procedure get_seed(seed : out integer; salt : string := ""); + + procedure get_uniform_seed(seed1, seed2 : out positive; salt : string := ""); + impure function enabled ( constant name : string) return boolean; diff --git a/vunit/vhdl/run/src/run_types.vhd b/vunit/vhdl/run/src/run_types.vhd index f0fed061f..4bfffbda8 100644 --- a/vunit/vhdl/run/src/run_types.vhd +++ b/vunit/vhdl/run/src/run_types.vhd @@ -12,6 +12,7 @@ use work.event_private_pkg.all; library ieee; use ieee.std_logic_1164.all; +use ieee.numeric_std.all; package run_types_pkg is constant max_n_test_cases : natural := 1024; @@ -34,6 +35,9 @@ package run_types_pkg is end record; type boolean_array_t is array (integer range <>) of boolean; + subtype string_seed_t is string(1 to 16); + subtype unsigned_seed_t is unsigned(63 downto 0); + subtype signed_seed_t is signed(63 downto 0); constant runner_exit_with_errors : std_logic := 'Z'; constant runner_exit_without_errors : std_logic := '1'; diff --git a/vunit/vhdl/run/src/runner_pkg.vhd b/vunit/vhdl/run/src/runner_pkg.vhd index f5b4d06c4..454c98cbd 100644 --- a/vunit/vhdl/run/src/runner_pkg.vhd +++ b/vunit/vhdl/run/src/runner_pkg.vhd @@ -40,6 +40,9 @@ package runner_pkg is procedure set_active_python_runner(runner : runner_t; value : boolean); impure function has_active_python_runner(runner : runner_t) return boolean; + impure function get_base_seed(runner : runner_t) return string; + procedure set_base_seed(runner : runner_t; value : string); + impure function get_entry_key(runner : runner_t; phase : runner_legal_phase_t) return key_t; impure function get_exit_key(runner : runner_t; phase : runner_legal_phase_t) return key_t; impure function is_locked(runner : runner_t; key : key_t) return boolean; @@ -156,7 +159,8 @@ package body runner_pkg is constant exit_locks_idx : natural := 18; constant timeout_idx : natural := 19; constant is_within_gates_idx : natural := 20; - constant runner_length : natural := 21; + constant base_seed_idx : natural := 21; + constant runner_length : natural := base_seed_idx + 1; constant n_locks : positive := 254; constant n_locked_idx : natural := 0; @@ -227,6 +231,7 @@ package body runner_pkg is set(runner.p_data, timeout_idx, to_integer(new_string_ptr(str_pool, encode(0 ns)))); set(runner.p_data, is_within_gates_idx, 0); + set(runner.p_data, base_seed_idx, to_integer(new_string_ptr(str_pool, "4dcb138c52fce0e7"))); end; procedure set_active_python_runner(runner : runner_t; value : boolean) is @@ -239,6 +244,16 @@ package body runner_pkg is return get(runner.p_data, active_python_runner_idx) = 1; end function; + impure function get_base_seed(runner : runner_t) return string is + begin + return to_string(to_string_ptr(get(runner.p_data, base_seed_idx))); + end function; + + procedure set_base_seed(runner : runner_t; value : string) is + begin + set(runner.p_data, base_seed_idx, to_integer(new_string_ptr(str_pool, value))); + end; + impure function get_entry_key(runner : runner_t; phase : runner_legal_phase_t) return key_t is variable key : key_t; constant entry_locks : integer_vector_ptr_t := to_integer_vector_ptr(get(runner.p_data, entry_locks_idx)); diff --git a/vunit/vhdl/run/test/run_tests.vhd b/vunit/vhdl/run/test/run_tests.vhd index 304591b90..df5e2b6dc 100644 --- a/vunit/vhdl/run/test/run_tests.vhd +++ b/vunit/vhdl/run/test/run_tests.vhd @@ -21,6 +21,7 @@ use vunit_lib.event_common_pkg.all; library ieee; use ieee.std_logic_1164.all; +use ieee.numeric_std.all; entity run_tests is generic (output_path : string); @@ -29,10 +30,12 @@ end entity; architecture test_fixture of run_tests is constant locking_proc1_logger : logger_t := get_logger("locking_proc1_logger"); constant locking_proc2_logger : logger_t := get_logger("locking_proc2"); + constant c : checker_t := new_checker("checker_t", default_log_level => failure); signal start_test_process, start_test_process2 : boolean := false; signal test_process_completed : boolean := false; signal start_locking_process : boolean := false; signal start_test_runner_watchdog, test_runner_watchdog_completed : boolean := false; + signal start_seed_process : boolean := false; impure function get_phase return runner_phase_t is begin @@ -156,6 +159,20 @@ begin runner(runner_exit_status_idx) <= runner_exit_with_errors; end process watchdog; + seed_process : process is + variable seed : string(1 to 16); + variable timestamp : time; + begin + wait until start_seed_process; + timestamp := now; + get_seed(seed); + check(c, now - timestamp = 1 ns, "Expected delay = 1 ns but got " & time'image(now - timestamp)); + timestamp := now; + get_seed(seed, "salt"); + check(c, now - timestamp = 0 ns, "Expected no delay but got " & time'image(now - timestamp)); + wait; + end process; + test_runner : process procedure banner ( constant s : in string) is @@ -176,8 +193,6 @@ begin notify(runner_phase); end; - constant c : checker_t := new_checker("checker_t", default_log_level => failure); - procedure test_case_cleanup is variable stat : checker_stat_t; begin @@ -196,6 +211,11 @@ begin variable t_start : time; variable runner_cfg : line; variable passed : boolean; + variable seed : string_seed_t; + variable seed_unsigned : unsigned_seed_t; + variable seed_signed : signed_seed_t; + variable seed_int : integer; + variable seed1, seed2 : positive; constant test_case_setup_entry_key : key_t := get_entry_key(test_case_setup); constant test_case_setup_exit_key : key_t := get_exit_key(test_case_setup); @@ -852,7 +872,8 @@ begin if runner_cfg /= null then deallocate(runner_cfg); end if; - write(runner_cfg, string'("active python runner : true, enabled_test_cases : foo,, bar, output path : some_dir/out, tb path : some_other_dir/test")); + write(runner_cfg, + string'("active python runner : true, enabled_test_cases : foo,, bar, output path : some_dir/out, tb path : some_other_dir/test")); check(c, active_python_runner(runner_cfg.all), "Expected active python runner to be true"); passed := vunit_lib.run_pkg.output_path(runner_cfg.all) = "some_dir/out"; check(c, passed, "Expected output path to be ""some_dir/out"" but got " & vunit_lib.run_pkg.output_path(runner_cfg.all)); @@ -868,7 +889,6 @@ begin check(c, passed, "Expected enabled_test_cases to be ""__all__"" but got " & enabled_test_cases("")); passed := vunit_lib.run_pkg.tb_path("") = ""; check(c, passed, "Expected tb path to be """" but got " & vunit_lib.run_pkg.tb_path("")); - test_case_cleanup; --------------------------------------------------------------------------- banner("Should recognize runner_cfg_t for backward compatibility"); @@ -876,6 +896,76 @@ begin check(runner_cfg_t'("foo") = string'("foo")); test_case_cleanup; + --------------------------------------------------------------------------- + banner("Should dervive seeds"); + test_case_setup; + + -- Golden values retrieved from https://md5calc.com/hash/fnv1a64 + seed := vunit_lib.run_pkg.get_seed("active python runner : true,enabled_test_cases : foo,seed : dbe809ff2fc90f23"); + passed := seed = "ea8ad912addacf64"; + check(c, passed, "Expected seed from runner_cfg to be ""ea8ad912addacf64"" but got """ & seed & """"); + + seed := vunit_lib.run_pkg.get_seed("active python runner : true,enabled_test_cases : foo,seed : 075bd4fa7dc381cb", "Adding some salt"); + passed := seed = "30b243ae942e63d8"; + check(c, passed, "Expected seed from runner_cfg to be ""30b243ae942e63d8"" but got """ & seed & """"); + + seed_unsigned := vunit_lib.run_pkg.get_seed("active python runner : true,enabled_test_cases : foo,seed : 075bd4fa7dc381cb", "Adding some salt"); + passed := seed_unsigned = x"30b243ae942e63d8"; + check(c, passed, "Expected seed from runner_cfg to be x""30b243ae942e63d8"" but got """ & + hex_image(std_logic_vector(seed_unsigned)) & """"); + + seed_signed := vunit_lib.run_pkg.get_seed("active python runner : true,enabled_test_cases : foo,seed : 075bd4fa7dc381cb", "Adding some salt"); + passed := seed_signed = x"30b243ae942e63d8"; + check(c, passed, "Expected seed from runner_cfg to be x""30b243ae942e63d8"" but got """ & + hex_image(std_logic_vector(seed_signed)) & """"); + + seed_int := vunit_lib.run_pkg.get_seed("active python runner : true,enabled_test_cases : foo,seed : 075bd4fa7dc381cb", "Adding some salt"); + passed := seed_int = -1808899112; + check(c, passed, "Expected seed from runner_cfg to be -1808899112 but got " & integer'image(seed_int)); + + start_seed_process <= true; + wait for 1 ns; + + test_runner_setup(runner, "active python runner : true,enabled_test_cases : foo,seed : ce29981ce6db0c97"); + + get_seed(seed); + passed := seed = "e288ad7e1c4dbcb5"; + check(c, passed, "Expected seed to be ""e288ad7e1c4dbcb5"" but got """ & seed & """"); + + get_seed(seed, "Another seed"); + passed := seed = "0237b40b3d893879"; + check(c, passed, "Expected seed to be ""0237b40b3d893879"" but got """ & seed & """"); + + get_seed(seed_unsigned, "Another seed"); + passed := seed_unsigned = x"0237b40b3d893879"; + check(c, passed, "Expected seed to be x""0237b40b3d893879"" but got """ & + hex_image(std_logic_vector(seed_unsigned)) & """"); + + get_seed(seed_signed, "Another seed"); + passed := seed_signed = x"0237b40b3d893879"; + check(c, passed, "Expected seed to be x""0237b40b3d893879"" but got """ & + hex_image(std_logic_vector(seed_signed)) & """"); + + get_seed(seed_int, "Another seed"); + passed := seed_int = 1032403065; + check(c, passed, "Expected seed to be 1032403065 but got " & integer'image(seed_int)); + + get_uniform_seed(seed1, seed2); + passed := seed1 = 1653124479; + check(c, passed, "Expected seed1 to be 1653124479 but got " & integer'image(seed1)); + passed := seed2 = 474856630; + check(c, passed, "Expected seed2 to be 474856630 but got " & integer'image(seed2)); + + get_uniform_seed(seed1, seed2, "Some seed!"); + passed := seed1 = 481347971; + check(c, passed, "Expected seed1 to be 481347971 but got " & integer'image(seed1)); + passed := seed2 = 80218126; + check(c, passed, "Expected seed2 to be 80218126 but got " & integer'image(seed2)); + + p_disable_simulation_exit(runner_state); + test_runner_cleanup(runner); + test_case_cleanup; + --------------------------------------------------------------------------- core_pkg.setup(output_path & "vunit_results"); diff --git a/vunit/vhdl/string_ops/src/string_ops.vhd b/vunit/vhdl/string_ops/src/string_ops.vhd index f60f29afc..3bc08316e 100644 --- a/vunit/vhdl/string_ops/src/string_ops.vhd +++ b/vunit/vhdl/string_ops/src/string_ops.vhd @@ -57,6 +57,9 @@ package string_ops is function hex_image ( constant data : std_logic_vector) return string; + function from_hex ( + constant s : string) + return std_logic_vector; function replace ( constant s : string; constant old_segment : character; @@ -263,6 +266,40 @@ package body string_ops is return ret_val; end; + function from_hex ( + constant s : string) + return std_logic_vector is + variable ret_val : std_logic_vector(4 * s'length - 1 downto 0); + variable ret_val_idx : natural := 0; + variable nibble_value : natural; + variable pos : natural; + + function within(c, low, high : character) return boolean is + begin + return (character'pos(c) >= character'pos(low)) and (character'pos(c) <= character'pos(high)); + end; + begin + assert s'length > 0 report "from_hex input parameter length must be greater than 0" severity failure; + + for idx in s'reverse_range loop + pos := character'pos(s(idx)); + if within(s(idx), '0', '9') then + nibble_value := pos - character'pos('0'); + elsif within(s(idx), 'a', 'f') then + nibble_value := pos - character'pos('a') + 10; + elsif within(s(idx), 'A', 'F') then + nibble_value := pos - character'pos('A') + 10; + else + report "Illegal hex digit: " & s(idx) severity failure; + end if; + + ret_val(ret_val_idx + 3 downto ret_val_idx) := std_logic_vector(to_unsigned(nibble_value, 4)); + ret_val_idx := ret_val_idx + 4; + end loop; + + return ret_val; + end; + function strip ( str : string; chars : string; diff --git a/vunit/vhdl/string_ops/test/tb_string_ops.vhd b/vunit/vhdl/string_ops/test/tb_string_ops.vhd index c21f98d09..5c96c722c 100644 --- a/vunit/vhdl/string_ops/test/tb_string_ops.vhd +++ b/vunit/vhdl/string_ops/test/tb_string_ops.vhd @@ -65,6 +65,7 @@ begin variable ascending_vector : std_logic_vector(3 to 11); variable descending_vector : std_logic_vector(13 downto 5); variable l : lines_t; + constant hex_characters : string(1 to 16) := "0123456789abcdef"; constant offset_string :string(10 to 16) := "foo bar"; constant reverse_string :string(16 downto 10) := "foo bar"; constant reversed_vector :unsigned(16 downto 4) := "1011010101001"; @@ -181,6 +182,14 @@ begin descending_vector := "10101U101"; check(hex_image(descending_vector) = "x""15X""", "Should handle descending vector range"); + elsif run("Test from_hex") then + for value in 0 to 15 loop + check(from_hex((1 => hex_characters(value + 1))) = std_logic_vector(to_unsigned(value, 4)), + "Should return """ & to_nibble_string(to_unsigned(value, 4)) & """ on from_hex(""" & hex_characters(value + 1) & """), got """ & + to_nibble_string(from_hex((1 => hex_characters(value + 1)))) & """."); + end loop; + check(from_hex("ABCDEF") = "101010111100110111101111", "Should handle upper case letters"); + elsif run("Test replace") then check(replace("", 'a', 'b') = "", "Should return empty string on empty input string."); check(replace("foo bar", 'a', "") = "foo br", "Should be possible to replace segments with nothing"); diff --git a/vunit/vunit_cli.py b/vunit/vunit_cli.py index bcad69d15..2da3b0100 100644 --- a/vunit/vunit_cli.py +++ b/vunit/vunit_cli.py @@ -246,6 +246,12 @@ def _create_argument_parser(description=None, for_documentation=False): parser.add_argument("--version", action="version", version=version()) + parser.add_argument( + "--seed", + default=None, + help="Base seed provided to the simulation. Must be 16 hex digits. Default seed is generated from system time.", + ) + SIMULATOR_FACTORY.add_arguments(parser) return parser