Add function to wrap verify_only_loopback_routes with assert and wait…

…_until (#16523) Description of PR Summary: Partially tackles #16577 Approach What is the motivation for this PR? Tests are flaky, sometimes failing on verify_only_loopback_routes_are_announced_to_neighs: Failed to verify routes on nbr in TSA How did you do it? Add a function wrapping verify_only_loopback_routes_are_announced_to_neighs with wait_until and assert to allow time for neighbor to update routes How did you verify/test it? These logs were seen in passed tests, showing that wait_until helps avoid false negatives 17/01/2025 05:42:18 utilities.wait_until L0153 DEBUG | verify_only_loopback_routes_are_announced_to_neighs is False, wait 10 seconds and check again 05:42:12 route_checker.verify_loopback_route_with L0014 INFO | Verifying only loopback routes are announced to bgp neighbors 05:42:18 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71002> terminated with exit code None 05:42:18 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71003> terminated with exit code None 05:42:18 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71001> terminated with exit code None 05:42:18 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71000> terminated with exit code None 05:42:18 parallel.parallel_run L0221 INFO | Completed running processes for target "parse_routes_process" in 0:00:03.470030 seconds 05:42:18 route_checker.verify_loopback_route_with L0035 INFO | Verifying only loopback routes(ipv4) are announced to ARISTA04T3 05:42:18 route_checker.verify_loopback_route_with L0047 WARNING| missing loopback address or some other routes present on neighbor 05:42:28 route_checker.verify_loopback_route_with L0014 INFO | Verifying only loopback routes are announced to bgp neighbors 05:42:34 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71002> terminated with exit code None 05:42:34 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71000> terminated with exit code None 05:42:34 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71003> terminated with exit code None 05:42:34 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71001> terminated with exit code None 05:42:34 parallel.parallel_run L0221 INFO | Completed running processes for target "parse_routes_process" in 0:00:03.332572 seconds 05:42:34 route_checker.verify_loopback_route_with L0035 INFO | Verifying only loopback routes(ipv4) are announced to ARISTA04T3 05:42:34 route_checker.verify_loopback_route_with L0035 INFO | Verifying only loopback routes(ipv4) are announced to ARISTA01T3 05:42:34 route_checker.verify_loopback_route_with L0035 INFO | Verifying only loopback routes(ipv4) are announced to ARISTA06T3 05:42:34 route_checker.verify_loopback_route_with L0035 INFO | Verifying only loopback routes(ipv4) are announced to ARISTA03T3 05:42:34 route_checker.verify_loopback_route_with L0014 INFO | Verifying only loopback routes are announced to bgp neighbors 05:42:39 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71000> terminated with exit code None 05:42:39 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71002> terminated with exit code None 05:42:39 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71001> terminated with exit code None 05:42:39 parallel.on_terminate L0085 INFO | process parse_routes_process--<EosHost VM71003> terminated with exit code None 05:42:39 parallel.parallel_run L0221 INFO | Completed running processes for target "parse_routes_process" in 0:00:02.681303 seconds All affected test suites were run Signed-off-by: Javier Tan [email protected]
sonic-net · Jan 29, 2025 · d6cfd63 · d6cfd63
1 parent 817a6af
commit d6cfd63
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 103 deletions.
diff --git a/tests/bgp/route_checker.py b/tests/bgp/route_checker.py
@@ -6,6 +6,7 @@
 from tests.common.devices.eos import EosHost
 from tests.common.helpers.assertions import pytest_assert
 from tests.common.helpers.parallel import parallel_run
+from tests.common.utilities import wait_until
 
 logger = logging.getLogger(__name__)
 
@@ -206,6 +207,18 @@ def verify_only_loopback_routes_are_announced_to_neighs(dut_hosts, duthost, neig
             dut_hosts, duthost, neigh_hosts, 6, community)
 
 
+def assert_only_loopback_routes_announced_to_neighs(dut_hosts, duthost, neigh_hosts, community,
+                                                    error_msg=""):
+    if not error_msg:
+        error_msg = "Failed to verify only loopback routes are announced to neighbours"
+
+    pytest_assert(
+        wait_until(180, 10, 5, verify_only_loopback_routes_are_announced_to_neighs,
+                   dut_hosts, duthost, neigh_hosts, community),
+        error_msg
+    )
+
+
 def parse_routes_on_neighbors(dut_host, neigh_hosts, ip_ver, exp_community=[]):
     if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost):
         routes_on_all_nbrs = parse_routes_on_eos(dut_host, neigh_hosts, ip_ver, exp_community)

diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py
@@ -13,7 +13,7 @@
 from tests.bgp.bgp_helpers import get_tsa_chassisdb_config, get_sup_cfggen_tsa_value, verify_dut_configdb_tsa_value
 from tests.bgp.traffic_checker import get_traffic_shift_state
 from tests.bgp.route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \
-    verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs
+    verify_current_routes_announced_to_neighs, assert_only_loopback_routes_announced_to_neighs
 from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE
 from tests.bgp.test_startup_tsa_tsb_service import get_tsa_tsb_service_uptime, get_tsa_tsb_service_status, \
     get_startup_tsb_timer, enable_disable_startup_tsa_tsb_service     # noqa: F401
@@ -309,9 +309,9 @@ def verify_linecard_after_sup_tsa(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -381,9 +381,9 @@ def verify_linecard_after_sup_tsa(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -439,9 +439,9 @@ def verify_linecard_after_sup_tsa(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Issue TSB on the supervisor
         suphost.shell('TSB')
@@ -537,9 +537,9 @@ def verify_linecard_after_sup_tsb(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -597,9 +597,9 @@ def run_tsa_on_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Verify supervisor still has tsa_enabled 'false' config
         pytest_assert('false' == get_tsa_chassisdb_config(suphost),
@@ -680,9 +680,9 @@ def run_tsa_on_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Verify supervisor still has tsa_enabled 'true' config
         pytest_assert('true' == get_tsa_chassisdb_config(suphost),
@@ -747,9 +747,9 @@ def run_tsa_on_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         def run_tsb_on_linecard_and_verify(lc):
             lc.shell('TSB')
@@ -845,9 +845,9 @@ def run_tsb_on_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Verify supervisor still has tsa_enabled 'true' config
         pytest_assert('true' == get_tsa_chassisdb_config(suphost),
@@ -991,9 +991,9 @@ def verify_linecard_tsa_tsb(lc):
                 executor.submit(verify_linecard_tsa_tsb, linecard)
 
         for linecard in duthosts.frontend_nodes:
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1098,9 +1098,9 @@ def verify_line_card_after_sup_config_reload(lc):
                 executor.submit(verify_line_card_after_sup_config_reload, linecard)
 
         for linecard in duthosts.frontend_nodes:
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1214,9 +1214,9 @@ def reboot_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced to neighbors when the linecards are in TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Verify supervisor still has tsa_enabled 'false' config
         pytest_assert('false' == get_tsa_chassisdb_config(suphost),
@@ -1302,10 +1302,10 @@ def verify_line_card_after_sup_tsa(lc):
         # Verify DUT is in maintenance state.
         pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(first_linecard, cmd='TSC no-stats'),
                       "DUT is not in maintenance state after config reload")
-
-        pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-            duthosts, first_linecard, dut_nbrhosts[first_linecard], traffic_shift_community),
-            "Failed to verify routes on nbr in TSA")
+        assert_only_loopback_routes_announced_to_neighs(duthosts, first_linecard,
+                                                        dut_nbrhosts[first_linecard],
+                                                        traffic_shift_community,
+                                                        "Failed to verify routes on nbr in TSA")
 
         # Verify supervisor still has tsa_enabled 'true' config
         pytest_assert('true' == get_tsa_chassisdb_config(suphost),
@@ -1393,9 +1393,9 @@ def verify_linecard_after_sup_tsa(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced with TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1478,9 +1478,9 @@ def verify_linecard_after_sup_tsb(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced with TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1602,9 +1602,9 @@ def verify_linecard_after_sup_tsa(lc):
 
         # Verify only loopback routes are announced to neighbors at this state
         for linecard in duthosts.frontend_nodes:
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1776,9 +1776,9 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb(
             pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'),
                           "DUT is not in maintenance state")
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         # Issue TSB on the supervisor
         suphost.shell('TSB')
@@ -1897,9 +1897,9 @@ def restart_bgp_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)
@@ -1962,9 +1962,9 @@ def run_tsa_on_linecard_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
 
         def run_tsb_on_linecard_and_verify(lc):
             lc.shell('TSB')
@@ -2112,9 +2112,9 @@ def restart_bgp_and_verify(lc):
 
         for linecard in duthosts.frontend_nodes:
             # Verify only loopback routes are announced after TSA
-            pytest_assert(verify_only_loopback_routes_are_announced_to_neighs(
-                duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community),
-                "Failed to verify routes on nbr in TSA")
+            assert_only_loopback_routes_announced_to_neighs(duthosts, linecard, dut_nbrhosts[linecard],
+                                                            traffic_shift_community,
+                                                            "Failed to verify routes on nbr in TSA")
     finally:
         # Bring back the supervisor and line cards to the normal state
         set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname)