[Impacted Area Based PR testing] Roll out rest of the PR checkers (#1…

…6565) What is the motivation for this PR? In #15666, we introduced a new approach to PR testing called Impacted Area-Based PR Testing. This model will be rolled out in phases. This PR implement the left PR checkers. How did you do it? Roll out the left PR checkers. How did you verify/test it? Test by pipeline itself, to see if PR checkers are running as expected.
sonic-net · Jan 20, 2025 · 3732b8e · 3732b8e
1 parent a3abf26
commit 3732b8e
Show file tree

Hide file tree

Showing 3 changed files with 171 additions and 36 deletions.
diff --git a/.azure-pipelines/impacted_area_testing/constant.py b/.azure-pipelines/impacted_area_testing/constant.py
@@ -6,7 +6,7 @@
 # - t0-2vlans
 # - t0-sonic
 # - t1- lag
-PR_TOPOLOGY_TYPE = ["t0", "t0-2vlans", "t0-sonic", "t1", "t1-multi-asic", "dpu", "dualtor"]
+PR_TOPOLOGY_TYPE = ["t0_checker", "t0-2vlans", "t0-sonic", "t1_checker", "t1-multi-asic", "dpu", "dualtor"]
 
 EXCLUDE_TEST_SCRIPTS = [
     "test_posttest.py",

diff --git a/.azure-pipelines/impacted_area_testing/get_test_scripts.py b/.azure-pipelines/impacted_area_testing/get_test_scripts.py
@@ -31,7 +31,7 @@ def topo_name_to_type(topo_name):
     if topo_type in ['mgmttor', 'm0', 'mc0', 'mx', 't0-52', 't0-mclag']:
         # certain testbed types are in 't0' category with different names.
         topo_type = 't0'
-    if topo_type in ['t1-lag', 't1-56-lag', 't1-64-lag']:
+    elif topo_type in ['t1-lag', 't1-56-lag', 't1-64-lag']:
         topo_type = 't1'
     return topo_type
 
@@ -40,7 +40,7 @@ def distribute_scripts_to_PR_checkers(match, script_name, test_scripts_per_topol
     for topology in match.group(1).split(","):
         topology_mark = topology.strip().strip('"').strip("'")
         if topology_mark == "any":
-            for key in ["t0", "t1"]:
+            for key in ["t0_checker", "t1_checker"]:
                 if script_name not in test_scripts_per_topology_type[key]:
                     test_scripts_per_topology_type[key].append(script_name)
         else:
@@ -65,11 +65,6 @@ def collect_scripts_by_topology_type(features: str, location: str) -> dict:
     # Note: The full path and name of files are stored in a list named "files"
     scripts = []
 
-    # This is just for the first stage of rolling out
-    # To avoid the overuse of resource, we will ignore the PR which modifies the common part.
-    if features == "":
-        return {}
-
     for feature in features.split(","):
         feature_path = os.path.join(location, feature)
         for root, dirs, script in os.walk(feature_path):
@@ -103,7 +98,15 @@ def collect_scripts_by_topology_type(features: str, location: str) -> dict:
         except Exception as e:
             raise Exception('Exception occurred while trying to get topology in {}, error {}'.format(s, e))
 
-    return {k: v for k, v in test_scripts_per_topology_type.items() if v}
+    test_scripts = {k: v for k, v in test_scripts_per_topology_type.items() if v}
+
+    # This is just for the first stage of rolling out
+    # To avoid the overuse of resource, we will ignore the PR which modifies the common part.
+    if features == "":
+        test_scripts.pop("t0_checker")
+        test_scripts.pop("t1_checker")
+
+    return test_scripts
 
 
 def main(features, location):

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -74,6 +74,7 @@ stages:
       value: $(Build.SourceBranchName)
 
   jobs:
+# Below are Impacted Area Based PR checkers
   - job: get_impacted_area
     displayName: "Get impacted area"
     timeoutInMinutes: 240
@@ -85,7 +86,7 @@ stages:
   - job: impacted_area_t0_elastictest
     displayName: "impacted-area-kvmtest-t0 by Elastictest - optional"
     dependsOn: get_impacted_area
-    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0')
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0_checker')
     variables:
       TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
     timeoutInMinutes: 240
@@ -104,9 +105,165 @@ stages:
           MIN_WORKER: $(INSTANCE_NUMBER)
           MAX_WORKER: $(INSTANCE_NUMBER)
           KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
-          MGMT_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+  - job: impacted_area_t0_2vlans_elastictest
+    displayName: "impacted-area-kvmtest-t0_2vlans by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-2vlans')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: t0-2vlans
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: t0
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+  - job: impacted_area_t1_lag_elastictest
+    displayName: "impacted-area-kvmtest-t1-lag by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1_checker')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: t1
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: t1-lag
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+  - job: impacted_area_dualtor_elastictest
+    displayName: "impacted-area-kvmtest-dualtor by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dualtor')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: dualtor
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: dualtor
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
           STOP_ON_FAILURE: "False"
 
+  - job: impacted_area_multi_asic_elastictest
+    displayName: "impacted-area-kvmtest-multi_asic by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1-multi-asic')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: t1-multi-asic
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: t1-8-lag
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          NUM_ASIC: 4
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+  - job: impacted_area_sonic_t0_elastictest
+    displayName: "impacted-area-kvmtest-sonic_t0 by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-sonic')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: t0-sonic
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: t0-64-32
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic "
+          VM_TYPE: vsonic
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+  - job: impacted_area_dpu_elastictest
+    displayName: "impacted-area-kvmtest-dpu by Elastictest - optional"
+    dependsOn: get_impacted_area
+    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dpu')
+    variables:
+      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
+    timeoutInMinutes: 240
+    continueOnError: true
+    pool: sonic-ubuntu-1c
+    steps:
+      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
+        parameters:
+          TOPOLOGY: dpu
+          BUILD_BRANCH: $(BUILD_BRANCH)
+
+      - template: .azure-pipelines/run-test-elastictest-template.yml
+        parameters:
+          TOPOLOGY: dpu
+          SCRIPTS: $(SCRIPTS)
+          MIN_WORKER: $(INSTANCE_NUMBER)
+          MAX_WORKER: $(INSTANCE_NUMBER)
+          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
+          MGMT_BRANCH: "master"
+          STOP_ON_FAILURE: "False"
+
+# Below is the original PR checkers
   - job: t0_elastictest
     displayName: "kvmtest-t0 by Elastictest"
     timeoutInMinutes: 240
@@ -151,31 +308,6 @@ stages:
         KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
         MGMT_BRANCH: "master"
 
-  - job: impacted_area_t1_lag_elastictest
-    displayName: "impacted-area-kvmtest-t1-lag by Elastictest - optional"
-    dependsOn: get_impacted_area
-    condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1')
-    variables:
-      TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ]
-    timeoutInMinutes: 240
-    continueOnError: true
-    pool: sonic-ubuntu-1c
-    steps:
-      - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml
-        parameters:
-          TOPOLOGY: t1
-          BUILD_BRANCH: $(BUILD_BRANCH)
-
-      - template: .azure-pipelines/run-test-elastictest-template.yml
-        parameters:
-          TOPOLOGY: t1-lag
-          SCRIPTS: $(SCRIPTS)
-          MIN_WORKER: $(INSTANCE_NUMBER)
-          MAX_WORKER: $(INSTANCE_NUMBER)
-          KVM_IMAGE_BRANCH: $(BUILD_BRANCH)
-          MGMT_BRANCH: $(BUILD_BRANCH)
-          STOP_ON_FAILURE: "False"
-
   - job: dualtor_elastictest
     displayName: "kvmtest-dualtor-t0 by Elastictest"
     timeoutInMinutes: 240