Perf Converter Update for TMA 4.8

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 .idea
-bandit_scan_results.sarif
 scripts/inputs
 scripts/outputs
 scripts/*.bat
+=======
+bandit_scan_results.sarif

scripts/config/replacements_config.json

@@ -12,12 +12,15 @@
         "PERF_METRICS.BRANCH_MISPREDICTS":"topdown\\-br\\-mispredict",
         "PERF_METRICS.FETCH_LATENCY":"topdown\\-fetch\\-lat",
         "PERF_METRICS.MEMORY_BOUND":"topdown\\-mem\\-bound",
-        "TOPDOWN.SLOTS:perf_metrics":"slots",
-        "TOPDOWN.SLOTS:percore":"TOPDOWN.SLOTS",
+        "TOPDOWN.SLOTS:PERF_METRICS":"slots",
+        "TOPDOWN.SLOTS:PERCORE":"TOPDOWN.SLOTS",
+        "SLOTS":"slots",
         "SOCKET_COUNT":"#num_packages",
         "HYPERTHREADING_ON":"#SMT_on",
+        "SMT_ON":"#SMT_on",
         "CORES_PER_SOCKET":"#num_cores / #num_packages",
         "DURATIONTIMEINMILLISECONDS":"( duration_time * 1000 )",
+        "DURATION_TIME":"duration_time",
         "UNC_IIO_PAYLOAD_BYTES_IN.MEM_READ.PART0":"UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0",
         "UNC_IIO_PAYLOAD_BYTES_IN.MEM_READ.PART1":"UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1",
         "UNC_IIO_PAYLOAD_BYTES_IN.MEM_READ.PART2":"UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2",
@@ -26,11 +29,12 @@
         "MSR_EVENT:msr=0x611:type=FREERUN:scope=PACKAGE":"power@energy\\-pkg@",
         "CPU_CLK_UNHALTED.THREAD_P:SUP":"CPU_CLK_UNHALTED.THREAD_P:k",
         "CPU_CLK_UNHALTED.THREAD:SUP":"CPU_CLK_UNHALTED.THREAD:k",
-        "UNC_M_CLOCKTICKS:one_unit": "imc_0@event\\=0x0@",
-        "UNC_C_CLOCKTICKS:one_unit": "cbox_0@event\\=0x0@",
-        "UNC_CHA_CLOCKTICKS:one_unit": "cha_0@event\\=0x0@",
+        "UNC_M_CLOCKTICKS:ONE_UNIT": "imc_0@event\\=0x0@",
+        "UNC_C_CLOCKTICKS:ONE_UNIT": "cbox_0@event\\=0x0@",
+        "UNC_CHA_CLOCKTICKS:ONE_UNIT": "cha_0@event\\=0x0@",
         "FREERUN_PKG_ENERGY_STATUS":"power@energy\\-pkg@",
-        "FREERUN_DRAM_ENERGY_STATUS":"power@energy\\-ram@"
+        "FREERUN_DRAM_ENERGY_STATUS":"power@energy\\-ram@",
+        "NUM_CPUS":"#num_cpus_online"
     },
     "metric_source_events":{
         "CHAS_PER_SOCKET":"UNC_CHA([^\\s]*)",
@@ -40,30 +44,30 @@
         {
             "events": ["ICACHE_", "INT_MISC", "UOPS_", "IDQ", "OFFCORE_", 
                 "L1D_", "DTLB_", "AMX_", "ITLB_", "EXE_", "INST_", "ASSISTS",
-                "SW_", "RS", "DSB2MITE_"],
+                "SW_", "RS", "DSB2MITE_", "ARB_"],
             "unit":"cpu",
             "translations":{
-                "c":"cmask",
-                "u":"umask",
-                "i":"inv",
-                "e":"edge"
+                "C":"cmask",
+                "U":"umask",
+                "I":"inv",
+                "E":"edge"
             },
             "scale":1
         },
         {
             "events": ["UNC_CHA_"],
             "unit":"cha",
             "translations":{
-                "filter1":"config1"
+                "FILTER1":"config1"
             },
             "scale":100000000
         },
         {
             "events":["UNC_C_"],
             "unit":"cbox",
             "translations":{
-                "opc":"filter_opc",
-                "tid":"filter_tid"
+                "OPC":"filter_opc",
+                "TID":"filter_tid"
             },
             "scale":1
         }

scripts/perf_format_converter.py

@@ -32,6 +32,8 @@
 # Fields to always display event if empty
 PERSISTENT_FIELDS = ["MetricGroup", "BriefDescription"]
 
+# Operators
+OPERATORS = ["+", "-", "/", "*", "(", ")", "max(", "min(", "if", "<", ">", ",", "else"]
 
 def main():
     # Get file pointers from args
@@ -114,6 +116,21 @@ def pad(string):
     return " " + string.strip() + " "
 
 
+def isNum(string):
+    """
+    Takes an inputted string and outputs if the string is a num.
+    eg. 1.0, 1e9, 29
+
+    @param path: string to check
+    @returns: if string is a num
+    """
+    if string.isdigit():
+        return True
+    if string.replace('.', '', 1).isdigit():
+        return True
+    if string.replace('e', '', 1).isdigit() and not string.startswith("e") and not string.endswith("e"):
+        return True
+
 class PerfFormatConverter:
     """
     Perf Format Converter class. Used to convert the json files. Contains all
@@ -169,9 +186,10 @@ def convert_to_perf_metrics(self):
                 new_metric = Metric(
                     brief_description=metric["BriefDescription"],
                     metric_expr=self.get_expression(metric),
-                    metric_group=self.fix_groups(metric),
+                    metric_group=self.get_groups(metric),
                     metric_name=self.translate_metric_name(metric),
-                    scale_unit=self.get_scale_unit(metric))
+                    scale_unit=self.get_scale_unit(metric),
+                    threshold=self.get_threshold(metric))
                 metrics.append(new_metric)
         except KeyError as error:
             sys.exit("Error in input JSON format during convert_to_perf_metrics():" + str(error) + ". Exiting")
@@ -186,37 +204,60 @@ def get_expression(self, metric):
         @param metric: metric data as a dictionary
         @returns: string containing un-aliased expression
         """
-        try:
-            # Get formula and events for conversion
-            base_formula = metric["Formula"].replace("DURATIONTIMEINSECONDS", "duration_time")
-            if base_formula.startswith("100 *") and metric["UnitOfMeasure"] == "percent":
-                base_formula = base_formula.replace("100 *", "");
-            events = metric["Events"]
-            constants = metric["Constants"]
-
-            # Replace event/const aliases with names
-            expression = base_formula.lower()
-            for event in events:
-                reg = r"((?<=[\s+\-*\/\(\)])|(?<=^))({})((?=[\s+\-*\/\(\)])|(?=$))".format(event["Alias"].lower())
-                expression = re.sub(reg,
-                                    pad(self.translate_metric_event(event["Name"])),
-                                    expression)
-            for const in constants:
-                reg = r"((?<=[\s+\-*\/\(\)])|(?<=^))({})((?=[\s+\-*\/\(\)])|(?=$))".format(const["Alias"].lower())
-                expression = re.sub(reg,
-                                    pad(self.translate_metric_constant(const["Name"], metric)),
-                                    expression)
-
-            # Add slots to metrics that have topdown events but not slots
-            if any(event["Name"] for event in events if "PERF_METRICS" in event["Name"]):
-                if not any(event["Name"] for event in events if "SLOTS" in event["Name"]):
-                    expression = "( " + expression + " ) + ( 0 * slots )"
-
-        except KeyError as error:
-            sys.exit("Error in input JSON format during get_expressions(): " + str(error) + ". Exiting")
-
-        # Remove any extra spaces in expression
-        return re.sub(r"[\s]{2,}", " ", expression.strip())
+        # TMA metric
+        if "TMA" in metric["Category"]:
+            if "BaseFormula" in metric and metric["BaseFormula"] != "":
+                expression_list = [a.strip() for a in metric["BaseFormula"].split(" ")]
+                for i, term in enumerate(expression_list):
+                    if term not in OPERATORS and not isNum(term):
+                        # Term is not an operator or a numeric value
+                        if "tma_" not in term:
+                            # Translate any event names
+                            expression_list[i] = self.translate_metric_event(term.upper())
+
+                # Combine into formula
+                expression = " ".join(expression_list).strip()
+
+                # Add slots to metrics that have topdown events but not slots
+                if "topdown" in expression and "slots" not in expression:
+                        expression = "( " + expression + " ) + ( 0 * slots )"
+
+                return expression
+            else:
+                print("Error: TMA metric without base formula found")
+        # Non TMA metric
+        else:
+            try:
+                # Get formula and events for conversion
+                base_formula = metric["Formula"].replace("DURATIONTIMEINSECONDS", "duration_time")
+                if base_formula.startswith("100 *") and metric["UnitOfMeasure"] == "percent":
+                    base_formula = base_formula.replace("100 *", "")
+                events = metric["Events"]
+                constants = metric["Constants"]
+
+                # Replace event/const aliases with names
+                expression = base_formula.lower()
+                for event in events:
+                    reg = r"((?<=[\s+\-*\/\(\)])|(?<=^))({})((?=[\s+\-*\/\(\)])|(?=$))".format(event["Alias"].lower())
+                    expression = re.sub(reg,
+                                        pad(self.translate_metric_event(event["Name"])),
+                                        expression)
+                for const in constants:
+                    reg = r"((?<=[\s+\-*\/\(\)])|(?<=^))({})((?=[\s+\-*\/\(\)])|(?=$))".format(const["Alias"].lower())
+                    expression = re.sub(reg,
+                                        pad(self.translate_metric_constant(const["Name"], metric)),
+                                        expression)
+
+                # Add slots to metrics that have topdown events but not slots
+                if any(event["Name"] for event in events if "PERF_METRICS" in event["Name"]):
+                    if not any(event["Name"] for event in events if "SLOTS" in event["Name"]):
+                        expression = "( " + expression + " ) + ( 0 * slots )"
+
+            except KeyError as error:
+                sys.exit("Error in input JSON format during get_expressions(): " + str(error) + ". Exiting")
+
+            # Remove any extra spaces in expression
+            return re.sub(r"[\s]{2,}", " ", expression.strip())
 
     def translate_metric_name(self, metric):
         """
@@ -269,19 +310,19 @@ def translate_event_options(self, split, event_info):
             if "=" in option:
                 split = option.split("=")
                 if split[0] in event_info["translations"]:
-                    if "x" in split[1]:
-                        translation += "\\\\," + event_info["translations"][split[0]] + "\\\\=" + split[1]
+                    if "x" in split[1] or "X" in split[1]:
+                        translation += "\\," + event_info["translations"][split[0]] + "\\=" + split[1]
                     else:
-                        translation += "\\\\," + event_info["translations"][split[0]] + "\\\\=" + (int(split[1]) * event_info["scale"])
+                        translation += "\\," + event_info["translations"][split[0]] + "\\=" + (int(split[1]) * event_info["scale"])
             elif "0x" in option:
                 split = option.split("0x")
                 if split[0] in event_info["translations"]:
-                    translation += "\\\\," + event_info["translations"][split[0]] + "\\\\=" + "0x" + split[1]
+                    translation += "\\," + event_info["translations"][split[0]] + "\\=" + "0x" + split[1]
             else:
                 match = re.match(r"([a-zA-Z]+)([\d]+)", option)
                 if match:
                     if match[1] in event_info["translations"]:
-                        translation += "\\\\,"+ event_info["translations"][match[1]] + "\\\\=" + "0x" + match[2]
+                        translation += "\\,"+ event_info["translations"][match[1]] + "\\=" + "0x" + match[2]
 
         return translation + "@"
 
@@ -344,15 +385,17 @@ def get_scale_unit(self, metric):
         else:
             return None
 
-    def fix_groups(self, metric):
+    def get_groups(self, metric):
         """
         Converts a metrics group field delimited by commas to a new list
         delimited by semi-colons
 
         @param metric: metric json object
         @returns: new string list of groups delimited by semi-colons
         """
-
+        if "MetricGroup" not in metric:
+            return ""
+
         # Get current groups
         groups = metric["MetricGroup"]
         if groups.isspace() or groups == "":
@@ -361,29 +404,41 @@ def fix_groups(self, metric):
             #new_groups = [g.strip() for g in groups.split(";") if not g.isspace() and g != ""]
             new_groups = [g.strip() for g in re.split(";|,", groups) if not g.isspace() and g != ""]
 
-
         # Add level and parent groups
-        if metric["Category"] == "TMA":
+        if metric["Category"] == "TMA" and ("Info" not in metric["MetricName"] or "TmaL1" in metric["MetricGroup"]):
             new_groups.append("TopdownL" + str(metric["Level"]))
             new_groups.append("tma_L" + str(metric["Level"]) + "_group")
             if "ParentCategory" in metric:
                 new_groups.append("tma_" + metric["ParentCategory"].lower().replace(" ", "_") + "_group")
+
+        # Add count domain
+        if "CountDomain" in metric and metric["CountDomain"] != "":
+            new_groups.append(metric["CountDomain"])
 
         return ";".join(new_groups) if new_groups.count != 0 else ""
 
+    def get_threshold(self, metric):
+        if "Threshold" in metric:
+            if "BaseFormula" in metric["Threshold"]:
+                return self.clean_metric_names(metric["Threshold"]["BaseFormula"])
+
+    def clean_metric_names(self, formula):
+        return re.sub(r'\([^\(\)]+\)', "", formula).lower().replace("metric_","").replace(".", "")
+
 
 class Metric:
     """
     Metric class. Only used to store data to be serialized into json
     """
 
     def __init__(self, brief_description, metric_expr,
-                 metric_group, metric_name, scale_unit):
+                 metric_group, metric_name, scale_unit, threshold):
         self.BriefDescription = brief_description
         self.MetricExpr = metric_expr
         self.MetricGroup = metric_group
         self.MetricName = metric_name
         self.ScaleUnit = scale_unit
+        self.Threshold = threshold
 
 
 if __name__ == "__main__":