WIP: ci: Verify mapfile.csv metric entries

mapfile.csv

@@ -137,6 +137,8 @@ GenuineIntel-6-8C,V1.18,/TGL/events/tigerlake_uncore.json,uncore,,,
 GenuineIntel-6-8D,V1.18,/TGL/events/tigerlake_uncore.json,uncore,,,
 GenuineIntel-6-8C,V1.18,/TGL/events/tigerlake_uncore_experimental.json,uncore experimental,,,
 GenuineIntel-6-8D,V1.18,/TGL/events/tigerlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-8C,V1.0,/TGL/metrics/tigerlake_metrics.json,metrics,,,
+GenuineIntel-6-8D,V1.0,/TGL/metrics/tigerlake_metrics.json,metrics,,,
 GenuineIntel-6-8F,V1.28,/SPR/events/sapphirerapids_core.json,core,,,
 GenuineIntel-6-8F,V1.28,/SPR/events/sapphirerapids_uncore.json,uncore,,,
 GenuineIntel-6-8F,V1.28,/SPR/events/sapphirerapids_uncore_experimental.json,uncore experimental,,,
@@ -159,37 +161,45 @@ GenuineIntel-6-97,V1.31,/ADL/events/alderlake_gracemont_core.json,hybridcore,0x2
 GenuineIntel-6-97,V1.31,/ADL/events/alderlake_goldencove_core.json,hybridcore,0x40,0x000001,Core
 GenuineIntel-6-97,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-97,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-97,V1.0,/ADL/metrics/alderlake_metrics_goldencove_core.json,metrics,,,
 GenuineIntel-6-9A,V1.31,/ADL/events/alderlake_gracemont_core.json,hybridcore,0x20,0x000001,Atom
 GenuineIntel-6-9A,V1.31,/ADL/events/alderlake_goldencove_core.json,hybridcore,0x40,0x000001,Core
 GenuineIntel-6-9A,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-9A,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-9A,V1.0,/ADL/metrics/alderlake_metrics_goldencove_core.json,metrics,,,
 GenuineIntel-6-B7,V1.31,/ADL/events/alderlake_gracemont_core.json,hybridcore,0x20,0x000001,Atom
 GenuineIntel-6-B7,V1.31,/ADL/events/alderlake_goldencove_core.json,hybridcore,0x40,0x000001,Core
 GenuineIntel-6-B7,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-B7,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-B7,V1.0,/ADL/metrics/alderlake_metrics_goldencove_core.json,metrics,,,
 GenuineIntel-6-BA,V1.31,/ADL/events/alderlake_gracemont_core.json,hybridcore,0x20,0x000001,Atom
 GenuineIntel-6-BA,V1.31,/ADL/events/alderlake_goldencove_core.json,hybridcore,0x40,0x000001,Core
 GenuineIntel-6-BA,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-BA,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-BA,V1.0,/ADL/metrics/alderlake_metrics_goldencove_core.json,metrics,,,
 GenuineIntel-6-BF,V1.31,/ADL/events/alderlake_gracemont_core.json,hybridcore,0x20,0x000001,Atom
 GenuineIntel-6-BF,V1.31,/ADL/events/alderlake_goldencove_core.json,hybridcore,0x40,0x000001,Core
 GenuineIntel-6-BF,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-BF,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-BF,V1.0,/ADL/metrics/alderlake_metrics_goldencove_core.json,metrics,,,
 GenuineIntel-6-BE,V1.31,/ADL/events/alderlake_gracemont_core.json,core,,,
 GenuineIntel-6-BE,V1.31,/ADL/events/alderlake_uncore.json,uncore,,,
 GenuineIntel-6-BE,V1.31,/ADL/events/alderlake_uncore_experimental.json,uncore experimental,,,
 GenuineIntel-6-AA,V1.14,/MTL/events/meteorlake_crestmont_core.json,hybridcore,0x20,0x000002,Atom
 GenuineIntel-6-AA,V1.14,/MTL/events/meteorlake_redwoodcove_core.json,hybridcore,0x40,0x000002,Core
 GenuineIntel-6-AA,V1.14,/MTL/events/meteorlake_uncore.json,uncore,,,
 GenuineIntel-6-AA,V1.14,/MTL/events/meteorlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AA,V1.0,/MTL/metrics/meteorlake_metrics_redwoodcove_core.json,metrics,,,
 GenuineIntel-6-AC,V1.14,/MTL/events/meteorlake_crestmont_core.json,hybridcore,0x20,0x000002,Atom
 GenuineIntel-6-AC,V1.14,/MTL/events/meteorlake_redwoodcove_core.json,hybridcore,0x40,0x000002,Core
 GenuineIntel-6-AC,V1.14,/MTL/events/meteorlake_uncore.json,uncore,,,
 GenuineIntel-6-AC,V1.14,/MTL/events/meteorlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AC,V1.0,/MTL/metrics/meteorlake_metrics_redwoodcove_core.json,metrics,,,
 GenuineIntel-6-B5,V1.14,/MTL/events/meteorlake_crestmont_core.json,hybridcore,0x20,0x000002,Atom
 GenuineIntel-6-B5,V1.14,/MTL/events/meteorlake_redwoodcove_core.json,hybridcore,0x40,0x000002,Core
 GenuineIntel-6-B5,V1.14,/MTL/events/meteorlake_uncore.json,uncore,,,
 GenuineIntel-6-B5,V1.14,/MTL/events/meteorlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-B5,V1.0,/MTL/metrics/meteorlake_metrics_redwoodcove_core.json,metrics,,,
 GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_core.json,core,,,
 GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
 GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
@@ -212,15 +222,18 @@ GenuineIntel-6-BD,V1.14,/LNL/events/lunarlake_skymont_core.json,hybridcore,0x20,
 GenuineIntel-6-BD,V1.14,/LNL/events/lunarlake_lioncove_core.json,hybridcore,0x40,0x000003,Core
 GenuineIntel-6-BD,V1.14,/LNL/events/lunarlake_uncore.json,uncore,,,
 GenuineIntel-6-BD,V1.14,/LNL/events/lunarlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-BD,V1.0,/LNL/metrics/lunarlake_metrics_lioncove_core.json,metrics,,,
 GenuineIntel-6-C5,V1.09,/ARL/events/arrowlake_skymont_core.json,hybridcore,0x20,0x000003,Atom
 GenuineIntel-6-C5,V1.09,/ARL/events/arrowlake_crestmont_core.json,hybridcore,0x20,0x000002,LowPower_Atom
 GenuineIntel-6-C5,V1.09,/ARL/events/arrowlake_lioncove_core.json,hybridcore,0x40,0x000003,Core
 GenuineIntel-6-C5,V1.09,/ARL/events/arrowlake_uncore.json,uncore,,,
 GenuineIntel-6-C5,V1.09,/ARL/events/arrowlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-C5,V1.0,/ARL/metrics/arrowlake_metrics_lioncove_core.json,metrics,,,
 GenuineIntel-6-C6,V1.09,/ARL/events/arrowlake_skymont_core.json,hybridcore,0x20,0x000003,Atom
 GenuineIntel-6-C6,V1.09,/ARL/events/arrowlake_lioncove_core.json,hybridcore,0x40,0x000003,Core
 GenuineIntel-6-C6,V1.09,/ARL/events/arrowlake_uncore.json,uncore,,,
 GenuineIntel-6-C6,V1.09,/ARL/events/arrowlake_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-C6,V1.0,/ARL/metrics/arrowlake_metrics_lioncove_core.json,metrics,,,
 GenuineIntel-6-CC,V1.00,/PTL/events/pantherlake_darkmont_core.json,hybridcore,0x20,0x000004,Atom
 GenuineIntel-6-CC,V1.00,/PTL/events/pantherlake_cougarcove_core.json,hybridcore,0x40,0x000004,Core
 GenuineIntel-6-DD,V1.00,/CWF/events/clearwaterforest_core.json,core,,,

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/GNR/events/graniterapids_core.json

@@ -0,0 +1,11 @@
+{
+  "Header": {
+    "Copyright": "",
+    "Info": "Placeholder core event file.",
+    "DatePublished": "",
+    "Version": "",
+    "Legend": ""
+  },
+  "Events": [
+  ]
+}

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/GNR/events/graniterapids_uncore.json

@@ -0,0 +1,11 @@
+{
+  "Header": {
+    "Copyright": "",
+    "Info": "Placeholder uncore event file.",
+    "DatePublished": "",
+    "Version": "",
+    "Legend": ""
+  },
+  "Events": [
+  ]
+}

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/GNR/events/graniterapids_uncore_experimental.json

@@ -0,0 +1,11 @@
+{
+  "Header": {
+    "Copyright": "",
+    "Info": "Placeholder uncore experimental event file.",
+    "DatePublished": "",
+    "Version": "",
+    "Legend": ""
+  },
+  "Events": [
+  ]
+}

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/GNR/metrics/graniterapids_metrics.json

@@ -0,0 +1,13 @@
+{
+  "Header": {
+    "Copyright": "",
+    "Info": "Placeholder metrics file.",
+    "DatePublished": "",
+    "Version": "",
+    "Legend": "",
+    "TmaVersion": "",
+    "TmaFlavor": ""
+  },
+  "Metrics": [
+  ]
+}

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/GNR/metrics/graniterapids_retire_latency.json

@@ -0,0 +1,16 @@
+{
+  "Platform": {
+    "Model name": "Placeholder model",
+    "CPU family": "",
+    "Model": "",
+    "Thread(s) per core": "",
+    "Core(s) per socket": "",
+    "Socket(s)": "",
+    "Stepping": "",
+    "L3 cache": "",
+    "NUMA node(s)": "",
+    "TMA version": ""
+  },
+  "Data": {
+  }
+}

scripts/ci/verify_mapfile/test_data/14_mapfile_missing_latency_file_reference/mapfile.csv

@@ -0,0 +1,10 @@
+Family-model,Version,Filename,EventType,Core Type,Native Model ID,Core Role Name
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_core.json,core,,,
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AD,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,
+GenuineIntel-6-AD,V1.08,/GNR/metrics/graniterapids_retire_latency.json,retire latency,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_core.json,core,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AE,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,

scripts/ci/verify_mapfile/test_data/15_platform_referencing_multiple_metric_dirs/mapfile.csv

@@ -0,0 +1,16 @@
+Family-model,Version,Filename,EventType,Core Type,Native Model ID,Core Role Name
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_core.json,core,,,
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AD,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,
+GenuineIntel-6-AD,V1.08,/GNR/metrics/graniterapids_retire_latency.json,retire latency,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_core.json,core,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-AE,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,
+GenuineIntel-6-AE,V1.08,/GNR/metrics/graniterapids_retire_latency.json,retire latency,,,
+GenuineIntel-6-B6,V1.09,/GRR/events/grandridge_core.json,core,,,
+GenuineIntel-6-B6,V1.09,/GRR/events/grandridge_uncore.json,uncore,,,
+GenuineIntel-6-B6,V1.09,/GRR/events/grandridge_uncore_experimental.json,uncore experimental,,,
+GenuineIntel-6-B6,V1.02,/GRR/metrics/grandridge_metrics.json,metrics,,,
+GenuineIntel-6-B6,V1.08,/GNR/metrics/grandridge_retire_latency.json,retire latency,,,

scripts/ci/verify_mapfile/test_verify_mapfile.py

@@ -112,6 +112,23 @@ def test_mapfile_missing_event_file_reference(self):
         self.assertRegex(str(assertion_context.exception), r'Event file .* is not referenced')
 
 
+class TestVerifyFamilyModelMapsToMetricFiles(unittest.TestCase):
+
+    def test_mapfile_missing_metric_file_reference(self):
+        perfmon_repo_dir = Path(_test_data_dir, '14_mapfile_missing_latency_file_reference')
+
+        with self.assertRaises(RuntimeError) as assertion_context:
+            verify_family_model_maps_to_metric_files(perfmon_repo_dir)
+        self.assertRegex(str(assertion_context.exception), r'Metric file .* is not referenced')
+
+    def test_platform_referencing_multiple_metric_directories(self):
+        perfmon_repo_dir = Path(_test_data_dir, '15_platform_referencing_multiple_metric_dirs')
+
+        with self.assertRaises(RuntimeError) as assertion_context:
+            verify_family_model_maps_to_metric_files(perfmon_repo_dir)
+        self.assertIn('references multiple metric directories', str(assertion_context.exception))
+
+
 class TestVerifyMapfileDuplicateTypes(unittest.TestCase):
 
     def test_duplicate_event_file_type(self):

scripts/ci/verify_mapfile/verify_mapfile.py

@@ -124,6 +124,7 @@ def verify_event_type_matches_file(perfmon_repo_path: Path):
         skylake_fp_arith_inst.json -> fp_arith_inst
         alderlake_gracemont_core.json -> hybridcore (except for ADL-N)
         sierraforest_metrics.json -> metrics
+        alderlake_metrics_goldencove_core.json -> metrics
         graniterapids_retire_latency.json -> retire latency
     """
     # Certain model IDs have unique comparisons. As an example, ADL-N is not listed in the
@@ -134,7 +135,7 @@ def verify_event_type_matches_file(perfmon_repo_path: Path):
                     (re.compile(r'.*_uncore\.json'), 'uncore'),
                     (re.compile(r'.*_uncore_experimental\.json'), 'uncore experimental'),
                     (re.compile(r'.*_matrix\.json'), 'offcore'),
-                    (re.compile(r'.*_metrics\.json'), 'metrics'),
+                    (re.compile(r'.*_metrics(_.*_core)?\.json'), 'metrics'),
                     (re.compile(r'.*_retire_latency\.json'), 'retire latency'),
                     (re.compile(r'.*_fp_arith_inst\.json'), 'fp_arith_inst')],
         'GenuineIntel-6-BE': [(re.compile(r'.*/[a-z]*_[a-z]*_core\.json'), 'core'),
@@ -236,6 +237,81 @@ def verify_family_model_maps_to_event_files(perfmon_repo_path: Path):
                 raise RuntimeError(msg)
 
 
+def verify_family_model_maps_to_metric_files(perfmon_repo_path: Path):
+    """
+    For each Family-Model verify that all metric files are mentioned in mapfile.csv
+
+    Failing example:
+        mapfile.csv:
+            <snip>
+            GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_core.json,core,,,
+            GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+            GenuineIntel-6-AD,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+            GenuineIntel-6-AD,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,
+            GenuineIntel-6-AD,V1.08,/GNR/metrics/graniterapids_retire_latency.json,retire latency,,,
+            GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_core.json,core,,,
+            GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore.json,uncore,,,
+            GenuineIntel-6-AE,V1.10,/GNR/events/graniterapids_uncore_experimental.json,uncore experimental,,,
+            GenuineIntel-6-AE,V1.02,/GNR/metrics/graniterapids_metrics.json,metrics,,,
+            <snip>
+        Files in GNR/metrics/:
+            graniterapids_metrics.json
+            graniterapids_retire_latency.json
+        Failure:
+            Family-Model 0xAE is missing graniterapids_retire_latency.json.
+    """
+    # Known exceptions. Do not flag these combinations of models and files as issues.
+    exceptions = {
+        # SPR-HBM metrics are not included in mapfile.csv
+        'GenuineIntel-6-8F': ['sapphirerapidshbm_metrics.json'],
+        # ADL-N is an E-Core only product. Refer to Table 1 in
+        # https://cdrdv2.intel.com/v1/dl/getContent/759603
+        'GenuineIntel-6-BE': ['alderlake_metrics_goldencove_core.json'],
+    }
+
+    logger.info('Checking mapfile.csv for missing metric files.')
+
+    mapfile_data = load_mapfile(perfmon_repo_path)
+
+    # With the set of unique models compare mapfile.csv rows to actual event files.
+    for model in get_unique_mapfile_models(mapfile_data):
+        # Filter for any rows with a matching model.
+        mapfile_model_rows = [x for x in mapfile_data if x['Family-model'] == model]
+        # Then extract file paths that are mentioned. The Filename column includes a leading slash
+        # which is trimmed before combining with the repository path.
+        mapfile_model_files = [
+            Path(perfmon_repo_path, x['Filename'][1:]) for x in mapfile_model_rows
+        ]
+
+        # Platforms should only reference metric files in one directory. Platform ABC should only
+        # use metrics files in ABC/metrics. ABC using events in ABC/metrics and XYZ/metrics is likely
+        # a typo or a mistake.
+        mapfile_metrics_dir = set([x.parent for x in mapfile_model_files])
+        mapfile_metrics_dir = [x for x in mapfile_metrics_dir if x.name == 'metrics']
+        if len(mapfile_metrics_dir) > 1:
+            msg = (f'Family-model {model} references multiple metric directories.\n'
+                   f'{mapfile_metrics_dir}')
+            raise RuntimeError(msg)
+
+        # Verify that all metric files are mentioned in the mapfile for this specific model.
+        model_metrics_dir = Path(mapfile_model_files[0].parents[1], 'metrics')
+        metric_files = sorted(model_metrics_dir.glob('*.json'))
+        for metric_file in metric_files:
+            logging.error(f'Checking {metric_file.name} for {model}')
+            if metric_file not in mapfile_model_files:
+                # First check known exceptions before flagging as an actual issue.
+                if model in exceptions and metric_file.name in exceptions[model]:
+                    logger.warning('\tmapfile.csv %s missing row for %s. Known exception, OK.',
+                                   model, metric_file.name)
+                    continue
+
+                metric_files_msg = '\n  '.join([x.name for x in metric_files])
+                msg = (f'Metric file {metric_file.name} is not referenced by {model}.\n'
+                       f'mapfile.csv\n{json.dumps(mapfile_model_rows, indent=2)}\n'
+                       f'Files in {model_metrics_dir}\n  {metric_files_msg}')
+                raise RuntimeError(msg)
+
+
 def verify_mapfile_duplicate_types(perfmon_repo_path: Path):
     """
     Per Family-Model verify that there is only one instance of each event file type.
@@ -307,7 +383,7 @@ def verify_mapfile_model_event_versions(perfmon_repo_path: Path):
 
     for model in get_unique_mapfile_models(mapfile_data):
         # Extract file versions for rows with a matching model.
-        skip_types = ['metrics', 'retire latency'] # Not event file rows
+        skip_types = ['metrics', 'retire latency']  # Not event file rows
         event_file_versions = [
             x['Version']
             for x in mapfile_data
@@ -336,6 +412,7 @@ def verify_mapfile_model_event_versions(perfmon_repo_path: Path):
         verify_event_file_versions,
         verify_event_type_matches_file,
         verify_family_model_maps_to_event_files,
+        verify_family_model_maps_to_metric_files,
         verify_mapfile_duplicate_types,
         verify_mapfile_model_event_versions,
     ]