Merge pull request #447 from laraPPr/order_targets

Order the targets in available_software.py

Author: Neves-P

.github/workflows/scripts/test_overview_available_software.py

@@ -0,0 +1,64 @@
+import os
+import json
+from bs4 import BeautifulSoup
+
+path = os.path.dirname(os.path.realpath(__file__))
+path_overview = "/../../../docs/available_software/overview.md"
+path_data = "/../../../docs/available_software/data/json_data.json"
+if os.path.exists(path + path_overview) and os.path.exists(path + path_data):
+    with open(path + path_data) as json_data:
+        data = json.load(json_data)
+    with open(path + path_overview) as f:
+        soup = BeautifulSoup(f, "html.parser")
+else:
+    os.write(1, b'Error: Could not find overview.md and/or data/json_data.json')
+
+# parse the numbers for the different targets
+targets = data["targets"]
+ARM_targets = []
+x86_targets = []
+amd_targets = []
+intel_targets = []
+nvidia_targets = []
+
+for target in targets:
+    t = target.split('/')
+    if t[7] == 'aarch64':
+        ARM_targets.append(target)
+    else:
+        x86_targets.append(target)
+    if t[8] == "amd":
+        amd_targets.append(target)
+    elif t[8] == "intel":
+        intel_targets.append(target)
+    elif t[8] == 'nvidia':
+        nvidia_targets.append(target)
+
+# parse the overview.md page to check the number of colums in rows
+table = soup.find("table", {"class": "table"})
+for row in table.find_all("tr"):
+    for column in row.find_all('th'):
+        if column.text == "x86_64":
+            print(f'the value for x86_64 is {column.get("colspan")} in the overview page and there are {len(x86_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(x86_targets):
+                os.write(2, b'Error: Please make sure the values for x86_64 in json_data.json and overview.md are the same.')
+        elif column.text == "aarch64":
+            print(f'the value for aarch64 is {column.get("colspan")} in the overview page and there are {len(ARM_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(ARM_targets):
+                os.write(2, b'Error: Please make sure the values for aarch64 in json_data.json and overview.md are the same.')
+        elif column.text == "amd":
+            print(f'the value for amd is {column.get("colspan")} in the overview page and there are {len(amd_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(amd_targets):
+                os.write(2, b'Error:  Please make sure the values for amd in json_data.json and overview.md are the same.')
+        elif column.text == "intel":
+            print(f'the value for intel is {column.get("colspan")} in the overview page and there are {len(intel_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(intel_targets):
+                os.write(2, b'Error:  Please make sure the values for intel in json_data.json and overview.md are the same.')
+        elif column.text == "nvidia":
+            print(f'the value for nvidia is {column.get("colspan")} in the overview page and there are {len(nvidia_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(nvidia_targets):
+                os.write(2, b'Error: Please make sure the values for nvidia in json_data.json and overview.md are the same.')
+last_row = table.find_all("tr")[-1]
+print(f'there are {len(last_row.find_all("th"))} columns in the overview page and {len(targets)} targets in json_data.json.')
+if len(last_row.find_all("th")) != len(targets):
+    os.write(2, b'Error: Please make sure there are correct number of <th> elements in the last <tr> element in overview.md for JavaScript to generate the table.')

.github/workflows/test_overview_available_software.yml

@@ -0,0 +1,28 @@
+name: Test overview of available software in EESSI
+on:
+  push:
+    paths:
+        - ".github/**"
+        - "docs/available_software/data/**"
+jobs:
+  check_targets:
+    name: check targets in overview.md and json_data.json
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+
+      - name: set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: '3.10'
+          architecture: x64
+
+      - name: test overview available software
+        id: test_overview_available_software
+        run: |
+          # install required Python packages in virtual environment
+          python -m venv venv
+          . venv/bin/activate
+          pip install -r mkdocs-ldjson-plugin/requirements.txt
+
+          python .github/workflows/scripts/test_overview_available_software.py

.gitignore

@@ -1,3 +1,5 @@
 site/
 venv*
+scripts/available_software/__pycache__
+scripts/available_software/tests/__pycache__
 

docs/available_software/javascripts/populate_overview.js

@@ -43,7 +43,8 @@ function populate_overview(json_data) {
                         targets: "_all",
                         className: 'dt-body-center'
                     }
-                ]
+                ],
+                scrollX: true,
             });
             console.log(table)
 

docs/available_software/overview.md

@@ -10,23 +10,23 @@ This table gives an overview of all the available software in EESSI per specific
             <th colspan="3">aarch64</th>
             <th colspan="7">x86_64</th>
         </tr>
-        </tr>
+        <tr>
             <th colspan="3"></th>
             <th colspan="1"></th>
             <th colspan="3">amd</th>
             <th colspan="3">intel</th>
         </tr>
         <tr>
-            <th colspan="1">generic</th>
-            <th colspan="1">neoverse_n1</th>
-            <th colspan="1">neoverse_v1</th>
-            <th colspan="1">generic</th>
-            <th colspan="1">zen2</th>
-            <th colspan="1">zen3</th>
-            <th colspan="1">zen4</th>
-            <th colspan="1">haswell</th>
-            <th colspan="1">skylake_avx512</th>
-            <th colspan="1">sapphirerapids</th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
         </tr>
     </thead>
 </table>

scripts/available_software/available_software.py

@@ -24,6 +24,7 @@
 import numpy as np
 from mdutils.mdutils import MdUtils
 from natsort import natsorted
+from functools import cmp_to_key
 
 EESSI_TOPDIR = "/cvmfs/software.eessi.io/versions/2023.06"
 
@@ -220,6 +221,37 @@ def targets_eessi() -> np.ndarray:
     return targets
 
 
+def eessi_target_compare(a, b):
+    """
+    A comparison function to compare the EESSI targets and order them.
+    First the main architecture is ordered alphabetically, then within them
+    the CPU targets are again ordered alphabetically, except for the
+    generic target, which always comes first. Targets that include an extra
+    vendor subdir always after those without a vendor subdir.
+    @return: 0, 1, -1
+    """
+    if a == b:
+        return 0
+
+    a_split = a.rsplit('/')
+    b_split = b.rsplit('/')
+
+    # We first compare the main architecture (aarch64, x86_64, ...), which is the 7th field
+    if a_split[7] == b_split[7]:
+        # Check if one item is for generic builds (last field), These should always be listed first
+        if a_split[-1] == 'generic':
+            return -1
+        if b_split[-1] == 'generic':
+            return 1
+        # If the number of fields are not equal, one has an extra vendor subdirectory (e.g. amd, intel, nvidia).
+        # These should always come after the ones without this extra level.
+        if len(a_split) != len(b_split):
+            return 1 if len(a_split) > len(b_split) else -1
+
+    # In all other cases we just do an alphabetical sort of the strings.
+    return 1 if a > b else -1
+
+
 def modules_eessi() -> dict:
     """
     Returns names of all software module that are installed on EESSI.
@@ -233,7 +265,14 @@ def modules_eessi() -> dict:
     if modulepath:
         module_unuse(modulepath)
 
-    targets = [t for t in targets_eessi() if not any(t.endswith(x) for x in EXCLUDE_CPU_TARGETS)]
+    targets = targets_eessi()
+
+    # Order targets
+    eessi_target_compare_key = cmp_to_key(eessi_target_compare)
+    ordered_targets = sorted(targets, key=eessi_target_compare_key)
+
+    targets = [t for t in ordered_targets if not any(t.endswith(x) for x in EXCLUDE_CPU_TARGETS)]
+
     for target in targets:
         print(f"\t Collecting available modules for {target}... ", end="", flush=True)
         module_use(target + "/modules/all/")

scripts/available_software/tests/test_json.py

@@ -6,7 +6,7 @@
 import os
 import json
 
-GENERIC = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"
+GENERIC_ARM = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"
 ZEN2 = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
 
 
@@ -39,7 +39,7 @@ def test_json_generate_simple(self):
         modules = modules_eessi()
         json_data = generate_json_overview_data(modules)
         assert len(json_data.keys()) == 3
-        assert list(json_data["targets"]) == [GENERIC, ZEN2]
+        assert list(json_data["targets"]) == [GENERIC_ARM, ZEN2]
         assert json_data["modules"] == {
                 "Markov": [1, 0],
                 "cfd": [1, 1],