merge with main

Author: lara

.github/workflows/scripts/test_overview_available_software.py

@@ -0,0 +1,64 @@
+import os
+import json
+from bs4 import BeautifulSoup
+
+path = os.path.dirname(os.path.realpath(__file__))
+path_overview = "/../../../docs/available_software/overview.md"
+path_data = "/../../../docs/available_software/data/json_data.json"
+if os.path.exists(path + path_overview) and os.path.exists(path + path_data):
+    with open(path + path_data) as json_data:
+        data = json.load(json_data)
+    with open(path + path_overview) as f:
+        soup = BeautifulSoup(f, "html.parser")
+else:
+    os.write(1, b'Error: Could not find overview.md and/or data/json_data.json')
+
+# parse the numbers for the different targets
+targets = data["targets"]
+ARM_targets = []
+x86_targets = []
+amd_targets = []
+intel_targets = []
+nvidia_targets = []
+
+for target in targets:
+    t = target.split('/')
+    if t[7] == 'aarch64':
+        ARM_targets.append(target)
+    else:
+        x86_targets.append(target)
+    if t[8] == "amd":
+        amd_targets.append(target)
+    elif t[8] == "intel":
+        intel_targets.append(target)
+    elif t[8] == 'nvidia':
+        nvidia_targets.append(target)
+
+# parse the overview.md page to check the number of colums in rows
+table = soup.find("table", {"class": "table"})
+for row in table.find_all("tr"):
+    for column in row.find_all('th'):
+        if column.text == "x86_64":
+            print(f'the value for x86_64 is {column.get("colspan")} in the overview page and there are {len(x86_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(x86_targets):
+                os.write(2, b'Error: Please make sure the values for x86_64 in json_data.json and overview.md are the same.')
+        elif column.text == "aarch64":
+            print(f'the value for aarch64 is {column.get("colspan")} in the overview page and there are {len(ARM_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(ARM_targets):
+                os.write(2, b'Error: Please make sure the values for aarch64 in json_data.json and overview.md are the same.')
+        elif column.text == "amd":
+            print(f'the value for amd is {column.get("colspan")} in the overview page and there are {len(amd_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(amd_targets):
+                os.write(2, b'Error:  Please make sure the values for amd in json_data.json and overview.md are the same.')
+        elif column.text == "intel":
+            print(f'the value for intel is {column.get("colspan")} in the overview page and there are {len(intel_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(intel_targets):
+                os.write(2, b'Error:  Please make sure the values for intel in json_data.json and overview.md are the same.')
+        elif column.text == "nvidia":
+            print(f'the value for nvidia is {column.get("colspan")} in the overview page and there are {len(nvidia_targets)} targets in json_data.json.')
+            if int(column.get("colspan")) != len(nvidia_targets):
+                os.write(2, b'Error: Please make sure the values for nvidia in json_data.json and overview.md are the same.')
+last_row = table.find_all("tr")[-1]
+print(f'there are {len(last_row.find_all("th"))} columns in the overview page and {len(targets)} targets in json_data.json.')
+if len(last_row.find_all("th")) != len(targets):
+    os.write(2, b'Error: Please make sure there are correct number of <th> elements in the last <tr> element in overview.md for JavaScript to generate the table.')

.github/workflows/test_overview_available_software.yml

@@ -0,0 +1,28 @@
+name: Test overview of available software in EESSI
+on:
+  push:
+    paths:
+        - ".github/**"
+        - "docs/available_software/data/**"
+jobs:
+  check_targets:
+    name: check targets in overview.md and json_data.json
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+
+      - name: set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: '3.10'
+          architecture: x64
+
+      - name: test overview available software
+        id: test_overview_available_software
+        run: |
+          # install required Python packages in virtual environment
+          python -m venv venv
+          . venv/bin/activate
+          pip install -r mkdocs-ldjson-plugin/requirements.txt
+
+          python .github/workflows/scripts/test_overview_available_software.py

docs/available_software/data/json_data.json

@@ -25,7 +25,7 @@ json_ld:
   softwareRequirements: See https://www.eessi.io/docs/ for how to make EESSI available
     on your system
   softwareVersion: '[''Java/11(@Java/11.0.20)'', ''Java/11.0.20'', ''Java/17(@Java/17.0.6)'',
-    ''Java/17.0.6'']'
+    ''Java/17.0.6'', ''Java/21(@Java/21.0.5)'', ''Java/21.0.5'']'
   url: ''
 ---
 
@@ -40,13 +40,15 @@ The overview below shows which Java installations are available per target archi
 To start using Java, load one of these modules using a `module load` command like:
 
 ```shell
-module load Java/17.0.6
+module load Java/21.0.5
 ```
 
 *(This data was automatically generated on {{ generated_time }})*  
 
 | |aarch64/generic|aarch64/neoverse_n1|aarch64/neoverse_v1|x86_64/generic|x86_64/amd/zen2|x86_64/amd/zen3|x86_64/amd/zen4|x86_64/intel/haswell|x86_64/intel/sapphirerapids|x86_64/intel/skylake_avx512|
 | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+|Java/21.0.5|x|x|x|x|x|x|x|x|x|x|
+|Java/21(@Java/21.0.5)|x|x|x|x|x|x|x|x|x|x|
 |Java/17.0.6|x|x|x|x|x|x|x|x|x|x|
 |Java/17(@Java/17.0.6)|x|x|x|x|x|x|x|x|x|x|
 |Java/11.0.20|x|x|x|x|x|x|x|x|x|x|

docs/available_software/data/json_data_detail.json

@@ -25,7 +25,7 @@ json_ld:
       ratingValue: 5
   softwareRequirements: See https://www.eessi.io/docs/ for how to make EESSI available
     on your system
-  softwareVersion: '[''Nextflow/23.10.0'']'
+  softwareVersion: '[''Nextflow/23.10.0'', ''Nextflow/24.10.2'']'
   url: https://www.nextflow.io/
 ---
 
@@ -44,11 +44,12 @@ The overview below shows which Nextflow installations are available per target a
 To start using Nextflow, load one of these modules using a `module load` command like:
 
 ```shell
-module load Nextflow/23.10.0
+module load Nextflow/24.10.2
 ```
 
 *(This data was automatically generated on {{ generated_time }})*  
 
 | |aarch64/generic|aarch64/neoverse_n1|aarch64/neoverse_v1|x86_64/generic|x86_64/amd/zen2|x86_64/amd/zen3|x86_64/amd/zen4|x86_64/intel/haswell|x86_64/intel/sapphirerapids|x86_64/intel/skylake_avx512|
 | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+|Nextflow/24.10.2|x|x|x|x|x|x|x|x|x|x|
 |Nextflow/23.10.0|x|x|x|x|x|x|x|x|x|x|

docs/available_software/detail/Java.md

@@ -7,28 +7,28 @@ This table gives an overview of all the available software in EESSI per specific
     <thead>
         <tr>
             <th rowspan="3">Name</th>
-            <th colspan="7">x86_64</th>
             <th colspan="4">aarch64</th>
+            <th colspan="7">x86_64</th>
         </tr>
-        </tr>
+        <tr>
+            <th colspan="3"></th>
+            <th colspan="1">nvidia</th>
             <th colspan="1"></th>
             <th colspan="3">amd</th>
             <th colspan="3">intel</th>
-            <th colspan="3"></th>
-            <th colspan="1">nvidia</th>
         </tr>
         <tr>
-            <th colspan="1">generic</th>
-            <th colspan="1">zen2</th>
-            <th colspan="1">zen3</th>
-            <th colspan="1">zen4</th>
-            <th colspan="1">haswell</th>
-            <th colspan="1">skylake_avx512</th>
-            <th colspan="1">sapphirerapids</th>
-            <th colspan="1">generic</th>
-            <th colspan="1">neoverse_n1</th>
-            <th colspan="1">neoverse_v1</th>
-            <th colspan="1">grace</th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
+            <th colspan="1"></th>
         </tr>
     </thead>
 </table>

docs/available_software/detail/Nextflow.md

@@ -23,12 +23,14 @@ All sessions will be recorded. Recordings, slides, and materials used will be ma
 
 ## Sessions
 
-- Monday 5 May 2025 (13:30-15:30 CEST): **Introduction to EESSI**
-- Monday 12 May 2025 (13:30-15:30 CEST): **Introduction to CernVM-FS**
+- Monday 5 May 2025 (13:30-15:30 CEST): **Introduction to EESSI** ([slides](EESSI-webinars-MayJune-2025-001-Introduction-to-EESSI-20250505.pdf), [recording](https://www.youtube.com/watch?v=FvVbzKLn-C8))
+- Monday 12 May 2025 (13:30-15:30 CEST): **Introduction to CernVM-FS** ([slides](EESSI-webinars-MayJune-2025-001-Introduction-to-CernVM-FS-20250512.pdf), [recording](https://www.youtube.com/watch?v=5-IYnxCz_aQ))
 - Monday 19 May 2025 (13:30-15:30 CEST): **Introduction to EasyBuild** (incl. EasyBuild 5.0.0)
 - Monday 26 May 2025 (13:30-15:30 CEST): **EESSI for CI/CD**
 - Monday 2 June 2025 (13:30-15:30 CEST): **Using EESSI as the base for a system stack**
 
+[YouTube playlist with recordings](https://www.youtube.com/playlist?list=PL6_PkP_6pUtbPSfa7Upa0R0_P4NAbqC9a)
+
 ## Format
 
 - Online webinars (via Zoom)
@@ -40,6 +42,15 @@ Attendance is free of cost, but registration is required.
 
 **Register via [https://event.ugent.be/registration/eessi202505](https://event.ugent.be/registration/eessi202505)**
 
+## Q&A via Slack
+
+For posting questions or comments during the webinar, we strongly prefer that you post them
+in the **`#webinar-series-2025q2` channel in the EESSI Slack** ([direct link to that channel](https://eessi-hpc.slack.com/archives/C068DV7GY3V)).
+
+If you haven't joined the EESSI Slack yet, first use the "`Slack channel`" link on the [EESSI website (https://eessi.io)](https://eessi.io).
+
+There will also be an opportunity at the end of the webinar to ask questions directly to the speakers, should you wish to do so.
+
 ## Useful links
 
 - EESSI website: [https://eessi.io](https://eessi.io)
@@ -74,6 +85,17 @@ a fully pre-built, modular software environment — no matter where you compute.
 sysadmin, or developer, this webinar will show how EESSI can help you save time, improve reproducibility,
 and simplify your scientific workflows.
 
+#### Materials
+
+- [slides (PDF)](EESSI-webinars-MayJune-2025-001-Introduction-to-EESSI-20250505.pdf)
+- [recording (YouTube)](https://www.youtube.com/watch?v=FvVbzKLn-C8)
+
+<div align="center">
+<iframe width="560" height="315" src="https://www.youtube.com/embed/FvVbzKLn-C8?si=WSTP8vw51zcpdtRB" title="YouTube video player"
+frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
+referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+</div>
+
 ---
 
 ### Introduction to CernVM-FS
@@ -95,6 +117,15 @@ cover some aspects specific to using CernVM-FS on HPC systems.
 It is intended for people who are interested in CernVM-FS (system administrators, support team members, researchers, etc.),
 no specific prior knowledge or experience with it is required.
 
+#### Materials
+
+- [slides (PDF)](EESSI-webinars-MayJune-2025-001-Introduction-to-CernVM-FS-20250512.pdf)
+- [recording (YouTube)](https://www.youtube.com/watch?v=5-IYnxCz_aQ)
+
+<div align="center">
+<iframe width="560" height="315" src="https://www.youtube.com/embed/5-IYnxCz_aQ?si=zqgYBiZCdY5islK8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+</div>
+
 ---
 
 ### Introduction to EasyBuild

docs/available_software/overview.md

@@ -67,7 +67,9 @@ You can use the following template for this:
 acl local_nodes src YOUR_CLIENT_IPS
 
 # Destination domains that are allowed
-acl stratum_ones dstdomain .YOURDOMAIN.ORG
+# cern.ch + opensciencegrid.org domains because of cvmfs-config.cern.ch repository,
+# which are provided via Stratum-1 mirror servers hosted by CERN and OSG
+acl stratum_ones dstdomain .cern.ch .opensciencegrid.org .YOURDOMAIN.ORG
 
 # Squid port
 http_port 3128
@@ -103,7 +105,8 @@ In this template, you *must* change two things in the Access Control List (ACL)
 For example, to allow connecting to the EESSI Stratum 1 replica servers, use:
 
 ```{ .apache .copy }
-acl stratum_ones dstdomain .eessi.science
+# public Stratum-1 mirror servers for EESSI are hosted under eessi.science domain
+acl stratum_ones dstdomain .cern.ch .opensciencegrid.org .eessi.science
 ```
 
 Note that this configuration assumes that port 3128 is accessible on the proxy server.

docs/training/2025/EESSI-webinars-MayJune-2025-001-Introduction-to-CernVM-FS-20250512.pdf

@@ -29,25 +29,26 @@ Examples include *(click to browse repository contents)*:
 
 ## LCG Releases
 
-The LCG Software Stack which is distributed via the CernVM-FS repository `/cvmfs/sft.cern.ch`
-contains over 450 external packages as well as HEP specific tools and generators.
-
-There are usually two releases per year as well as development builds every night.
+The [LCG Software Stacks](https://lcginfo.cern.ch/) which is distributed via the CernVM-FS repository
+`/cvmfs/sft.cern.ch` contains in total over 800 external packages as well as HEP-specific tools and generators.
 
 Software installations included often come with a script that updates your shell environment
 for using them by sourcing it. In addition, through so-called *views* a complete software stack
 can be made available in your shell environment.
 
-For more information, see <https://ep-dep-sft.web.cern.ch/document/lcg-releases> and
-<https://lcgdocs.web.cern.ch/lcgdocs/lcgreleases/introduction>.
+For example, loading software for the specific view [*LCG_107*](https://lcginfo.cern.ch/release/107/) for RHEL9 can be dona via:
+```
+source /cvmfs/sft.cern.ch/lcg/views/LCG_107/x86_64-el9-gcc14-opt/setup.sh
+```
+
 
 ## The Alliance
 
 The [Digital Research Alliance of Canada](https://alliancecan.ca/en/about/alliance), a.k.a. *The Alliance* and formerly
 known as *Compute Canada*, uses CernVM-FS to distribute the software stack for the Canadian national compute clusters.
 
-Documentation on using their CernVM-FS repository `/cvmfs/soft.computecanada.ca` is available
-[here](https://docs.alliancecan.ca/wiki/Accessing_CVMFS/en), and an overview of available software is available
+Documentation on using their CernVM-FS repository `/cvmfs/soft.computecanada.ca` can be found
+[here](https://docs.alliancecan.ca/wiki/Accessing_CVMFS/en), and an overview of all available software can be found
 [here](https://docs.alliancecan.ca/wiki/Available_software).
 
 ## Unpacked containers
@@ -69,7 +70,7 @@ More information on `unpacked.cern.ch` is available in the CernVM-FS documentati
 
 The [*European Environment for Scientific Software Installations (EESSI)*](https://eessi.io) provides optimized installations
 of scientific software for `x86_64` (Intel + AMD) and `aarch64` (64-bit Arm) systems that work on any Linux
-distribution.
+distribution via the CernVM-FS repository `/cvmfs/software.eessi.io`.
 
 We will use EESSI as an example CernVM-FS repository throughout this tutorial.
 

docs/training/2025/EESSI-webinars-MayJune-2025-001-Introduction-to-EESSI-20250505.pdf

@@ -57,9 +57,9 @@ In certain cases, the CernVM-FS has also been used to
 The metadata and content of files included in a CernVM-FS repository are **automatically downloaded on-demand**
 as files and directories are being accessed, which is akin to *streaming* services for music, movies, and TV series.
 
-This happens fully transparently, as the contents of a repository are exposed by CernVM-FS as if it were
+This happens fully transparently, all the contents of a repository are exposed by CernVM-FS as if they were
 a local (read-only) file system. Hence, clients that access a CernVM-FS repository typically do not
-actually have a local copy of all files included in that repository, but only have a limited set of files and
+have a local copy of all the files included in that repository, but only have a limited set of files and
 metadata directly available: those which were most recently accessed.
 
 
@@ -115,6 +115,8 @@ This can result in a significant reduction in storage capacity that is required
 especially when identical files are spread out across the repository,
 as often happens with particular files like example data files across multiple versions of the same software.
 
+For the CernVM-FS client, data de-duplication will also happen across repositories when the local client cached is shared across all repositories (the default).
+
 
 ### Compression of data { #features-compression }
 

docs/training/2025/webinar-series-2025Q2.md

@@ -11,4 +11,54 @@
 
 # Monitoring CernVM-FS
 
-*(coming soon)*
+There are multiple options available to automate the monitoring of CernVM-FS clients (see [CernVM-FS documentation](https://cvmfs.readthedocs.io/en/stable/cpt-configure.html#monitoring) ):
+
+- Nagios
+- Telemetry Aggregator
+- HTTP Tracing Header
+
+## Nagios
+CernVM-FS supports the Nagios monitoring system. A checker plugin is available on the [CernVM-FS download page](https://cernvm.cern.ch/fs/#download).
+
+## Telemetry Aggregator
+
+The [Telemetry Aggregator](https://cvmfs.readthedocs.io/en/stable/cpt-telemetry.html#cpt-telemetry) allows remote monitoring of the internal state of the client. These counters count access to internal data structures and number of errors, and can be accessed locally via `cvmfs_talk -i <repo> internal affairs`. The Telemetry Aggregator extends this data by also providing the timestamp of the counter collection, the current revision of the repository, and providing these counters in absolute values and delta values relative to the previous measurement. This e.g. allows monitoring if some clients fall behind in updating the repository to the latest revision.
+
+By default, the data is sent in Influx data format. The Influx Telemetry Aggregator also allows defining custom user-defined, static tags and fields.
+Other formats are possible but must be provided as a source code plugin.
+
+To activate it the following lines need to be added to the client config (e.g. `/etc/cvmfs/default.local`):
+```
+CVMFS_TELEMETRY_SEND=ON
+CVMFS_TELEMETRY_RATE=<rate in seconds> # minimum send rate >= 5 sec
+CVMFS_INFLUX_HOST=localhost                 # IP address
+CVMFS_INFLUX_PORT=8092                      # Port
+CVMFS_INFLUX_METRIC_NAME=<measurement name> # "Table" name
+CVMFS_INFLUX_EXTRA_TAGS="some_tag=42,some_tag2=27" # optional // tags are always sent
+CVMFS_INFLUX_EXTRA_FIELDS="somefield=3"            # optional // fields are only sent in absolute data
+```
+
+## HTTP Tracing Header
+
+CernVM-FS offers to add user-defined, static key-value pairs to the HTTP header for any of its client HTTP request. This includes download of uncached files, selection of proxy and servers (S0/S1).
+
+For this the following lines need to be added to the client config (e.g. `/etc/cvmfs/default.local`):
+```
+CVMFS_HTTP_TRACING=on 
+CVMFS_HTTP_TRACING_HEADERS='node:pn214|tag2:some_tag'
+```
+
+This results to the following header
+```
+(download) CURL Header for URL: /data/81/7c882d4a2e9dd7f9c5c2bfb4e04ff316e436dfC is:
+Connection: Keep-Alive
+Pragma:
+User-Agent: cvmfs Fuse 2.12.6
+X-CVMFS-node: pn214
+X-CVMFS-tag2: some_tag
+X-CVMFS-PID: 481270
+X-CVMFS-GID: 0
+X-CVMFS-UID: 0
+```
+
+