From 01aad62edb799e18d3d615c4ac0cd2220c64c048 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Feb 2025 10:39:03 -0800 Subject: [PATCH 001/128] Initial commit for issue #59. This is still a WIP and requires thorough testing. Includes a new transfer controller CFSToHPSSTransferController() with logic for handling single files vs directories using HPSS best practices. Moves create_sfapi_client() to the same level as transfer_controller.py, such that it can be easily accessed by multiple components. Includes new documentation in MkDocs for HPSS. Added an HPSS endpoint to config.yml. Updates orchestration/_tests/test_sfapi_flow.py to reflect the new location of create_sfapi_client(). --- config.yml | 5 + docs/mkdocs/docs/hpss.md | 179 ++++++++++++++ docs/mkdocs/mkdocs.yml | 1 + orchestration/_tests/test_sfapi_flow.py | 67 ++---- orchestration/flows/bl832/job_controller.py | 3 +- orchestration/flows/bl832/nersc.py | 70 +++--- orchestration/sfapi.py | 46 ++++ orchestration/transfer_controller.py | 250 +++++++++++++++++++- 8 files changed, 544 insertions(+), 77 deletions(-) create mode 100644 docs/mkdocs/docs/hpss.md create mode 100644 orchestration/sfapi.py diff --git a/config.yml b/config.yml index e9936adc..087ffbd7 100644 --- a/config.yml +++ b/config.yml @@ -107,6 +107,11 @@ globus: client_id: ${GLOBUS_CLIENT_ID} client_secret: ${GLOBUS_CLIENT_SECRET} +hpss_als_endpoint: + root_path: /home/a/alsdata + uri: nersc.gov + name: hpss_als + harbor_images832: recon_image: tomorecon_nersc_mpi_hdf5@sha256:cc098a2cfb6b1632ea872a202c66cb7566908da066fd8f8c123b92fa95c2a43c multires_image: tomorecon_nersc_mpi_hdf5@sha256:cc098a2cfb6b1632ea872a202c66cb7566908da066fd8f8c123b92fa95c2a43c diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md new file mode 100644 index 00000000..03681278 --- /dev/null +++ b/docs/mkdocs/docs/hpss.md @@ -0,0 +1,179 @@ +# Working with The High Performance Storage System (HPSS) at NERSC + +HPSS is the tape-based data storage system we use for long term storage of experimental data at the ALS. Tape storage, while it may seem antiquated, is still a very economical and secure medium for infrequently accessed data as tape does not need to be powered except for reading and writing. This requires certain considerations when working with this system. + +In `orchestration/transfer_controller.py` we have included two transfer classes for moving data from CFS to HPSS and vice versa (HPSS to CFS). We are following the [HPSS best practices](https://docs.nersc.gov/filesystems/HPSS-best-practices/) outlined in the NERSC documentation. + +HPSS is intended for long-term storage of data that is not frequently accessed, and users should aim for file sizes between 100 GB and 2 TB. Since HPSS is a tape system, we need to ensure storage and retrieval commands are done efficiently, as it is a mechanical process to load in a tape and then scroll to the correct region on the tape. + +While there are Globus endpoints for HPSS, the NERSC documentation recommends against it as there are certain conditions (i.e. network disconnection) that are not as robust as their recommended HPSS tools `hsi` and `htar`, which they say is the fastest approach. Together, these tools allow us to work with the HPSS filesystem and carefully bundle our projects into `tar` archives that are built directly on HPSS. + +## Working with `hsi` + +We use `hsi` for handling individual files on HPSS. [Here is the official NERSC documentation for `hsi`.](https://docs.nersc.gov/filesystems/hsi/) + + +**Login to HPSS using `hsi`** + +``` +nersc$ hsi +``` + + +**Common `hsi` commands** +``` +hsi ls: show the contents of your HPSS home directory +hsi mkdir [new_dir]: create a remote directory in your home +hsi put [local_file_name]: Transfer a single file into HPSS with the same name +hsi put -R [local_directory]: Transfer a directory tree into HPSS, creating sub-dirs when needed +hsi get [/path/to/hpss_file]: Transfer a single file from HPSS into the local directory without renaming +hsi rm [/path/to/hpss_file]: Prune a file from HPSS +hsi rm -r [/path/to/hpss_file]: Prune a directory from HPSS +hsi rmdir /path/to/my_hpss_dir/: Prune an empty directory + +``` + +**Examples** + +Find files that are more than 20 days old and redirects the output to the file temp.txt: + +``` +hsi -q "find . -ctime 20" > temp.txt 2>&1 +``` + +## Working with `htar` + +We can use `htar` to efficiently work with groups of files on HPSS. The basic syntax of `htar` is similar to the standard `tar` utility: + +``` +htar -{c|K|t|x|X} -f tarfile [directories] [files] + +-c : Create +-K : Verify existing tarfile in HPSS +-t : List +-x : Extract +-X : re-create the index file for an existing archive +``` + +You cannot add or append files to an existing htar file. The following examples [can also be found here](https://docs.nersc.gov/filesystems/htar/#htar-usage-examples). + +**Create an archive with a directory and file** + +``` +nersc$ htar -cvf archive.tar project_directory some_extra_file.json +``` +**List the contents of a `tar` archive** +``` +nersc$ htar -tf archive.tar +HTAR: drwx------ als/als 0 2010-09-24 14:24 project_directory/cool_scan1 +HTAR: -rwx------ als/als 9331200 2010-09-24 14:24 project_directory/cool_scan2 +HTAR: -rwx------ als/als 9331200 2010-09-24 14:24 project_directory/cool_scan3 +HTAR: -rwx------ als/als 9331200 2010-09-24 14:24 project_directory/cool_scan4 +HTAR: -rwx------ als/als 398552 2010-09-24 17:35 some_extra_file.json +HTAR: HTAR SUCCESSFUL + +``` + +**Extract the entire `htar` file** + +``` +htar -xvf archive.tar +``` + +**Extract a single file from `htar`** + +``` +htar -xvf archive.tar project_directory/cool_scan4 +``` + +**`-Hnostage` option** + +If your `htar` files are >100GB, and you only want to extract one or two small member files, you may find faster retrieval rates by skipping staging the file to the HPSS disk cache with `-Hnostage`. + +``` +htar -Hnostage -xvf archive.tar project_directory/cool_scan4 +``` + +## Transferring Data from CFS to HPSS + +NERSC provides a special `xfer` QOS ("Quality of Service") for interacting with HPSS, which we can use with our SFAPI Slurm job scripts. + +### Single Files + +We can transfer single files over to HPSS using `hsi put` in a Slurm script: + +**Example `hsi` transfer job** + +``` +#SBATCH --qos=xfer +#SBATCH -C cron +#SBATCH --time=12:00:00 +#SBATCH --job-name=my_transfer +#SBATCH --licenses=SCRATCH +#SBATCH --mem=20GB + +# Archive a user's project folder to HPSS +hsi put /global/cfs/cdirs/als/data_mover/8.3.2/raw/als_user_project_folder/cool_scan1.h5 +``` + +Notes: +- `xfer` jobs specifying -N nodes will be rejected at submission time. By default, `xfer` jobs get 2GB of memory allocated. The memory footprint scales somewhat with the size of the file, so if you're archiving larger files, you'll need to request more memory. You can do this by adding `#SBATCH --mem=XGB` to the above script (where X in the range of 5 - 10 GB is a good starting point for large files). +- NERSC users are at most allowed 15 concurrent `xfer` sessions, which can be used strategically for parallel transfers and reads. + + +### Multiple Files + +NERSC recommends that when serving many files smaller than 100 GB we use `htar` to bundle them together before archiving. Since individual scans within a project may not be this large, we try to archive all of the scans in a project into a single `tar` file. If projects end up being larger than 2 TB, we can create multiple `tar` files. + +One great part about `htar` is that it builds the archive directly on `HPSS`, so you do not need to worry about needing the additional storage allocation on the CFS side for the `tar` file. + +**Example `xfer` transfer job** +``` +#SBATCH --qos=xfer +#SBATCH -C cron +#SBATCH --time=12:00:00 +#SBATCH --job-name=my_transfer +#SBATCH --licenses=SCRATCH +#SBATCH --mem=100GB + +# Archive a user's project folder to HPSS +htar -cvf als_user_project.tar /global/cfs/cdirs/als/data_mover/8.3.2/raw/als_user_project_folder +``` + +## Transferring Data from HPSS to CFS + +At some point you may want to access data from HPSS. An important thing to consider is whether you need to access single or multiple files. + +You could extract an entire `htar` file + +``` +htar -xvf als_user_project_folder.tar +``` + +Or maybe a single file + +``` +htar -xvf als_user_project_folder.tar cool_scan1.h5 +``` + +## Prefect Flows for HPSS Transfers + +Most of the time we expect transfers to occur from CFS to HPSS on a scheduled basis, after users have completed scanning during their alotted beamtime. + +### Transfer to HPSS Implementation +**`orchestration/transfer_controller.py`: `CFSToHPSSTransferController()`** + +Input +Output + +### Transfer to CFS Implementation +**`orchestration/transfer_controller.py`: `HPSSToCFSTransferController()`** + +Input +Output + +## Update SciCat with HPSS file paths + +TBD + + diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 728a990e..68bcc959 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -17,6 +17,7 @@ nav: - Compute at NERSC: nersc832.md - Orchestration: orchestration.md - Configuration: configuration.md +- HPSS Tape Archive Access: hpss.md # - Troubleshooting: troubleshooting.md - Glossary: glossary.md - About: about.md diff --git a/orchestration/_tests/test_sfapi_flow.py b/orchestration/_tests/test_sfapi_flow.py index a1809686..49a169d6 100644 --- a/orchestration/_tests/test_sfapi_flow.py +++ b/orchestration/_tests/test_sfapi_flow.py @@ -37,9 +37,13 @@ def test_create_sfapi_client_success(): """ Test successful creation of the SFAPI client. """ - from orchestration.flows.bl832.nersc import NERSCTomographyHPCController + from orchestration.sfapi import create_sfapi_client + + # Define fake credential file paths + fake_client_id_path = "/path/to/client_id" + fake_client_secret_path = "/path/to/client_secret" - # Mock data for client_id and client_secret files + # Mock file contents mock_client_id = 'value' mock_client_secret = '{"key": "value"}' @@ -47,34 +51,26 @@ def test_create_sfapi_client_success(): mock_open_client_id = mock_open(read_data=mock_client_id) mock_open_client_secret = mock_open(read_data=mock_client_secret) - with patch("orchestration.flows.bl832.nersc.os.getenv") as mock_getenv, \ - patch("orchestration.flows.bl832.nersc.os.path.isfile") as mock_isfile, \ + with patch("orchestration.sfapi.os.path.isfile") as mock_isfile, \ patch("builtins.open", side_effect=[ mock_open_client_id.return_value, mock_open_client_secret.return_value ]), \ - patch("orchestration.flows.bl832.nersc.JsonWebKey.import_key") as mock_import_key, \ - patch("orchestration.flows.bl832.nersc.Client") as MockClient: - - # Mock environment variables - mock_getenv.side_effect = lambda x: { - "PATH_NERSC_CLIENT_ID": "/path/to/client_id", - "PATH_NERSC_PRI_KEY": "/path/to/client_secret" - }.get(x, None) + patch("orchestration.sfapi.JsonWebKey.import_key") as mock_import_key, \ + patch("orchestration.sfapi.Client") as MockClient: - # Mock file existence + # Simulate that both credential files exist mock_isfile.return_value = True - # Mock JsonWebKey.import_key to return a mock secret + # Mock key import to return a fake secret mock_import_key.return_value = "mock_secret" - # Create the client - client = NERSCTomographyHPCController.create_sfapi_client() + # Create the client using the provided fake paths + client = create_sfapi_client(fake_client_id_path, fake_client_secret_path) - # Assert that Client was instantiated with 'value' and 'mock_secret' + # Verify that Client was instantiated with the expected arguments MockClient.assert_called_once_with("value", "mock_secret") - - # Assert that the returned client is the mocked client + # Assert that the returned client is the mocked Client instance assert client == MockClient.return_value, "Client should be the mocked sfapi_client.Client instance" @@ -82,36 +78,25 @@ def test_create_sfapi_client_missing_paths(): """ Test creation of the SFAPI client with missing credential paths. """ - from orchestration.flows.bl832.nersc import NERSCTomographyHPCController + from orchestration.sfapi import create_sfapi_client - with patch("orchestration.flows.bl832.nersc.os.getenv", return_value=None): - with pytest.raises(ValueError, match="Missing NERSC credentials paths."): - NERSCTomographyHPCController.create_sfapi_client() + # Passing None for both paths should trigger a ValueError. + with pytest.raises(ValueError, match="Missing NERSC credentials paths."): + create_sfapi_client(None, None) def test_create_sfapi_client_missing_files(): """ Test creation of the SFAPI client with missing credential files. """ - with ( - # Mock environment variables - patch( - "orchestration.flows.bl832.nersc.os.getenv", - side_effect=lambda x: { - "PATH_NERSC_CLIENT_ID": "/path/to/client_id", - "PATH_NERSC_PRI_KEY": "/path/to/client_secret" - }.get(x, None) - ), - - # Mock file existence to simulate missing files - patch("orchestration.flows.bl832.nersc.os.path.isfile", return_value=False) - ): - # Import the module after applying patches to ensure mocks are in place - from orchestration.flows.bl832.nersc import NERSCTomographyHPCController - - # Expect a FileNotFoundError due to missing credential files + from orchestration.sfapi import create_sfapi_client + fake_client_id_path = "/path/to/client_id" + fake_client_secret_path = "/path/to/client_secret" + + # Simulate missing credential files by patching os.path.isfile to return False. + with patch("orchestration.sfapi.os.path.isfile", return_value=False): with pytest.raises(FileNotFoundError, match="NERSC credential files are missing."): - NERSCTomographyHPCController.create_sfapi_client() + create_sfapi_client(fake_client_id_path, fake_client_secret_path) # ---------------------------- # Fixture for Mocking SFAPI Client diff --git a/orchestration/flows/bl832/job_controller.py b/orchestration/flows/bl832/job_controller.py index b2ff064b..55522cd6 100644 --- a/orchestration/flows/bl832/job_controller.py +++ b/orchestration/flows/bl832/job_controller.py @@ -86,9 +86,10 @@ def get_controller( config=config ) elif hpc_type == HPC.NERSC: + from orchestration.sfapi import create_sfapi_client from orchestration.flows.bl832.nersc import NERSCTomographyHPCController return NERSCTomographyHPCController( - client=NERSCTomographyHPCController.create_sfapi_client(), + client=create_sfapi_client(), config=config ) elif hpc_type == HPC.OLCF: diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 05680ed0..92d8c97f 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -1,13 +1,13 @@ import datetime from dotenv import load_dotenv -import json +# import json import logging -import os +# import os from pathlib import Path import re import time -from authlib.jose import JsonWebKey +# from authlib.jose import JsonWebKey from prefect import flow, get_run_logger from prefect.blocks.system import JSON from sfapi_client import Client @@ -42,37 +42,39 @@ def __init__( TomographyHPCController.__init__(self, config) self.client = client - @staticmethod - def create_sfapi_client() -> Client: - """Create and return an NERSC client instance""" - - # When generating the SFAPI Key in Iris, make sure to select "asldev" as the user! - # Otherwise, the key will not have the necessary permissions to access the data. - client_id_path = os.getenv("PATH_NERSC_CLIENT_ID") - client_secret_path = os.getenv("PATH_NERSC_PRI_KEY") - - if not client_id_path or not client_secret_path: - logger.error("NERSC credentials paths are missing.") - raise ValueError("Missing NERSC credentials paths.") - if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): - logger.error("NERSC credential files are missing.") - raise FileNotFoundError("NERSC credential files are missing.") - - client_id = None - client_secret = None - with open(client_id_path, "r") as f: - client_id = f.read() - - with open(client_secret_path, "r") as f: - client_secret = JsonWebKey.import_key(json.loads(f.read())) - - try: - client = Client(client_id, client_secret) - logger.info("NERSC client created successfully.") - return client - except Exception as e: - logger.error(f"Failed to create NERSC client: {e}") - raise e + # Moved this method to orchestration/sfapi.py: + + # @staticmethod + # def create_sfapi_client() -> Client: + # """Create and return an NERSC client instance""" + + # # When generating the SFAPI Key in Iris, make sure to select "asldev" as the user! + # # Otherwise, the key will not have the necessary permissions to access the data. + # client_id_path = os.getenv("PATH_NERSC_CLIENT_ID") + # client_secret_path = os.getenv("PATH_NERSC_PRI_KEY") + + # if not client_id_path or not client_secret_path: + # logger.error("NERSC credentials paths are missing.") + # raise ValueError("Missing NERSC credentials paths.") + # if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): + # logger.error("NERSC credential files are missing.") + # raise FileNotFoundError("NERSC credential files are missing.") + + # client_id = None + # client_secret = None + # with open(client_id_path, "r") as f: + # client_id = f.read() + + # with open(client_secret_path, "r") as f: + # client_secret = JsonWebKey.import_key(json.loads(f.read())) + + # try: + # client = Client(client_id, client_secret) + # logger.info("NERSC client created successfully.") + # return client + # except Exception as e: + # logger.error(f"Failed to create NERSC client: {e}") + # raise e def reconstruct( self, diff --git a/orchestration/sfapi.py b/orchestration/sfapi.py new file mode 100644 index 00000000..2ea5dfb1 --- /dev/null +++ b/orchestration/sfapi.py @@ -0,0 +1,46 @@ +from dotenv import load_dotenv +import json +import logging +import os + +from authlib.jose import JsonWebKey +from sfapi_client import Client + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +load_dotenv() + + +def create_sfapi_client( + client_id_path: str, + client_secret_path: str, +) -> Client: + """Create and return an NERSC client instance""" + + # When generating the SFAPI Key in Iris, make sure to select "asldev" as the user! + # Otherwise, the key will not have the necessary permissions to access the data. + # client_id_path = os.getenv("PATH_NERSC_CLIENT_ID") + # client_secret_path = os.getenv("PATH_NERSC_PRI_KEY") + + if not client_id_path or not client_secret_path: + logger.error("NERSC credentials paths are missing.") + raise ValueError("Missing NERSC credentials paths.") + if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): + logger.error("NERSC credential files are missing.") + raise FileNotFoundError("NERSC credential files are missing.") + + client_id = None + client_secret = None + with open(client_id_path, "r") as f: + client_id = f.read() + + with open(client_secret_path, "r") as f: + client_secret = JsonWebKey.import_key(json.loads(f.read())) + + try: + client = Client(client_id, client_secret) + logger.info("NERSC client created successfully.") + return client + except Exception as e: + logger.error(f"Failed to create NERSC client: {e}") + raise e diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 09e82649..092ef861 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -4,11 +4,17 @@ import datetime import logging import os +from pathlib import Path +import re import time from typing import Generic, TypeVar, Optional import globus_sdk +from sfapi_client import Client +from sfapi_client.compute import Machine + +# We should have a more generic Config class that can be used across beamlines... from orchestration.flows.bl832.config import Config832 from orchestration.flows.bl832.job_controller import HPC from orchestration.globus.transfer import GlobusEndpoint, start_transfer @@ -76,6 +82,21 @@ def full_path( return f"{self.root_path.rstrip('/')}/{path_suffix}" +class HPSSEndpoint(TransferEndpoint): + """ + An HPSS endpoint. + + Args: + TransferEndpoint: Abstract class for endpoints. + """ + def __init__( + self, + name: str, + root_path: str + ) -> None: + super().__init__(name, root_path) + + Endpoint = TypeVar("Endpoint", bound=TransferEndpoint) @@ -318,7 +339,10 @@ def copy( class SimpleTransferController(TransferController[FileSystemEndpoint]): - def __init__(self, config: Config832) -> None: + def __init__( + self, + config: Config832 + ) -> None: super().__init__(config) """ Use a simple 'cp' command to move data within the same system. @@ -380,6 +404,216 @@ def copy( logger.info(f"Transfer process took {elapsed_time:.2f} seconds.") +class CFSToHPSSTransferController(TransferController[HPSSEndpoint]): + def __init__( + self, + client: Client, + config: Config832 + ) -> None: + super().__init__(config) + self.client = client + """ + Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. + + This transfer contoller requires the source to be a FileSystemEndpoint on CFS and the destination to be an HPSSEndpoint. + If you want to move data from somewhere else to HPSS, you will first need to transfer to CFS using a different controller, + and then to HPSS with this one. + + Args: + TransferController: Abstract class for transferring data. + """ + def copy( + self, + file_path: str = "", + source: FileSystemEndpoint = None, + destination: HPSSEndpoint = None, + ) -> bool: + """ + Copy a file from a CFS source endpoint to an HPSS destination endpoint. + + For a single file, the transfer is done using hsi (via hsi put). + For a directory, the transfer is performed with htar: + - If the directory's total size is less than 2TB, a single tar archive is created. + - If the size exceeds 2TB, the files are split into multiple tar archives, + each not exceeding the 2TB threshold. + + The data is saved on HPSS in the correct location using the destination's root path. + + + Args: + file_path (str): The path of the file or directory to copy. + source (FileSystemEndpoint): The CFS source endpoint. + destination (HPSSEndpoint): The HPSS destination endpoint. + """ + + logger.info("Transferring data from CFS to HPSS") + if not file_path or not source or not destination: + logger.error("Missing required parameters for CFSToHPSSTransferController.") + return False + + path = Path(file_path) + folder_name = path.parent.name + if not folder_name: + folder_name = "" + + file_name = f"{path.stem}" + + logger.info(f"File name: {file_name}") + logger.info(f"Folder name: {folder_name}") + + # Construct absolute source path as visible on Perlmutter + abs_source_path = source.full_path(file_path) + dest_root = destination.root_path + job_name_suffix = Path(abs_source_path).name + + logs_path = "/global/cfs/cdirs/als/data_mover/hpss_transfer_logs" + + # IMPORTANT: job script must be deindented to the leftmost column or it will fail immediately + # Note: If q=debug, there is no minimum time limit + # However, if q=preempt, there is a minimum time limit of 2 hours. Otherwise the job won't run. + # The realtime queue can only be used for select accounts (e.g. ALS) + job_script = f"""#!/bin/bash +#SBATCH -q xfer +#SBATCH -A als +#SBATCH -C cron +#SBATCH --time=12:00:00 +#SBATCH --job-name=transfer_to_HPSS_{job_name_suffix} +#SBATCH --output={logs_path}/%j.out +#SBATCH --error={logs_path}/%j.err +#SBATCH --licenses=SCRATCH +#SBATCH --mem=100GB + +set -euo pipefail +date + +# Define source and destination variables +SOURCE_PATH="{abs_source_path}" +DEST_ROOT="{dest_root}" +FOLDER_NAME=$(basename "$SOURCE_PATH") +DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" + +# Create destination directory if it doesn't exist on HPSS +echo "Checking if HPSS destination directory $DEST_PATH exists." +if hsi ls "$DEST_PATH" >/dev/null 2>&1; then + echo "Destination directory $DEST_PATH already exists." +else + echo "Destination directory $DEST_PATH does not exist. Creating it now." + hsi mkdir "$DEST_PATH" +fi + +# Check if source is a file or directory, and run the appropriate transfer command (hsi vs htar) + +# Case: Single File +if [ -f "$SOURCE_PATH" ]; then + echo "Single file detected. Transferring via hsi put." + FILE_NAME=$(basename "$SOURCE_PATH") + hsi put "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + +# Case: Directory +elif [ -d "$SOURCE_PATH" ]; then + # Check directory size and split into multiple archives if necessary + + echo "Directory detected. Calculating total size..." + TOTAL_SIZE=$(du -sb "$SOURCE_PATH" | awk '{{print $1}}') + THRESHOLD=2199023255552 # 2 TB in bytes + echo "Total size: $TOTAL_SIZE bytes" + + # If directory size is less than 2TB, archive directly with htar + if [ "$TOTAL_SIZE" -lt "$THRESHOLD" ]; then + echo "Directory size is under 2TB. Archiving with htar." + htar -cvf "${{DEST_PATH}}/${{FOLDER_NAME}}.tar" "$SOURCE_PATH" + + # If directory size exceeds 2TB, split the project into multiple archives that are less than 2TB each + else + echo "Directory size exceeds 2TB. Splitting into multiple archives." + FILE_LIST=$(mktemp) + find "$SOURCE_PATH" -type f > "$FILE_LIST" + chunk=1 + current_size=0 + current_files=() + while IFS= read -r file; do + size=$(stat -c%s "$file") + if (( current_size + size > THRESHOLD )); then + tar_archive="${{DEST_PATH}}/${{FOLDER_NAME}}_part${{chunk}}.tar" + echo "Creating archive $tar_archive with size $current_size bytes" + htar -cvf "$tar_archive" "${{current_files[@]}}" + current_files=() + current_size=0 + ((chunk++)) + fi + current_files+=("$file") + current_size=$(( current_size + size )) + done < "$FILE_LIST" + if [ ${{#current_files[@]}} -gt 0 ]; then + tar_archive="${{DEST_PATH}}/${{FOLDER_NAME}}_part${{chunk}}.tar" + echo "Creating final archive $tar_archive with size $current_size bytes" + htar -cvf "$tar_archive" "${{current_files[@]}}" + fi + rm "$FILE_LIST" + fi +else + echo "Error: $SOURCE_PATH is neither a file nor a directory." + exit 1 +fi + +date +""" + try: + logger.info("Submitting HPSS transfer job to Perlmutter.") + perlmutter = self.client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + logger.info(f"Submitted job ID: {job.jobid}") + + try: + job.update() + except Exception as update_err: + logger.warning(f"Initial job update failed, continuing: {update_err}") + + time.sleep(60) + logger.info(f"Job {job.jobid} current state: {job.state}") + + job.complete() # Wait until the job completes + logger.info("Reconstruction job completed successfully.") + return True + + except Exception as e: + logger.info(f"Error during job submission or completion: {e}") + match = re.search(r"Job not found:\s*(\d+)", str(e)) + + if match: + jobid = match.group(1) + logger.info(f"Attempting to recover job {jobid}.") + try: + job = self.client.perlmutter.job(jobid=jobid) + time.sleep(30) + job.complete() + logger.info("Reconstruction job completed successfully after recovery.") + return True + except Exception as recovery_err: + logger.error(f"Failed to recover job {jobid}: {recovery_err}") + return False + else: + # Unknown error: cannot recover + return False + + +class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): + def __init__( + self, + client: Client, + config: Config832 + ) -> None: + super().__init__(config) + """ + Use SFAPI to move data between CFS and HPSS at NERSC. + + Args: + TransferController: Abstract class for transferring data. + """ + + pass + + class CopyMethod(Enum): """ Enum representing different transfer methods. @@ -387,6 +621,8 @@ class CopyMethod(Enum): """ GLOBUS = "globus" SIMPLE = "simple" + CFS_TO_HPSS = "cfs_to_hpss" + HPSS_TO_CFS = "hpss_to_cfs" def get_transfer_controller( @@ -408,6 +644,18 @@ def get_transfer_controller( return GlobusTransferController(config, prometheus_metrics) elif transfer_type == CopyMethod.SIMPLE: return SimpleTransferController(config) + elif transfer_type == CopyMethod.CFS_TO_HPSS: + from orchestration.sfapi import create_sfapi_client + return CFSToHPSSTransferController( + client=create_sfapi_client(), + config=config + ) + elif transfer_type == CopyMethod.HPSS_TO_CFS: + from orchestration.sfapi import create_sfapi_client + return HPSSToCFSTransferController( + client=create_sfapi_client(), + config=config + ) else: raise ValueError(f"Invalid transfer type: {transfer_type}") From 1d102137c054684f782702f49b51b5be76424df6 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Feb 2025 11:53:12 -0800 Subject: [PATCH 002/128] Adding a generic BeamlineConfig(ABC) class in orchestration/config.py that is inherited by specific Config implementations such as orchestration/flows/bl832/config.py: Config832(). Updating typing in TransferControllers to look for the generic Config. Updated pytest cases. --- orchestration/_tests/test_sfapi_flow.py | 4 +- orchestration/config.py | 35 ++++++++++++ orchestration/flows/bl832/config.py | 19 ++++--- orchestration/transfer_controller.py | 74 ++++++++++++------------- 4 files changed, 86 insertions(+), 46 deletions(-) diff --git a/orchestration/_tests/test_sfapi_flow.py b/orchestration/_tests/test_sfapi_flow.py index 49a169d6..aa5a9d50 100644 --- a/orchestration/_tests/test_sfapi_flow.py +++ b/orchestration/_tests/test_sfapi_flow.py @@ -108,7 +108,7 @@ def mock_sfapi_client(): """ Mock the sfapi_client.Client class with necessary methods. """ - with patch("orchestration.flows.bl832.nersc.Client") as MockClient: + with patch("orchestration.sfapi.Client") as MockClient: mock_client_instance = MockClient.return_value # Mock the user method @@ -136,7 +136,7 @@ def mock_config832(): """ Mock the Config832 class to provide necessary configurations. """ - with patch("orchestration.flows.bl832.nersc.Config832") as MockConfig: + with patch("orchestration.flows.bl832.config.Config832") as MockConfig: mock_config = MockConfig.return_value mock_config.harbor_images832 = { "recon_image": "mock_recon_image", diff --git a/orchestration/config.py b/orchestration/config.py index 79083375..80861c9c 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import collections import builtins from pathlib import Path @@ -41,3 +42,37 @@ def expand_environment_variables(config): return type(config)([expand_environment_variables(v) for v in config]) else: return config + + +class BeamlineConfig(ABC): + """ + Base class for beamline configurations. + + This class reads the common configuration from disk, builds endpoints and apps, + and initializes the Globus Transfer and Flows clients. Beamline-specific subclasses + must override the _setup_specific_config() method to assign their own attributes. + + Attributes: + beamline_id (str): Beamline identifier (e.g. "832" or "733"). + config (dict): The loaded configuration dictionary. + endpoints (dict): Endpoints built from the configuration. + apps (dict): Apps built from the configuration. + tc (TransferClient): Globus Transfer client. + flow_client: Globus Flows client. + """ + + def __init__(self, beamline_id: str) -> None: + self.beamline_id = beamline_id + self.config = read_config() + self._beam_specific_config() + + @abstractmethod + def _beam_specific_config(self) -> None: + """ + Set up beamline-specific configuration attributes. + + This method must be implemented by subclasses. Typical assignments + include selecting endpoints (using keys that include the beamline ID), + and other beamline-specific parameters. + """ + pass diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index ff19a9c3..0ba40510 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -1,12 +1,17 @@ from globus_sdk import TransferClient -from orchestration.globus import transfer, flows +from orchestration.config import BeamlineConfig +from orchestration.globus import flows, transfer -class Config832: + +class Config832(BeamlineConfig): def __init__(self) -> None: - config = transfer.get_config() - self.endpoints = transfer.build_endpoints(config) - self.apps = transfer.build_apps(config) + super().__init__(beamline_id="832") + + def _beam_specific_config(self) -> None: + # config = transfer.get_config() + self.endpoints = transfer.build_endpoints(self.config) + self.apps = transfer.build_apps(self.config) self.tc: TransferClient = transfer.init_transfer_client(self.apps["als_transfer"]) self.flow_client = flows.get_flows_client() self.spot832 = self.endpoints["spot832"] @@ -22,5 +27,5 @@ def __init__(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] - self.scicat = config["scicat"] - self.ghcr_images832 = config["ghcr_images832"] + self.scicat = self.config["scicat"] + self.ghcr_images832 = self.config["ghcr_images832"] diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 092ef861..57f71113 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -13,10 +13,8 @@ from sfapi_client import Client from sfapi_client.compute import Machine - -# We should have a more generic Config class that can be used across beamlines... -from orchestration.flows.bl832.config import Config832 -from orchestration.flows.bl832.job_controller import HPC +# Import the generic Beamline configuration class. +from orchestration.config import BeamlineConfig from orchestration.globus.transfer import GlobusEndpoint, start_transfer from orchestration.prometheus_utils import PrometheusMetrics @@ -109,7 +107,7 @@ class TransferController(Generic[Endpoint], ABC): """ def __init__( self, - config: Config832 + config: BeamlineConfig ) -> None: self.config = config @@ -135,20 +133,21 @@ def copy( class GlobusTransferController(TransferController[GlobusEndpoint]): - """ - Use Globus Transfer to move data between endpoints. - - Args: - TransferController: Abstract class for transferring data. - """ def __init__( self, - config: Config832, + config: BeamlineConfig, prometheus_metrics: Optional[PrometheusMetrics] = None ) -> None: super().__init__(config) self.prometheus_metrics = prometheus_metrics + """ + Use Globus Transfer to move data between endpoints. + + Args: + TransferController: Abstract class for transferring data. + """ + def get_transfer_file_info( self, task_id: str, @@ -156,23 +155,23 @@ def get_transfer_file_info( ) -> Optional[dict]: """ Get information about a completed transfer from the Globus API. - + Args: task_id (str): The Globus transfer task ID transfer_client (TransferClient, optional): TransferClient instance - + Returns: Optional[dict]: Task information including bytes_transferred, or None if unavailable """ if transfer_client is None: transfer_client = self.config.tc - + try: task_info = transfer_client.get_task(task_id) task_dict = task_info.data if task_dict.get('status') == 'SUCCEEDED': - bytes_transferred = task_dict.get('bytes_transferred', 0) + bytes_transferred = task_dict.get('bytes_transferred', 0) bytes_checksummed = task_dict.get('bytes_checksummed', 0) files_transferred = task_dict.get('files_transferred', 0) effective_bytes_per_second = task_dict.get('effective_bytes_per_second', 0) @@ -182,13 +181,13 @@ def get_transfer_file_info( 'files_transferred': files_transferred, 'effective_bytes_per_second': effective_bytes_per_second } - + return None - + except Exception as e: logger.error(f"Error getting transfer task info: {e}") return None - + def collect_and_push_metrics( self, start_time: float, @@ -202,7 +201,7 @@ def collect_and_push_metrics( ) -> None: """ Collect transfer metrics and push them to Prometheus. - + Args: start_time (float): Transfer start time as UNIX timestamp. end_time (float): Transfer end time as UNIX timestamp. @@ -222,10 +221,10 @@ def collect_and_push_metrics( end_datetime = datetime.datetime.fromtimestamp(end_time, tz=datetime.timezone.utc) start_timestamp = start_datetime.isoformat() end_timestamp = end_datetime.isoformat() - + # Calculate duration in seconds duration_seconds = end_time - start_time - + # Calculate transfer speed (bytes per second) # transfer_speed = file_size / duration_seconds if duration_seconds > 0 and file_size > 0 else 0 @@ -241,13 +240,13 @@ def collect_and_push_metrics( "status": "success" if success else "failed", "machine": machine_name } - + # Push metrics to Prometheus self.prometheus_metrics.push_metrics_to_prometheus(metrics, logger) - + except Exception as e: logger.error(f"Error collecting or pushing metrics: {e}") - + def copy( self, file_path: str = None, @@ -265,11 +264,11 @@ def copy( Returns: bool: True if the transfer was successful, False otherwise. """ - + if not file_path: logger.error("No file_path provided") return False - + if not source or not destination: logger.error("Source or destination endpoint not provided") return False @@ -289,7 +288,7 @@ def copy( success = False task_id = None # Initialize task_id here to prevent UnboundLocalError file_size = 0 # Initialize file_size here as well - + try: success, task_id = start_transfer( transfer_client=self.config.tc, @@ -305,10 +304,10 @@ def copy( logger.info("Transfer completed successfully.") else: logger.error("Transfer failed.") - + except globus_sdk.services.transfer.errors.TransferAPIError as e: logger.error(f"Failed to submit transfer: {e}") - + finally: # Stop the timer and calculate the duration transfer_end_time = time.time() @@ -321,7 +320,7 @@ def copy( transfer_speed = transfer_info.get('effective_bytes_per_second', 0) logger.info(f"Globus Task Info: Transferred {file_size} bytes ") logger.info(f"Globus Task Info: Effective speed: {transfer_speed} bytes/second") - + # Collect and push metrics if enabled if self.prometheus_metrics and file_size > 0: self.collect_and_push_metrics( @@ -334,14 +333,14 @@ def copy( transfer_speed=transfer_speed, success=success, ) - + return success class SimpleTransferController(TransferController[FileSystemEndpoint]): def __init__( self, - config: Config832 + config: BeamlineConfig ) -> None: super().__init__(config) """ @@ -408,7 +407,7 @@ class CFSToHPSSTransferController(TransferController[HPSSEndpoint]): def __init__( self, client: Client, - config: Config832 + config: BeamlineConfig ) -> None: super().__init__(config) self.client = client @@ -601,7 +600,7 @@ class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): def __init__( self, client: Client, - config: Config832 + config: BeamlineConfig ) -> None: super().__init__(config) """ @@ -627,7 +626,7 @@ class CopyMethod(Enum): def get_transfer_controller( transfer_type: CopyMethod, - config: Config832, + config: BeamlineConfig, prometheus_metrics: Optional[PrometheusMetrics] = None ) -> TransferController: """ @@ -661,6 +660,7 @@ def get_transfer_controller( if __name__ == "__main__": + from orchestration.flows.bl832.config import Config832 config = Config832() transfer_type = CopyMethod.GLOBUS globus_transfer_controller = get_transfer_controller(transfer_type, config) @@ -680,4 +680,4 @@ def get_transfer_controller( if success: logger.info("Simple transfer succeeded.") else: - logger.error("Simple transfer failed.") \ No newline at end of file + logger.error("Simple transfer failed.") From 1a4e893504acf1ce1c8935f792adfdd17c21f7bb Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Feb 2025 14:04:05 -0800 Subject: [PATCH 003/128] Added logic for HPSSToCFSTransferController() copy() method. Now it will launch an SFAPI Slurm job script that handles each case: a single file is requested (non-tar), an entire tar file, or specified files within a tar (partial). Added pytest cases in test_transfer_controller.py for the new HPSS controllers. --- .../_tests/test_transfer_controller.py | 213 ++++++++++++++++++ orchestration/transfer_controller.py | 157 ++++++++++++- 2 files changed, 366 insertions(+), 4 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index a1cae916..5f356442 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -2,6 +2,7 @@ import pytest from pytest_mock import MockFixture +import time from unittest.mock import MagicMock, patch from uuid import uuid4 @@ -12,6 +13,14 @@ from .test_globus import MockTransferClient +<<<<<<< HEAD +======= +@pytest.fixture(autouse=True) +def fast_sleep(monkeypatch): + """Patch time.sleep to return immediately to speed up tests.""" + monkeypatch.setattr(time, "sleep", lambda x: None) + +>>>>>>> 67e515a (Added logic for HPSSToCFSTransferController() copy() method. Now it will launch an SFAPI Slurm job script that handles each case: a single file is requested (non-tar), an entire tar file, or specified files within a tar (partial). Added pytest cases in test_transfer_controller.py for the new HPSS controllers.) @pytest.fixture(autouse=True, scope="session") def prefect_test_fixture(): @@ -45,6 +54,9 @@ def transfer_controller_module(): FileSystemEndpoint, GlobusTransferController, SimpleTransferController, + CFSToHPSSTransferController, + HPSSToCFSTransferController, + HPSSEndpoint, get_transfer_controller, CopyMethod, ) @@ -52,6 +64,9 @@ def transfer_controller_module(): "FileSystemEndpoint": FileSystemEndpoint, "GlobusTransferController": GlobusTransferController, "SimpleTransferController": SimpleTransferController, + "CFSToHPSSTransferController": CFSToHPSSTransferController, + "HPSSToCFSTransferController": HPSSToCFSTransferController, + "HPSSEndpoint": HPSSEndpoint, "get_transfer_controller": get_transfer_controller, "CopyMethod": CopyMethod, } @@ -349,3 +364,201 @@ def test_simple_transfer_controller_copy_exception( assert result is False, "Expected False when an exception is raised during copy." mock_os_system.assert_called_once() + + +# -------------------------------------------------------------------------- +# Tests for CFSToHPSSTransferController +# -------------------------------------------------------------------------- + +def test_cfs_to_hpss_transfer_controller_success(mock_config832, transfer_controller_module, mocker: MockFixture): + """ + Test a successful copy() operation using CFSToHPSSTransferController. + We simulate a successful job submission and completion. + """ + CFSToHPSSTransferController = transfer_controller_module["CFSToHPSSTransferController"] + HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] + FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] + + # Create mock endpoints for source (CFS) and destination (HPSS) + source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source") + destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest") + + # Create a fake job object that simulates successful completion. + fake_job = MagicMock() + fake_job.jobid = "12345" + fake_job.state = "COMPLETED" + fake_job.complete.return_value = None + + # Create a fake compute object that returns the fake job. + fake_compute = MagicMock() + fake_compute.submit_job.return_value = fake_job + + # Create a fake client whose compute() returns our fake_compute. + fake_client = MagicMock() + fake_client.compute.return_value = fake_compute + + controller = CFSToHPSSTransferController(fake_client, mock_config832) + result = controller.copy( + file_path="test_dir/test_file.txt", + source=source_endpoint, + destination=destination_endpoint, + ) + assert result is True, "Expected True when CFSToHPSSTransferController transfer completes successfully." + fake_compute.submit_job.assert_called_once() + fake_job.complete.assert_called_once() + + +def test_cfs_to_hpss_transfer_controller_failure(mock_config832, transfer_controller_module): + """ + Test a failing copy() operation using CFSToHPSSTransferController when job submission raises an exception. + """ + CFSToHPSSTransferController = transfer_controller_module["CFSToHPSSTransferController"] + HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] + FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] + + source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source") + destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest") + + # Create a fake client whose compute().submit_job raises an exception. + fake_client = MagicMock() + fake_compute = MagicMock() + fake_compute.submit_job.side_effect = Exception("Job submission failed") + fake_client.compute.return_value = fake_compute + + controller = CFSToHPSSTransferController(fake_client, mock_config832) + result = controller.copy( + file_path="test_dir/test_file.txt", + source=source_endpoint, + destination=destination_endpoint, + ) + assert result is False, "Expected False when CFSToHPSSTransferController transfer fails due to job submission error." + fake_compute.submit_job.assert_called_once() + + +# -------------------------------------------------------------------------- +# Tests for HPSSToCFSTransferController +# -------------------------------------------------------------------------- + +def test_hpss_to_cfs_transfer_controller_success(mock_config832, transfer_controller_module, mocker: MockFixture): + """ + Test a successful copy() operation using HPSSToCFSTransferController. + We simulate a successful job submission and completion. + """ + HPSSToCFSTransferController = transfer_controller_module["HPSSToCFSTransferController"] + HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] + FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] + + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + + # Create a fake job object for a successful transfer. + fake_job = MagicMock() + fake_job.jobid = "67890" + fake_job.state = "COMPLETED" + fake_job.complete.return_value = None + + fake_compute = MagicMock() + fake_compute.submit_job.return_value = fake_job + + fake_client = MagicMock() + fake_client.compute.return_value = fake_compute + + controller = HPSSToCFSTransferController(fake_client, mock_config832) + result = controller.copy( + file_path="archive.tar", + source=source_endpoint, + destination=destination_endpoint, + files_to_extract=["file1.txt", "file2.txt"] + ) + assert result is True, "Expected True when HPSSToCFSTransferController transfer completes successfully." + fake_compute.submit_job.assert_called_once() + fake_job.complete.assert_called_once() + + +def test_hpss_to_cfs_transfer_controller_missing_params(mock_config832, transfer_controller_module): + """ + Test that HPSSToCFSTransferController.copy() returns False when required parameters are missing. + """ + HPSSToCFSTransferController = transfer_controller_module["HPSSToCFSTransferController"] + fake_client = MagicMock() # Client is not used because the method returns early. + controller = HPSSToCFSTransferController(fake_client, mock_config832) + + result = controller.copy(file_path=None, source=None, destination=None) + assert result is False, "Expected False when required parameters are missing." + + +def test_hpss_to_cfs_transfer_controller_job_failure(mock_config832, transfer_controller_module): + """ + Test HPSSToCFSTransferController.transfer() returns False when job.complete() raises an exception. + """ + HPSSToCFSTransferController = transfer_controller_module["HPSSToCFSTransferController"] + HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] + FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] + + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + + fake_job = MagicMock() + fake_job.jobid = "67891" + fake_job.state = "FAILED" + fake_job.complete.side_effect = Exception("Job completion failed") + + fake_compute = MagicMock() + fake_compute.submit_job.return_value = fake_job + + fake_client = MagicMock() + fake_client.compute.return_value = fake_compute + + controller = HPSSToCFSTransferController(fake_client, mock_config832) + result = controller.copy( + file_path="archive.tar", + source=source_endpoint, + destination=destination_endpoint, + ) + assert result is False, "Expected False when HPSSToCFSTransferController job fails to complete." + fake_compute.submit_job.assert_called_once() + fake_job.complete.assert_called_once() + + +def test_hpss_to_cfs_transfer_controller_recovery(mock_config832, transfer_controller_module): + """ + Test HPSSToCFSTransferController recovery scenario when initial job.complete() fails with 'Job not found:'. + The controller should attempt to recover the job and complete successfully. + """ + HPSSToCFSTransferController = transfer_controller_module["HPSSToCFSTransferController"] + HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] + FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] + + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + + # Fake job that fails initially with a "Job not found:" error. + fake_job_initial = MagicMock() + fake_job_initial.jobid = "11111" + fake_job_initial.state = "UNKNOWN" + fake_job_initial.complete.side_effect = Exception("Job not found: 11111") + + fake_compute = MagicMock() + fake_compute.submit_job.return_value = fake_job_initial + + # When recovery is attempted, return a job that completes successfully. + fake_job_recovered = MagicMock() + fake_job_recovered.jobid = "11111" + fake_job_recovered.state = "COMPLETED" + fake_job_recovered.complete.return_value = None + + fake_client = MagicMock() + fake_client.compute.return_value = fake_compute + fake_client.perlmutter.job.return_value = fake_job_recovered + + controller = HPSSToCFSTransferController(fake_client, mock_config832) + result = controller.copy( + file_path="archive.tar", + source=source_endpoint, + destination=destination_endpoint, + ) + assert result is True, "Expected True after successful job recovery in HPSSToCFSTransferController." + fake_compute.submit_job.assert_called_once() + fake_job_initial.complete.assert_called_once() + fake_client.perlmutter.job.assert_called_once_with(jobid="11111") + fake_job_recovered.complete.assert_called_once() diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 57f71113..676876ad 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -7,7 +7,7 @@ from pathlib import Path import re import time -from typing import Generic, TypeVar, Optional +from typing import Generic, List, Optional, TypeVar import globus_sdk from sfapi_client import Client @@ -94,6 +94,21 @@ def __init__( ) -> None: super().__init__(name, root_path) + def full_path(self, path_suffix: str) -> str: + """ + Constructs the full path by appending the path_suffix to the HPSS endpoint's root_path. + This is used by the HPSS transfer controllers to compute the absolute path on HPSS. + + Args: + path_suffix (str): The relative path to append. + + Returns: + str: The full absolute path. + """ + if path_suffix.startswith("/"): + path_suffix = path_suffix[1:] + return f"{self.root_path.rstrip('/')}/{path_suffix}" + Endpoint = TypeVar("Endpoint", bound=TransferEndpoint) @@ -603,14 +618,148 @@ def __init__( config: BeamlineConfig ) -> None: super().__init__(config) + self.client = client """ - Use SFAPI to move data between CFS and HPSS at NERSC. + Use SFAPI to move data between HPSS and CFS at NERSC. + + This controller retrieves data from an HPSS source endpoint and places it on a CFS destination endpoint. + It supports: + - Single file retrieval via hsi get. + - Full tar archive extraction via htar -xvf. + - Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), + only the specified files will be extracted. + A single SLURM job script is generated that branches based on the mode. Args: - TransferController: Abstract class for transferring data. + file_path (str): Path to the file or tar archive on HPSS. + source (HPSSEndpoint): The HPSS source endpoint. + destination (FileSystemEndpoint): The CFS destination endpoint. + files_to_extract (List[str], optional): Specific files to extract from the tar archive. + If provided (and file_path ends with '.tar'), only these files will be extracted. + + Returns: + bool: True if the transfer job completes successfully, False otherwise. """ + def copy( + self, + file_path: str = None, + source: HPSSEndpoint = None, + destination: FileSystemEndpoint = None, + files_to_extract: Optional[List[str]] = None, + ) -> bool: + logger.info("Starting HPSS to CFS transfer.") + if not file_path or not source or not destination: + logger.error("Missing required parameters: file_path, source, or destination.") + return False + + # Set the job name suffix based on the file name (or archive stem) + job_name_suffix = Path(file_path).stem + + # Compute the full HPSS path from the source endpoint. + hpss_path = source.full_path(file_path) + dest_root = destination.root_path + logs_path = "/global/cfs/cdirs/als/data_mover/hpss_transfer_logs" + + # If files_to_extract is provided, join them as a spaceā€separated string. + files_to_extract_str = " ".join(files_to_extract) if files_to_extract else "" + + # The following SLURM script contains all logic to decide the transfer mode. + # It determines: + # - if HPSS_PATH ends with .tar, then if FILES_TO_EXTRACT is nonempty, MODE becomes "partial", + # else MODE is "tar". + # - Otherwise, MODE is "single" and hsi get is used. + job_script = fr"""#!/bin/bash +#SBATCH -q xfer +#SBATCH -A als +#SBATCH -C cron +#SBATCH --time=12:00:00 +#SBATCH --job-name=transfer_from_HPSS_{job_name_suffix} +#SBATCH --output={logs_path}/%j.out +#SBATCH --error={logs_path}/%j.err +#SBATCH --licenses=SCRATCH +#SBATCH --mem=100GB + +set -euo pipefail +date + +# Environment variables provided by Python. +HPSS_PATH="{hpss_path}" +DEST_ROOT="{dest_root}" +FILES_TO_EXTRACT="{files_to_extract_str}" + +# Determine the transfer mode in bash. +if [[ "$HPSS_PATH" =~ \.tar$ ]]; then + if [ -n "${{FILES_TO_EXTRACT}}" ]; then + MODE="partial" + else + MODE="tar" + fi +else + MODE="single" +fi + +echo "Transfer mode: $MODE" +if [ "$MODE" = "single" ]; then + echo "Single file detected. Using hsi get." + mkdir -p "$DEST_ROOT" + hsi get "$HPSS_PATH" "$DEST_ROOT/" +elif [ "$MODE" = "tar" ]; then + echo "Tar archive detected. Extracting entire archive using htar." + ARCHIVE_BASENAME=$(basename "$HPSS_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + mkdir -p "$DEST_PATH" + htar -xvf "$HPSS_PATH" -C "$DEST_PATH" +elif [ "$MODE" = "partial" ]; then + echo "Partial extraction detected. Extracting selected files using htar." + ARCHIVE_BASENAME=$(basename "$HPSS_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + mkdir -p "$DEST_PATH" + echo "Files to extract: $FILES_TO_EXTRACT" + htar -xvf "$HPSS_PATH" -C "$DEST_PATH" $FILES_TO_EXTRACT +else + echo "Error: Unknown mode: $MODE" + exit 1 +fi + +date +""" + logger.info("Submitting HPSS to CFS transfer job to Perlmutter.") + try: + perlmutter = self.client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + logger.info(f"Submitted job ID: {job.jobid}") + + try: + job.update() + except Exception as update_err: + logger.warning(f"Initial job update failed, continuing: {update_err}") + + time.sleep(60) + logger.info(f"Job {job.jobid} current state: {job.state}") + + job.complete() # Wait until the job completes. + logger.info("HPSS to CFS transfer job completed successfully.") + return True - pass + except Exception as e: + logger.error(f"Error during job submission or completion: {e}") + match = re.search(r"Job not found:\s*(\d+)", str(e)) + if match: + jobid = match.group(1) + logger.info(f"Attempting to recover job {jobid}.") + try: + job = self.client.perlmutter.job(jobid=jobid) + time.sleep(30) + job.complete() + logger.info("HPSS to CFS transfer job completed successfully after recovery.") + return True + except Exception as recovery_err: + logger.error(f"Failed to recover job {jobid}: {recovery_err}") + return False + else: + return False class CopyMethod(Enum): From cfa5a0c52843ec6fba56396a81e6a59ac86c3239 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Feb 2025 15:22:46 -0800 Subject: [PATCH 004/128] Moving endpoint definitions to orchestration/transfer_endpoint.py so they can be easily used by both transfer_controller.py and the new prune_controller.py file. Prune_controller.py follows the same ABC pattern, uses the same transfer_endpoint definitions, and defines a prune() function for HPSSPruneController, FileSystemPruneController, and GlobusPruneController. Only GlobusPruneController is implemented, however, this approach should greatly simplify our pruning code throughout the project and significantly reduce the number of pruning deployments we register in Prefect. --- orchestration/_tests/test_prune_controller.py | 202 ++++++++++++++++++ orchestration/config.py | 9 +- orchestration/prune_controller.py | 149 +++++++++++++ orchestration/transfer_controller.py | 88 +------- orchestration/transfer_endpoints.py | 88 ++++++++ 5 files changed, 444 insertions(+), 92 deletions(-) create mode 100644 orchestration/_tests/test_prune_controller.py create mode 100644 orchestration/prune_controller.py create mode 100644 orchestration/transfer_endpoints.py diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py new file mode 100644 index 00000000..9e6ef35b --- /dev/null +++ b/orchestration/_tests/test_prune_controller.py @@ -0,0 +1,202 @@ +import os +import shutil +from datetime import datetime, timedelta +from pathlib import Path + +import pytest + +# Import the abstract PruneController base +from orchestration.prune_controller import PruneController + + +############################################################################### +# Dummy Implementation for Testing +############################################################################### +class DummyPruneController(PruneController): + """ + A concrete dummy implementation of PruneController for testing purposes. + + This class uses a local directory (self.base_dir) to simulate file pruning. + It provides additional helper methods: + - get_files_to_delete(retention): returns files older than the given retention period. + - prune_files(retention): deletes files older than the given retention period. + """ + def __init__(self, base_dir: Path): + # Create a dummy configuration object (the real config isnā€™t used in these tests) + dummy_config = type("DummyConfig", (), {})() + dummy_config.tc = None + super().__init__(dummy_config) + self.base_dir = base_dir + + def get_files_to_delete(self, retention: int): + """ + Return a list of files in self.base_dir whose modification time is older than + 'retention' days. + + Args: + retention (int): Retention period in days. Must be > 0. + + Returns: + List[Path]: Files older than the retention period. + + Raises: + ValueError: If retention is not positive. + """ + if retention <= 0: + raise ValueError("Retention must be a positive number of days") + now_ts = datetime.now().timestamp() + retention_seconds = retention * 24 * 3600 + files_to_delete = [] + for f in self.base_dir.glob("*"): + if f.is_file(): + mod_time = f.stat().st_mtime + if now_ts - mod_time > retention_seconds: + files_to_delete.append(f) + return files_to_delete + + def prune_files(self, retention: int): + """ + Delete files older than 'retention' days and return the list of deleted files. + + Args: + retention (int): Retention period in days. Must be > 0. + + Returns: + List[Path]: List of files that were deleted. + + Raises: + ValueError: If retention is not positive. + """ + if retention <= 0: + raise ValueError("Retention must be a positive number of days") + files_to_delete = self.get_files_to_delete(retention) + for f in files_to_delete: + f.unlink() # Delete the file + return files_to_delete + + def prune( + self, + file_path: str = None, + source_endpoint=None, + check_endpoint=None, + days_from_now: timedelta = timedelta(0) + ) -> bool: + """ + Dummy implementation of the abstract method. + (Not used in these tests.) + """ + return True + + +############################################################################### +# Pytest Fixtures +############################################################################### +@pytest.fixture +def test_dir(): + """ + Fixture that creates (and later cleans up) a temporary directory for tests. + """ + test_path = Path("test_prune_data") + test_path.mkdir(exist_ok=True) + yield test_path + if test_path.exists(): + shutil.rmtree(test_path) + + +@pytest.fixture +def prune_controller(test_dir): + """ + Fixture that returns an instance of DummyPruneController using the temporary directory. + """ + return DummyPruneController(base_dir=test_dir) + + +############################################################################### +# Helper Function for Creating Test Files +############################################################################### +def create_test_files(directory: Path, dates): + """ + Create test files in the specified directory with modification times given by `dates`. + + Args: + directory (Path): The directory in which to create files. + dates (List[datetime]): List of datetimes to set as the file's modification time. + + Returns: + List[Path]: List of created file paths. + """ + files = [] + for date in dates: + filepath = directory / f"test_file_{date.strftime('%Y%m%d')}.txt" + filepath.touch() # Create the empty file + os.utime(filepath, (date.timestamp(), date.timestamp())) + files.append(filepath) + return files + + +############################################################################### +# Tests +############################################################################### +def test_prune_controller_initialization(prune_controller): + from orchestration.prune_controller import PruneController + # Verify that our dummy controller is an instance of the abstract base class + assert isinstance(prune_controller, PruneController) + # And that the base directory exists + assert prune_controller.base_dir.exists() + + +def test_get_files_to_delete(prune_controller, test_dir): + # Create test files with various modification times. + now = datetime.now() + dates = [ + now - timedelta(days=31), # Old enough to be pruned + now - timedelta(days=20), # Recent + now - timedelta(days=40), # Old enough to be pruned + now - timedelta(days=10), # Recent + ] + test_files = create_test_files(test_dir, dates) + + # When using a 30-day retention, the two older files should be flagged. + files_to_delete = prune_controller.get_files_to_delete(30) + assert len(files_to_delete) == 2 + + # Check that the names of the older files are in the returned list. + file_names_to_delete = {f.name for f in files_to_delete} + assert test_files[0].name in file_names_to_delete + assert test_files[2].name in file_names_to_delete + + +def test_prune_files(prune_controller, test_dir): + # Create two files: one older than 30 days and one newer. + now = datetime.now() + dates = [ + now - timedelta(days=31), # Should be deleted + now - timedelta(days=20), # Should remain + ] + test_files = create_test_files(test_dir, dates) + + # Prune files older than 30 days. + deleted_files = prune_controller.prune_files(30) + # One file should have been deleted. + assert len(deleted_files) == 1 + # The older file should no longer exist. + assert not test_files[0].exists() + # The newer file should still exist. + assert test_files[1].exists() + + +def test_empty_directory(prune_controller): + # Ensure the test directory is empty. + for f in list(prune_controller.base_dir.glob("*")): + f.unlink() + deleted_files = prune_controller.prune_files(30) + # There should be no files to delete. + assert len(deleted_files) == 0 + + +def test_invalid_retention_period(prune_controller): + # Using retention periods <= 0 should raise a ValueError. + with pytest.raises(ValueError): + prune_controller.prune_files(-1) + with pytest.raises(ValueError): + prune_controller.prune_files(0) diff --git a/orchestration/config.py b/orchestration/config.py index 80861c9c..40663aee 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -55,13 +55,12 @@ class BeamlineConfig(ABC): Attributes: beamline_id (str): Beamline identifier (e.g. "832" or "733"). config (dict): The loaded configuration dictionary. - endpoints (dict): Endpoints built from the configuration. - apps (dict): Apps built from the configuration. - tc (TransferClient): Globus Transfer client. - flow_client: Globus Flows client. """ - def __init__(self, beamline_id: str) -> None: + def __init__( + self, + beamline_id: str + ) -> None: self.beamline_id = beamline_id self.config = read_config() self._beam_specific_config() diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py new file mode 100644 index 00000000..bb0ca63f --- /dev/null +++ b/orchestration/prune_controller.py @@ -0,0 +1,149 @@ +from abc import ABC, abstractmethod +import datetime +import logging +from typing import Generic, TypeVar, Union + +from prefect import flow +from prefect.blocks.system import JSON + +from orchestration.config import BeamlineConfig +from orchestration.globus.transfer import GlobusEndpoint, prune_one_safe +from orchestration.prefect import schedule_prefect_flow +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint, TransferEndpoint + + +logger = logging.getLogger(__name__) + +Endpoint = TypeVar("Endpoint", bound=TransferEndpoint) + + +class PruneController(Generic[Endpoint], ABC): + """ + Abstract class for pruning controllers. + Provides interface methods for pruning data. + """ + def __init__( + self, + config: BeamlineConfig, + ) -> None: + self.config = config + + @abstractmethod + def prune( + self, + file_path: str = None, + source_endpoint: Endpoint = None, + check_endpoint: Endpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + """Prune data from the source endpoint. + + Args: + file_path (str): The path to the file to prune. + source (Endpoint): The source endpoint. + destination (Endpoint): The destination endpoint. + days_from_now (datetime.timedelta): The number of days from now to prune. Defaults to 0. + + Returns: + bool: True if successful, False otherwise. + """ + pass + + +class HPSSPruneController(PruneController[HPSSEndpoint]): + def __init__( + self, + config: BeamlineConfig, + ) -> None: + super().__init__(config) + + def prune( + self, + file_path: str = None, + source_endpoint: HPSSEndpoint = None, + check_endpoint: FileSystemEndpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + pass + + +class FileSystemPruneController(PruneController[FileSystemEndpoint]): + def __init__( + self, + config + ) -> None: + super().__init__(config) + + def prune( + self, + file_path: str = None, + source_endpoint: FileSystemEndpoint = None, + check_endpoint: FileSystemEndpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + pass + + +class GlobusPruneController(PruneController[GlobusEndpoint]): + def __init__( + self, + config + ) -> None: + super().__init__(config) + + def prune( + self, + file_path: str = None, + source_endpoint: GlobusEndpoint = None, + check_endpoint: GlobusEndpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + + # globus_settings = JSON.load("globus-settings").value + # max_wait_seconds = globus_settings["max_wait_seconds"] + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + schedule_prefect_flow( + "prune_globus_endpoint/prune_globus_endpoint", + flow_name, + { + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + }, + + datetime.timedelta(days=days_from_now), + ) + return True + + +@flow(name="prune_globus_endpoint") +def prune_globus_files( + relative_path: str, + source_endpoint: GlobusEndpoint, + check_endpoint: Union[GlobusEndpoint, None] = None, + config=None +): + """ + Prune files from a source endpoint. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. + check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + globus_settings = JSON.load("globus-settings").value + max_wait_seconds = globus_settings["max_wait_seconds"] + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + prune_one_safe( + file=relative_path, + if_older_than_days=0, + tranfer_client=config.tc, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + logger=logger, + max_wait_seconds=max_wait_seconds + ) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 676876ad..13a04f4e 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -17,99 +17,13 @@ from orchestration.config import BeamlineConfig from orchestration.globus.transfer import GlobusEndpoint, start_transfer from orchestration.prometheus_utils import PrometheusMetrics +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint, TransferEndpoint logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) load_dotenv() -class TransferEndpoint(ABC): - """ - Abstract base class for endpoints. - """ - def __init__( - self, - name: str, - root_path: str - ) -> None: - self.name = name - self.root_path = root_path - - def name(self) -> str: - """ - A human-readable or reference name for the endpoint. - """ - return self.name - - def root_path(self) -> str: - """ - Root path or base directory for this endpoint. - """ - return self.root_path - - -class FileSystemEndpoint(TransferEndpoint): - """ - A file system endpoint. - - Args: - TransferEndpoint: Abstract class for endpoints. - """ - def __init__( - self, - name: str, - root_path: str - ) -> None: - super().__init__(name, root_path) - - def full_path( - self, - path_suffix: str - ) -> str: - """ - Constructs the full path by appending the path_suffix to the root_path. - - Args: - path_suffix (str): The relative path to append. - - Returns: - str: The full absolute path. - """ - if path_suffix.startswith("/"): - path_suffix = path_suffix[1:] - return f"{self.root_path.rstrip('/')}/{path_suffix}" - - -class HPSSEndpoint(TransferEndpoint): - """ - An HPSS endpoint. - - Args: - TransferEndpoint: Abstract class for endpoints. - """ - def __init__( - self, - name: str, - root_path: str - ) -> None: - super().__init__(name, root_path) - - def full_path(self, path_suffix: str) -> str: - """ - Constructs the full path by appending the path_suffix to the HPSS endpoint's root_path. - This is used by the HPSS transfer controllers to compute the absolute path on HPSS. - - Args: - path_suffix (str): The relative path to append. - - Returns: - str: The full absolute path. - """ - if path_suffix.startswith("/"): - path_suffix = path_suffix[1:] - return f"{self.root_path.rstrip('/')}/{path_suffix}" - - Endpoint = TypeVar("Endpoint", bound=TransferEndpoint) diff --git a/orchestration/transfer_endpoints.py b/orchestration/transfer_endpoints.py new file mode 100644 index 00000000..8fbae8fa --- /dev/null +++ b/orchestration/transfer_endpoints.py @@ -0,0 +1,88 @@ +from abc import ABC + + +class TransferEndpoint(ABC): + """ + Abstract base class for endpoints. + """ + def __init__( + self, + name: str, + root_path: str + ) -> None: + self.name = name + self.root_path = root_path + + def name(self) -> str: + """ + A human-readable or reference name for the endpoint. + """ + return self.name + + def root_path(self) -> str: + """ + Root path or base directory for this endpoint. + """ + return self.root_path + + +class FileSystemEndpoint(TransferEndpoint): + """ + A file system endpoint. + + Args: + TransferEndpoint: Abstract class for endpoints. + """ + def __init__( + self, + name: str, + root_path: str + ) -> None: + super().__init__(name, root_path) + + def full_path( + self, + path_suffix: str + ) -> str: + """ + Constructs the full path by appending the path_suffix to the root_path. + + Args: + path_suffix (str): The relative path to append. + + Returns: + str: The full absolute path. + """ + if path_suffix.startswith("/"): + path_suffix = path_suffix[1:] + return f"{self.root_path.rstrip('/')}/{path_suffix}" + + +class HPSSEndpoint(TransferEndpoint): + """ + An HPSS endpoint. + + Args: + TransferEndpoint: Abstract class for endpoints. + """ + def __init__( + self, + name: str, + root_path: str + ) -> None: + super().__init__(name, root_path) + + def full_path(self, path_suffix: str) -> str: + """ + Constructs the full path by appending the path_suffix to the HPSS endpoint's root_path. + This is used by the HPSS transfer controllers to compute the absolute path on HPSS. + + Args: + path_suffix (str): The relative path to append. + + Returns: + str: The full absolute path. + """ + if path_suffix.startswith("/"): + path_suffix = path_suffix[1:] + return f"{self.root_path.rstrip('/')}/{path_suffix}" From 92c2eb6272e2f463219bdcdd0e9c2ce84707edf5 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Feb 2025 15:39:44 -0800 Subject: [PATCH 005/128] Added a get_prune_controller() method. --- orchestration/prune_controller.py | 87 +++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index bb0ca63f..74b82a29 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -64,7 +64,22 @@ def prune( check_endpoint: FileSystemEndpoint = None, days_from_now: datetime.timedelta = 0 ) -> bool: - pass + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + schedule_prefect_flow( + "prune_hpss_endpoint/prune_hpss_endpoint", + flow_name, + { + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + + datetime.timedelta(days=days_from_now), + ) + return True class FileSystemPruneController(PruneController[FileSystemEndpoint]): @@ -81,7 +96,22 @@ def prune( check_endpoint: FileSystemEndpoint = None, days_from_now: datetime.timedelta = 0 ) -> bool: - pass + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + schedule_prefect_flow( + "prune_filesystem_endpoint/prune_filesystem_endpoint", + flow_name, + { + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + + datetime.timedelta(days=days_from_now), + ) + return True class GlobusPruneController(PruneController[GlobusEndpoint]): @@ -111,6 +141,7 @@ def prune( "relative_path": file_path, "source_endpoint": source_endpoint, "check_endpoint": check_endpoint, + "config": self.config }, datetime.timedelta(days=days_from_now), @@ -118,12 +149,26 @@ def prune( return True +def get_prune_controller( + endpoint: TransferEndpoint, + config: BeamlineConfig +) -> PruneController: + if isinstance(endpoint, HPSSEndpoint): + return HPSSPruneController(config) + elif isinstance(endpoint, FileSystemEndpoint): + return FileSystemPruneController(config) + elif isinstance(endpoint, GlobusEndpoint): + return GlobusPruneController(config) + else: + raise ValueError(f"Unsupported endpoint type: {type(endpoint)}") + + @flow(name="prune_globus_endpoint") def prune_globus_files( relative_path: str, source_endpoint: GlobusEndpoint, check_endpoint: Union[GlobusEndpoint, None] = None, - config=None + config: BeamlineConfig = None ): """ Prune files from a source endpoint. @@ -147,3 +192,39 @@ def prune_globus_files( logger=logger, max_wait_seconds=max_wait_seconds ) + + +@flow(name="prune_filesystem_endpoint") +def prune_filesystem_files( + relative_path: str, + source_endpoint: FileSystemEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None +): + """ + Prune files from a source endpoint. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. + check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + pass + + +@flow(name="prune_hpss_endpoint") +def prune_hpss_files( + relative_path: str, + source_endpoint: HPSSEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None +): + """ + Prune files from a source endpoint. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. + check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + pass From 3a02357d6c5f2fe005a94ac9b39a5cdf9d8727c0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 14 Feb 2025 14:10:22 -0800 Subject: [PATCH 006/128] Getting ready to test CFSToHPSSTransferController flow. Created a Prefect deployment script for the HPSS flow. Added logic to structure the filename in the htar according to year and beamcycle, added additional logging to slurm script. Commented out the htar and hsi commands to verify in the logs that things are happening as expected. Once that's confirmed, I'll test a real transfer. --- config.yml | 6 +- create_deployment_832_hpss.sh | 7 + docs/mkdocs/docs/733.md | 141 ++++++++++ docs/mkdocs/docs/tomography_workflow.md | 129 +++++++++ orchestration/flows/bl832/config.py | 3 +- orchestration/flows/bl832/hpss.py | 108 ++++++++ orchestration/flows/bl832/nersc.py | 34 --- orchestration/sfapi.py | 7 +- orchestration/transfer_controller.py | 330 +++++++++++++++--------- 9 files changed, 598 insertions(+), 167 deletions(-) create mode 100644 create_deployment_832_hpss.sh create mode 100644 docs/mkdocs/docs/733.md create mode 100644 docs/mkdocs/docs/tomography_workflow.md create mode 100644 orchestration/flows/bl832/hpss.py diff --git a/config.yml b/config.yml index 087ffbd7..d17d856f 100644 --- a/config.yml +++ b/config.yml @@ -107,10 +107,10 @@ globus: client_id: ${GLOBUS_CLIENT_ID} client_secret: ${GLOBUS_CLIENT_SECRET} -hpss_als_endpoint: - root_path: /home/a/alsdata +hpss_alsdev: + root_path: /home/a/alsdev/data_mover uri: nersc.gov - name: hpss_als + name: hpss_alsdev harbor_images832: recon_image: tomorecon_nersc_mpi_hdf5@sha256:cc098a2cfb6b1632ea872a202c66cb7566908da066fd8f8c123b92fa95c2a43c diff --git a/create_deployment_832_hpss.sh b/create_deployment_832_hpss.sh new file mode 100644 index 00000000..0da5a38a --- /dev/null +++ b/create_deployment_832_hpss.sh @@ -0,0 +1,7 @@ +export $(grep -v '^#' .env | xargs) + + +prefect work-pool create 'hpss_pool' + +prefect deployment build ./orchestration/flows/bl832/hpss.py:cfs_to_hpss_flow -n cfs_to_hpss_flow -q cfs_to_hpss_queue -p hpss_poolr_pool +prefect deployment apply cfs_to_hpss_flow-deployment.yaml \ No newline at end of file diff --git a/docs/mkdocs/docs/733.md b/docs/mkdocs/docs/733.md new file mode 100644 index 00000000..ea0c8918 --- /dev/null +++ b/docs/mkdocs/docs/733.md @@ -0,0 +1,141 @@ +```mermaid +sequenceDiagram + %% Participants + participant Detector as Detector + participant FileWatcher as File Watcher (data733) + participant Dispatcher as Dispatcher [Prefect Worker] + + %% Flow 1: new_file_733 Prefect Flow + participant F1_Data as Flow1: data733 + participant F1_CFS as Flow1: NERSC CFS + participant F1_SciCat as Flow1: SciCat (Metadata DB) + + %% Flow 2: Scheduled HPSS Transfer Prefect Flow + participant F2_CFS as Flow2: NERSC CFS + participant F2_HPSS as Flow2: HPSS Tape Archive + participant F2_SciCat as Flow2: SciCat (Metadata DB) + + %% Flow 3: HPC Downstream Analysis Prefect Flow + participant F3_Data as Flow3: data733 + participant HPC_FS as HPC Filesystem + participant HPC_Compute as HPC Compute + + %% Scheduled Pruning (triggered by all flows) + participant SPruning as Scheduled Pruning (Prefect Workers) + participant P_CFS as Prune Target: NERSC CFS + participant P_Data as Prune Target: data733 + + %% Initial Trigger Sequence + Detector->>FileWatcher: Send Raw Data + FileWatcher->>Dispatcher: File Watcher Trigger + Dispatcher->>F1_Data: Start new_file_733 Flow + Dispatcher->>F2_CFS: Start Scheduled HPSS Transfer Flow + Dispatcher->>F3_Data: Start HPC Downstream Analysis Flow + + %% Flow 1 interactions + F1_Data->>F1_CFS: Globus Transfer: Raw Data + F1_CFS->>F1_SciCat: SciCat Ingestion: Metadata + + %% Flow 2 interactions + F2_CFS->>F2_HPSS: SFAPI Slurm htar Transfer: Raw Data + F2_HPSS->>F2_SciCat: SciCat Ingestion: Metadata + + %% Flow 3 interactions + F3_Data->>HPC_FS: Globus Transfer: Raw Data + HPC_FS->>HPC_Compute: Transfer Raw Data + HPC_Compute->>HPC_FS: Return Scratch Data + HPC_FS->>F3_Data: Globus Transfer: Scratch Data + + %% Scheduled Pruning triggered by flows + F1_SciCat-->>SPruning: Trigger Pruning + F2_SciCat-->>SPruning: Trigger Pruning + F3_Data-->>SPruning: Trigger Pruning + + SPruning->>P_CFS: Prune NERSC CFS + SPruning->>P_Data: Prune data733 +``` + + +```mermaid +--- +config: + theme: neo + layout: elk + look: neo +--- +flowchart LR + subgraph s1["new_file_733
[Prefect Flow]"] + n20["data733"] + n21["NERSC CFS"] + n22@{ label: "SciCat
[Metadata Database]" } + end + subgraph s2["Scheduled HPSS Transfer
[Prefect Flow]"] + n38["NERSC CFS"] + n39["HPSS Tape Archive"] + n40["SciCat
[Metadata Database]"] + end + subgraph s3["HPC Downstream Analysis
[Prefect Flow]"] + n41["data733"] + n42["HPC
Filesystem"] + n43["HPC
Compute"] + end + n23["data733"] -- File Watcher --> n24["Dispatcher
[Prefect Worker]"] + n25["Detector"] -- Raw Data --> n23 + n24 --> s1 & s2 & s3 + n20 -- Raw Data [Globus Transfer] --> n21 + n21 -- "Metadata [SciCat Ingestion]" --> n22 + n32["Scheduled Pruning
[Prefect Workers]"] --> n35["NERSC CFS"] & n34["data733"] + n38 -- Raw Data [SFAPI Slurm htar Transfer] --> n39 + n39 -- "Metadata [SciCat Ingestion]" --> n40 + s2 --> n32 + s3 --> n32 + s1 --> n32 + n41 -- Raw Data [Globus Transfer] --> n42 + n42 -- Raw Data --> n43 + n43 -- Scratch Data --> n42 + n42 -- Scratch Data [Globus Transfer] --> n41 + n20@{ shape: internal-storage} + n21@{ shape: disk} + n22@{ shape: db} + n38@{ shape: disk} + n39@{ shape: paper-tape} + n40@{ shape: db} + n41@{ shape: internal-storage} + n42@{ shape: disk} + n23@{ shape: internal-storage} + n24@{ shape: rect} + n25@{ shape: rounded} + n35@{ shape: disk} + n34@{ shape: internal-storage} + n20:::storage + n20:::Peach + n21:::Sky + n22:::Sky + n38:::Sky + n39:::storage + n40:::Sky + n41:::Peach + n42:::Sky + n43:::compute + n23:::collection + n23:::storage + n23:::Peach + n24:::collection + n24:::Rose + n25:::Ash + n32:::Rose + n35:::Sky + n34:::Peach + classDef collection fill:#D3A6A1, stroke:#D3A6A1, stroke-width:2px, color:#000000 + classDef Rose stroke-width:1px, stroke-dasharray:none, stroke:#FF5978, fill:#FFDFE5, color:#8E2236 + classDef storage fill:#A3C1DA, stroke:#A3C1DA, stroke-width:2px, color:#000000 + classDef Ash stroke-width:1px, stroke-dasharray:none, stroke:#999999, fill:#EEEEEE, color:#000000 + classDef visualization fill:#E8D5A6, stroke:#E8D5A6, stroke-width:2px, color:#000000 + classDef Peach stroke-width:1px, stroke-dasharray:none, stroke:#FBB35A, fill:#FFEFDB, color:#8F632D + classDef Sky stroke-width:1px, stroke-dasharray:none, stroke:#374D7C, fill:#E2EBFF, color:#374D7C + classDef compute fill:#A9C0C9, stroke:#A9C0C9, stroke-width:2px, color:#000000 + style s1 stroke:#757575 + style s2 stroke:#757575 + style s3 stroke:#757575 + +``` \ No newline at end of file diff --git a/docs/mkdocs/docs/tomography_workflow.md b/docs/mkdocs/docs/tomography_workflow.md new file mode 100644 index 00000000..d50ef8ac --- /dev/null +++ b/docs/mkdocs/docs/tomography_workflow.md @@ -0,0 +1,129 @@ +```mermaid + +--- +config: + theme: neo + layout: elk + look: neo +--- +flowchart LR + subgraph s2["ALCF Reconstruction [Prefect Flow]"] + n17["data832"] + n18["ALCF Eagle
[Filesystem]"] + n26["Reconstruction on ALCF Polaris with Globus Compute Endpoint"] + end + subgraph s1["new_file_832 [Prefect Flow]"] + n20["data832"] + n21["NERSC CFS"] + n22@{ label: "SciCat
[Metadata Database]" } + end + subgraph s3["NERSC Reconstruction [Prefect Flow]"] + n28["NERSC CFS"] + n29["NERSC Scratch"] + n30["Reconstruction on NERSC Perlmutter with SFAPI, Slurm, Docker"] + n42["data832"] + end + subgraph s4["Scheduled HPSS Transfer [Prefect Flow]"] + n38["NERSC CFS"] + n39["HPSS Tape Archive"] + n40["SciCat
[Metadata Database]"] + end + subgraph s5["Data Visualization at the Beamline"] + n41["data832"] + n43["Tiled Server
[Reconstruction Database]"] + n44(["ITK-VTK-Viewer
[Web Application]"]) + n45["SciCat
[Metadata Database]"] + end + n17 -- Raw Data [Globus Transfer] --> n18 + n23["spot832"] -- File Watcher --> n24["Dispatcher
[Prefect Worker]"] + n23 -- "Raw Data [Globus Transfer]" --> n20 + n25["Detector"] -- Raw Data --> n23 + n24 --> s2 & s1 & s3 & s4 + n20 -- Raw Data [Globus Transfer] --> n21 + n21 -- "Metadata [SciCat Ingestion]" --> n22 + n18 -- Raw Data --> n26 + n26 -- Recon Data --> n18 + n18 -- Recon Data [Globus Transfer] --> n17 + n28 -- Raw Data --> n29 + n29 -- Raw Data --> n30 + n29 -- Recon Data --> n28 + n30 -- Recon Data --> n29 + s1 --> n32["Scheduled Pruning
[Prefect Workers]"] + s3 --> n32 + s2 --> n32 + n32 --> n33["ALCF Eagle"] & n35["NERSC CFS"] & n34["data832"] & n36["spot832"] + n38 -- Raw Data [SFAPI Slurm htar Transfer] --> n39 + s4 --> n32 + n39 -- "Metadata [SciCat Ingestion]" --> n40 + n28 -- "Recon Data" --> n42 + n41 -- Recon Data --> n43 + n43 -- Recon Data --> n44 + n43 -- Metadata [SciCat Ingestion] --> n45 + n45 -- Hyperlink --> n44 + n17@{ shape: internal-storage} + n18@{ shape: disk} + n20@{ shape: internal-storage} + n21@{ shape: disk} + n22@{ shape: db} + n28@{ shape: disk} + n29@{ shape: disk} + n42@{ shape: internal-storage} + n38@{ shape: disk} + n39@{ shape: paper-tape} + n40@{ shape: db} + n41@{ shape: internal-storage} + n43@{ shape: db} + n45@{ shape: db} + n23@{ shape: internal-storage} + n24@{ shape: rect} + n25@{ shape: rounded} + n33@{ shape: disk} + n35@{ shape: disk} + n34@{ shape: internal-storage} + n36@{ shape: internal-storage} + n17:::storage + n17:::Peach + n18:::storage + n18:::Sky + n26:::compute + n20:::storage + n20:::Peach + n21:::Sky + n22:::Sky + n28:::Sky + n29:::storage + n29:::Sky + n30:::compute + n42:::Peach + n38:::Sky + n39:::storage + n40:::Sky + n41:::Peach + n43:::Sky + n44:::visualization + n45:::Sky + n23:::collection + n23:::storage + n23:::Peach + n24:::collection + n24:::Rose + n25:::Ash + n32:::Rose + n33:::Sky + n35:::Sky + n34:::Peach + n36:::Peach + classDef collection fill:#D3A6A1, stroke:#D3A6A1, stroke-width:2px, color:#000000 + classDef compute fill:#A9C0C9, stroke:#A9C0C9, stroke-width:2px, color:#000000 + classDef Rose stroke-width:1px, stroke-dasharray:none, stroke:#FF5978, fill:#FFDFE5, color:#8E2236 + classDef storage fill:#A3C1DA, stroke:#A3C1DA, stroke-width:2px, color:#000000 + classDef Ash stroke-width:1px, stroke-dasharray:none, stroke:#999999, fill:#EEEEEE, color:#000000 + classDef Peach stroke-width:1px, stroke-dasharray:none, stroke:#FBB35A, fill:#FFEFDB, color:#8F632D + classDef visualization fill:#E8D5A6, stroke:#E8D5A6, stroke-width:2px, color:#000000 + classDef Sky stroke-width:1px, stroke-dasharray:none, stroke:#374D7C, fill:#E2EBFF, color:#374D7C + style s2 stroke:#757575 + style s1 stroke:#757575 + style s3 stroke:#757575 + style s4 stroke:#757575 + style s5 stroke:#757575 +``` \ No newline at end of file diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index 0ba40510..55aecd95 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -6,7 +6,7 @@ class Config832(BeamlineConfig): def __init__(self) -> None: - super().__init__(beamline_id="832") + super().__init__(beamline_id="8.3.2") def _beam_specific_config(self) -> None: # config = transfer.get_config() @@ -27,5 +27,6 @@ def _beam_specific_config(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] + self.hpss_alsdev = self.config["hpss_alsdev"] self.scicat = self.config["scicat"] self.ghcr_images832 = self.config["ghcr_images832"] diff --git a/orchestration/flows/bl832/hpss.py b/orchestration/flows/bl832/hpss.py new file mode 100644 index 00000000..d4a69b25 --- /dev/null +++ b/orchestration/flows/bl832/hpss.py @@ -0,0 +1,108 @@ +""" +This module contains the HPSS flow for BL832. +""" +import logging +from typing import List, Optional + +from prefect import flow + +from orchestration.config import BeamlineConfig +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint +from orchestration.flows.bl832.config import Config832 +from orchestration.transfer_controller import get_transfer_controller, CopyMethod + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@flow(name="cfs_to_hpss_flow") +def cfs_to_hpss_flow( + file_path: str = None, + source_endpoint: FileSystemEndpoint = None, + destination_endpoint: HPSSEndpoint = None, + config: BeamlineConfig = Config832() +) -> bool: + """ + The CFS to HPSS flow for BL832. + + Parameters + ---------- + file_path : str + The path of the file to transfer. + source_endpoint : FileSystemEndpoint + The source endpoint. + destination_endpoint : HPSSEndpoint + The destination endpoint. + """ + + logger.info("Running cfs_to_hpss_flow") + logger.info(f"Transferring {file_path} from {source_endpoint.name} to {destination_endpoint.name}") + + logger.info("Configuring transfer controller for CFS_TO_HPSS.") + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.CFS_TO_HPSS, + config=config + ) + + logger.info("CFSToHPSSTransferController selected. Initiating transfer.") + result = transfer_controller.copy( + file_path=file_path, + source_endpoint=source_endpoint, + destination_endpoint=destination_endpoint + ) + + return result + + +@flow(name="hpss_to_cfs_flow") +def hpss_to_cfs_flow( + file_path: str = None, + source_endpoint: HPSSEndpoint = None, + destination_endpoint: FileSystemEndpoint = None, + files_to_extract: Optional[List[str]] = None, + config: BeamlineConfig = Config832() +) -> bool: + """ + The HPSS to CFS flow for BL832. + + Parameters + ---------- + file_path : str + The path of the file to transfer. + source_endpoint : HPSSEndpoint + The source endpoint. + destination_endpoint : FileSystemEndpoint + The destination endpoint. + """ + + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.HPSS_TO_CFS, + config=config + ) + + result = transfer_controller.copy( + file_path=file_path, + source_endpoint=source_endpoint, + destination_endpoint=destination_endpoint, + files_to_extract=files_to_extract, + ) + + return result + + +if __name__ == "__main__": + + config = Config832() + project_name = "ALS-11193_nbalsara" + source_endpoint = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/" + ) + destination_endpoint = config.hpss_alsdev + + cfs_to_hpss_flow( + file_path=f"{project_name}", + source_endpoint=source_endpoint, + destination_endpoint=destination_endpoint, + config=config + ) diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 92d8c97f..c4b9f720 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -42,40 +42,6 @@ def __init__( TomographyHPCController.__init__(self, config) self.client = client - # Moved this method to orchestration/sfapi.py: - - # @staticmethod - # def create_sfapi_client() -> Client: - # """Create and return an NERSC client instance""" - - # # When generating the SFAPI Key in Iris, make sure to select "asldev" as the user! - # # Otherwise, the key will not have the necessary permissions to access the data. - # client_id_path = os.getenv("PATH_NERSC_CLIENT_ID") - # client_secret_path = os.getenv("PATH_NERSC_PRI_KEY") - - # if not client_id_path or not client_secret_path: - # logger.error("NERSC credentials paths are missing.") - # raise ValueError("Missing NERSC credentials paths.") - # if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): - # logger.error("NERSC credential files are missing.") - # raise FileNotFoundError("NERSC credential files are missing.") - - # client_id = None - # client_secret = None - # with open(client_id_path, "r") as f: - # client_id = f.read() - - # with open(client_secret_path, "r") as f: - # client_secret = JsonWebKey.import_key(json.loads(f.read())) - - # try: - # client = Client(client_id, client_secret) - # logger.info("NERSC client created successfully.") - # return client - # except Exception as e: - # logger.error(f"Failed to create NERSC client: {e}") - # raise e - def reconstruct( self, file_path: str = "", diff --git a/orchestration/sfapi.py b/orchestration/sfapi.py index 2ea5dfb1..b1d2f46f 100644 --- a/orchestration/sfapi.py +++ b/orchestration/sfapi.py @@ -11,16 +11,15 @@ load_dotenv() +# TODO: we need a better way to store the client_id and client_secret def create_sfapi_client( - client_id_path: str, - client_secret_path: str, + client_id_path: str = os.getenv("PATH_NERSC_CLIENT_ID"), + client_secret_path: str = os.getenv("PATH_NERSC_PRI_KEY"), ) -> Client: """Create and return an NERSC client instance""" # When generating the SFAPI Key in Iris, make sure to select "asldev" as the user! # Otherwise, the key will not have the necessary permissions to access the data. - # client_id_path = os.getenv("PATH_NERSC_CLIENT_ID") - # client_secret_path = os.getenv("PATH_NERSC_PRI_KEY") if not client_id_path or not client_secret_path: logger.error("NERSC credentials paths are missing.") diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 13a04f4e..2d1eaa36 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -281,7 +281,7 @@ def __init__( def copy( self, - file_path: str = "", + file_path: str = None, source: FileSystemEndpoint = None, destination: FileSystemEndpoint = None, ) -> bool: @@ -333,6 +333,30 @@ def copy( class CFSToHPSSTransferController(TransferController[HPSSEndpoint]): + """ + Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. + + This controller requires the source to be a FileSystemEndpoint on CFS and the + destination to be an HPSSEndpoint. For a single file, the transfer is done using hsi (via hsi cput). + For a directory, the transfer is performed with htar. In this updated version, if the source is a + directory then the files are bundled into tar archives based on their modification dates as follows: + - Files with modification dates between Jan 1 and Jul 15 (inclusive) are grouped together + (Cycle 1 for that year). + - Files with modification dates between Jul 16 and Dec 31 are grouped together (Cycle 2). + + Within each group, if the total size exceeds 2 TB the files are partitioned into multiple tar bundles. + The resulting naming convention on HPSS is: + + /home/a/alsdev/data_mover/[beamline]/raw/[proposal_name]/ + [proposal_name]_[year]-[cycle].tar + [proposal_name]_[year]-[cycle]_part0.tar + [proposal_name]_[year]-[cycle]_part1.tar + ... + + At the end of the SLURM script, the directory tree for both the source (CFS) and destination (HPSS) + is echoed for logging purposes. + """ + def __init__( self, client: Client, @@ -340,150 +364,206 @@ def __init__( ) -> None: super().__init__(config) self.client = client - """ - Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. - - This transfer contoller requires the source to be a FileSystemEndpoint on CFS and the destination to be an HPSSEndpoint. - If you want to move data from somewhere else to HPSS, you will first need to transfer to CFS using a different controller, - and then to HPSS with this one. - Args: - TransferController: Abstract class for transferring data. - """ def copy( self, - file_path: str = "", + file_path: str = None, source: FileSystemEndpoint = None, - destination: HPSSEndpoint = None, + destination: HPSSEndpoint = None ) -> bool: - """ - Copy a file from a CFS source endpoint to an HPSS destination endpoint. - - For a single file, the transfer is done using hsi (via hsi put). - For a directory, the transfer is performed with htar: - - If the directory's total size is less than 2TB, a single tar archive is created. - - If the size exceeds 2TB, the files are split into multiple tar archives, - each not exceeding the 2TB threshold. - - The data is saved on HPSS in the correct location using the destination's root path. - - - Args: - file_path (str): The path of the file or directory to copy. - source (FileSystemEndpoint): The CFS source endpoint. - destination (HPSSEndpoint): The HPSS destination endpoint. - """ - logger.info("Transferring data from CFS to HPSS") if not file_path or not source or not destination: logger.error("Missing required parameters for CFSToHPSSTransferController.") return False - path = Path(file_path) - folder_name = path.parent.name - if not folder_name: - folder_name = "" - - file_name = f"{path.stem}" + # Compute the full path on CFS for the file/directory. + full_cfs_path = source.full_path(file_path) + # Get the beamline_id from the configuration. + beamline_id = self.config.beamline_id + # Build the HPSS destination root path using the convention: [destination.root_path]/[beamline_id]/raw + hpss_root_path = f"{destination.root_path.rstrip('/')}/{beamline_id}/raw" - logger.info(f"File name: {file_name}") - logger.info(f"Folder name: {folder_name}") - - # Construct absolute source path as visible on Perlmutter - abs_source_path = source.full_path(file_path) - dest_root = destination.root_path - job_name_suffix = Path(abs_source_path).name + # Determine the proposal (project) folder name from the file_path. + path = Path(file_path) + proposal_name = path.parent.name + if not proposal_name or proposal_name == ".": # if file_path is in the root directory + proposal_name = file_path logs_path = "/global/cfs/cdirs/als/data_mover/hpss_transfer_logs" - # IMPORTANT: job script must be deindented to the leftmost column or it will fail immediately - # Note: If q=debug, there is no minimum time limit - # However, if q=preempt, there is a minimum time limit of 2 hours. Otherwise the job won't run. - # The realtime queue can only be used for select accounts (e.g. ALS) - job_script = f"""#!/bin/bash -#SBATCH -q xfer -#SBATCH -A als -#SBATCH -C cron -#SBATCH --time=12:00:00 -#SBATCH --job-name=transfer_to_HPSS_{job_name_suffix} -#SBATCH --output={logs_path}/%j.out -#SBATCH --error={logs_path}/%j.err -#SBATCH --licenses=SCRATCH -#SBATCH --mem=100GB - -set -euo pipefail -date - -# Define source and destination variables -SOURCE_PATH="{abs_source_path}" -DEST_ROOT="{dest_root}" -FOLDER_NAME=$(basename "$SOURCE_PATH") + # Build the SLURM job script with detailed inline comments for clarity. + job_script = rf"""#!/bin/bash +# ------------------------------------------------------------------ +# SLURM Job Script for Transferring Data from CFS to HPSS +# This script will: +# 1. Define the source (CFS) and destination (HPSS) paths. +# 2. Create the destination directory on HPSS if it doesn't exist. +# 3. Determine if the source is a file or a directory. +# - If a file, transfer it using 'hsi cput'. +# - If a directory, group files by beam cycle and archive them. +# * Cycle 1: Jan 1 - Jul 15 +# * Cycle 2: Jul 16 - Dec 31 +# * If a group exceeds 2 TB, it is partitioned into multiple tar archives. +# * Archive names: +# [proposal_name]_[year]-[cycle].tar +# [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. +# 4. Echo directory trees for both source and destination for logging. +# ------------------------------------------------------------------ + +#SBATCH -q xfer # Specify the SLURM queue to use. +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_to_HPSS_{file_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/%j.out # Standard output log file. +#SBATCH --error={logs_path}/%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=4GB # Request #GB of memory. Defult 2GB. + +set -euo pipefail # Enable strict error checking. +date # Print current date/time for logging. + +# ------------------------------------------------------------------ +# Define source and destination variables. +# ------------------------------------------------------------------ + +# SOURCE_PATH: Full path of the file or directory on CFS. +SOURCE_PATH="{full_cfs_path}" + +# DEST_ROOT: Root destination on HPSS built from configuration. +DEST_ROOT="{hpss_root_path}" + +# FOLDER_NAME: Proposal name (project folder) derived from the file path. +FOLDER_NAME="{proposal_name}" + +# DEST_PATH: Final HPSS destination directory. DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" -# Create destination directory if it doesn't exist on HPSS +# ------------------------------------------------------------------ +# Create destination directory on HPSS if it doesn't exist. +# ------------------------------------------------------------------ + echo "Checking if HPSS destination directory $DEST_PATH exists." -if hsi ls "$DEST_PATH" >/dev/null 2>&1; then +if hsi ls "$DEST_PATH"; then echo "Destination directory $DEST_PATH already exists." else echo "Destination directory $DEST_PATH does not exist. Creating it now." - hsi mkdir "$DEST_PATH" + # hsi mkdir "$DEST_PATH" fi -# Check if source is a file or directory, and run the appropriate transfer command (hsi vs htar) +# ------------------------------------------------------------------ +# Transfer Logic: Check if SOURCE_PATH is a file or directory. +# ------------------------------------------------------------------ -# Case: Single File if [ -f "$SOURCE_PATH" ]; then - echo "Single file detected. Transferring via hsi put." + # Case: Single file detected. + echo "Single file detected. Transferring via hsi cput." FILE_NAME=$(basename "$SOURCE_PATH") - hsi put "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + # Transfer the file to HPSS. 'hsi cput' will only overwrite if the source is newer. + # hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" -# Case: Directory elif [ -d "$SOURCE_PATH" ]; then - # Check directory size and split into multiple archives if necessary + # Case: Directory detected. + echo "Directory detected. Bundling scans by beam cycle." + THRESHOLD=2199023255552 # 2 TB in bytes. + + # ------------------------------------------------------------------ + # Group files based on modification date: + # - Cycle 1: Jan 1 - Jul 15 + # - Cycle 2: Jul 16 - Dec 31 + # For each group, if total size exceeds THRESHOLD, partition into multiple tar archives. + # Naming convention: + # [proposal_name]_[year]-[cycle].tar + # [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. + # ------------------------------------------------------------------ + + # Create a temporary file to store list of files. + FILE_LIST=$(mktemp) + find "$SOURCE_PATH" -type f > "$FILE_LIST" + + # Declare associative arrays to hold grouped file paths and sizes. + declare -A group_files + declare -A group_sizes + + # Read each file and determine its group based on modification date. + while IFS= read -r file; do + mtime=$(stat -c %Y "$file") + year=$(date -d @"$mtime" +%Y) + month=$(date -d @"$mtime" +%m | sed 's/^0*//') + day=$(date -d @"$mtime" +%d | sed 's/^0*//') + # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ] }}; then + cycle=1 + else + cycle=2 + fi + key="${{year}}-${{cycle}}" + group_files["$key"]="${{group_files["$key"]}} $file" + fsize=$(stat -c %s "$file") + group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) + done < "$FILE_LIST" + rm "$FILE_LIST" + + # Iterate over each (year,cycle) group and create tar archives. + for key in "${{!group_files[@]}}"; do + files=(${{group_files["$key"]}}) + total_group_size=${{group_sizes["$key"]}} + echo "Group $key has ${{#files[@]}} files, total size $total_group_size bytes." + + part=0 + current_size=0 + current_files=() + # Bundle files until the THRESHOLD is reached, then create an archive. + for f in "${{files[@]}}"; do + fsize=$(stat -c %s "$f") + if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then + # Determine tar archive name based on whether it is the first bundle or a subsequent part. + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "Creating archive $tar_name with ${{#current_files[@]}} files, size $current_size bytes." + # htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + part=$((part+1)) + current_files=() + current_size=0 + fi + # Add the current file to the bundle and update the cumulative size. + current_files+=("$f") + current_size=$(( current_size + fsize )) + done + # Create the final archive for the group if there are remaining files. + if [ ${{#current_files[@]}} -gt 0 ]; then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "Creating final archive $tar_name with ${{#current_files[@]}} files, size $current_size bytes." + # htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + fi + done - echo "Directory detected. Calculating total size..." - TOTAL_SIZE=$(du -sb "$SOURCE_PATH" | awk '{{print $1}}') - THRESHOLD=2199023255552 # 2 TB in bytes - echo "Total size: $TOTAL_SIZE bytes" +else + echo "Error: $SOURCE_PATH is neither a file nor a directory. Are you sure it exists?" + exit 1 +fi - # If directory size is less than 2TB, archive directly with htar - if [ "$TOTAL_SIZE" -lt "$THRESHOLD" ]; then - echo "Directory size is under 2TB. Archiving with htar." - htar -cvf "${{DEST_PATH}}/${{FOLDER_NAME}}.tar" "$SOURCE_PATH" +# ------------------------------------------------------------------ +# Logging: Display directory trees for both source and destination. +# ------------------------------------------------------------------ - # If directory size exceeds 2TB, split the project into multiple archives that are less than 2TB each - else - echo "Directory size exceeds 2TB. Splitting into multiple archives." - FILE_LIST=$(mktemp) - find "$SOURCE_PATH" -type f > "$FILE_LIST" - chunk=1 - current_size=0 - current_files=() - while IFS= read -r file; do - size=$(stat -c%s "$file") - if (( current_size + size > THRESHOLD )); then - tar_archive="${{DEST_PATH}}/${{FOLDER_NAME}}_part${{chunk}}.tar" - echo "Creating archive $tar_archive with size $current_size bytes" - htar -cvf "$tar_archive" "${{current_files[@]}}" - current_files=() - current_size=0 - ((chunk++)) - fi - current_files+=("$file") - current_size=$(( current_size + size )) - done < "$FILE_LIST" - if [ ${{#current_files[@]}} -gt 0 ]; then - tar_archive="${{DEST_PATH}}/${{FOLDER_NAME}}_part${{chunk}}.tar" - echo "Creating final archive $tar_archive with size $current_size bytes" - htar -cvf "$tar_archive" "${{current_files[@]}}" - fi - rm "$FILE_LIST" - fi +echo "=== Listing Source (CFS) Tree ===" +if [ -d "$SOURCE_PATH" ]; then + find "$SOURCE_PATH" -print else - echo "Error: $SOURCE_PATH is neither a file nor a directory." - exit 1 + echo "$SOURCE_PATH is a file." fi +echo "=== Listing Destination (HPSS) Tree ===" +# hsi ls -R "$DEST_PATH" || echo "Failed to list HPSS tree at $DEST_PATH" + date """ try: @@ -500,14 +580,13 @@ def copy( time.sleep(60) logger.info(f"Job {job.jobid} current state: {job.state}") - job.complete() # Wait until the job completes - logger.info("Reconstruction job completed successfully.") + job.complete() # Wait until the job completes. + logger.info("Transfer job completed successfully.") return True except Exception as e: - logger.info(f"Error during job submission or completion: {e}") + logger.error(f"Error during job submission or completion: {e}") match = re.search(r"Job not found:\s*(\d+)", str(e)) - if match: jobid = match.group(1) logger.info(f"Attempting to recover job {jobid}.") @@ -515,24 +594,16 @@ def copy( job = self.client.perlmutter.job(jobid=jobid) time.sleep(30) job.complete() - logger.info("Reconstruction job completed successfully after recovery.") + logger.info("Transfer job completed successfully after recovery.") return True except Exception as recovery_err: logger.error(f"Failed to recover job {jobid}: {recovery_err}") return False else: - # Unknown error: cannot recover return False class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): - def __init__( - self, - client: Client, - config: BeamlineConfig - ) -> None: - super().__init__(config) - self.client = client """ Use SFAPI to move data between HPSS and CFS at NERSC. @@ -554,6 +625,15 @@ def __init__( Returns: bool: True if the transfer job completes successfully, False otherwise. """ + + def __init__( + self, + client: Client, + config: BeamlineConfig + ) -> None: + super().__init__(config) + self.client = client + def copy( self, file_path: str = None, From 7db34e915338b7b2d40a70d3167f29bbaf6c0435 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 14 Feb 2025 14:40:55 -0800 Subject: [PATCH 007/128] Fixed syntax errors --- create_deployment_832_hpss.sh | 2 +- orchestration/flows/bl832/hpss.py | 34 +++++++++++++++------------- orchestration/transfer_controller.py | 8 +++---- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/create_deployment_832_hpss.sh b/create_deployment_832_hpss.sh index 0da5a38a..972b5dba 100644 --- a/create_deployment_832_hpss.sh +++ b/create_deployment_832_hpss.sh @@ -3,5 +3,5 @@ export $(grep -v '^#' .env | xargs) prefect work-pool create 'hpss_pool' -prefect deployment build ./orchestration/flows/bl832/hpss.py:cfs_to_hpss_flow -n cfs_to_hpss_flow -q cfs_to_hpss_queue -p hpss_poolr_pool +prefect deployment build ./orchestration/flows/bl832/hpss.py:cfs_to_hpss_flow -n cfs_to_hpss_flow -q cfs_to_hpss_queue -p hpss_pool prefect deployment apply cfs_to_hpss_flow-deployment.yaml \ No newline at end of file diff --git a/orchestration/flows/bl832/hpss.py b/orchestration/flows/bl832/hpss.py index d4a69b25..2142490f 100644 --- a/orchestration/flows/bl832/hpss.py +++ b/orchestration/flows/bl832/hpss.py @@ -18,8 +18,8 @@ @flow(name="cfs_to_hpss_flow") def cfs_to_hpss_flow( file_path: str = None, - source_endpoint: FileSystemEndpoint = None, - destination_endpoint: HPSSEndpoint = None, + source: FileSystemEndpoint = None, + destination: HPSSEndpoint = None, config: BeamlineConfig = Config832() ) -> bool: """ @@ -29,14 +29,14 @@ def cfs_to_hpss_flow( ---------- file_path : str The path of the file to transfer. - source_endpoint : FileSystemEndpoint + source : FileSystemEndpoint The source endpoint. - destination_endpoint : HPSSEndpoint + destination : HPSSEndpoints The destination endpoint. """ logger.info("Running cfs_to_hpss_flow") - logger.info(f"Transferring {file_path} from {source_endpoint.name} to {destination_endpoint.name}") + logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") logger.info("Configuring transfer controller for CFS_TO_HPSS.") transfer_controller = get_transfer_controller( @@ -47,8 +47,8 @@ def cfs_to_hpss_flow( logger.info("CFSToHPSSTransferController selected. Initiating transfer.") result = transfer_controller.copy( file_path=file_path, - source_endpoint=source_endpoint, - destination_endpoint=destination_endpoint + source=source, + destination=destination ) return result @@ -57,8 +57,8 @@ def cfs_to_hpss_flow( @flow(name="hpss_to_cfs_flow") def hpss_to_cfs_flow( file_path: str = None, - source_endpoint: HPSSEndpoint = None, - destination_endpoint: FileSystemEndpoint = None, + source: HPSSEndpoint = None, + destination: FileSystemEndpoint = None, files_to_extract: Optional[List[str]] = None, config: BeamlineConfig = Config832() ) -> bool: @@ -82,8 +82,8 @@ def hpss_to_cfs_flow( result = transfer_controller.copy( file_path=file_path, - source_endpoint=source_endpoint, - destination_endpoint=destination_endpoint, + source=source, + destination=destination, files_to_extract=files_to_extract, ) @@ -94,15 +94,17 @@ def hpss_to_cfs_flow( config = Config832() project_name = "ALS-11193_nbalsara" - source_endpoint = FileSystemEndpoint( + source = FileSystemEndpoint( name="CFS", root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/" ) - destination_endpoint = config.hpss_alsdev - + destination = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"] + ) cfs_to_hpss_flow( file_path=f"{project_name}", - source_endpoint=source_endpoint, - destination_endpoint=destination_endpoint, + source=source, + destination=destination, config=config ) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 2d1eaa36..3347595b 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -415,8 +415,8 @@ def copy( #SBATCH -C cron # Use the 'cron' constraint. #SBATCH --time=12:00:00 # Maximum runtime of 12 hours. #SBATCH --job-name=transfer_to_HPSS_{file_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/%j.out # Standard output log file. -#SBATCH --error={logs_path}/%j.err # Standard error log file. +#SBATCH --output={logs_path}/{file_path}_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{file_path}_%j.err # Standard error log file. #SBATCH --licenses=SCRATCH # Request the SCRATCH license. #SBATCH --mem=4GB # Request #GB of memory. Defult 2GB. @@ -492,13 +492,13 @@ def copy( month=$(date -d @"$mtime" +%m | sed 's/^0*//') day=$(date -d @"$mtime" +%d | sed 's/^0*//') # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. - if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ] }}; then + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then cycle=1 else cycle=2 fi key="${{year}}-${{cycle}}" - group_files["$key"]="${{group_files["$key"]}} $file" + group_files["$key"]="${{group_files["$key"]:-}} $file" fsize=$(stat -c %s "$file") group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) done < "$FILE_LIST" From 65ab2e08b55bb5cbae9f7ead8b41511ab5725c91 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 14 Feb 2025 16:09:16 -0800 Subject: [PATCH 008/128] Cleaned up logging for the CFSToHPSSTransferController slurm job script to clearly show what files are in each tar bundle. Ready to test with a real project. --- docs/mkdocs/docs/hpss.md | 60 ++++++++- orchestration/transfer_controller.py | 182 +++++++++++++++------------ 2 files changed, 158 insertions(+), 84 deletions(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index 03681278..ab5ddeb2 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -163,8 +163,64 @@ Most of the time we expect transfers to occur from CFS to HPSS on a scheduled ba ### Transfer to HPSS Implementation **`orchestration/transfer_controller.py`: `CFSToHPSSTransferController()`** -Input -Output + +```mermaid + +flowchart TD + subgraph "Parameter & Path Setup" + A["Validate Params:
file_path, source, destination"] + B["Compute CFS Path
and get beamline_id"] + C["Build HPSS Root Path
and determine proposal name"] + D["Set Logs Path"] + end + + subgraph "SLURM Script" + E["Set SLURM Header Directives"] + F["Enable Strict Error Handling"] + G["Define Variables:
SOURCE_PATH, DEST_ROOT,
FOLDER_NAME, DEST_PATH"] + H["Check if Destination Directory Exists"] + I{"Directory Exists?"} + J["If Yes: Log Exists"] + K["If No: Create Directory"] + L["Determine Source Type"] + M{"File or Directory?"} + N["If File:
Transfer via hsi cput"] + O["If Directory:
List files, group by date,
bundle and create tar archives"] + end + + subgraph "Job Submission" + P["Log Directory Trees"] + Q["Submit Job via Perlmutter"] + R["Update Job Status & Wait"] + S{"Job Successful?"} + T["Return True"] + U["Attempt Recovery & Log Error
Return False"] + end + + %% Connections + A --> B + B --> C + C --> D + D --> E + E --> F + F --> G + G --> H + H --> I + I -- "Yes" --> J + I -- "No" --> K + J --> L + K --> L + L --> M + M -- "File" --> N + M -- "Directory" --> O + N --> P + O --> P + P --> Q + Q --> R + R --> S + S -- "Yes" --> T + S -- "No" --> U +``` ### Transfer to CFS Implementation **`orchestration/transfer_controller.py`: `HPSSToCFSTransferController()`** diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 3347595b..b5a7720a 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -389,7 +389,7 @@ def copy( if not proposal_name or proposal_name == ".": # if file_path is in the root directory proposal_name = file_path - logs_path = "/global/cfs/cdirs/als/data_mover/hpss_transfer_logs" + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" # Build the SLURM job script with detailed inline comments for clarity. job_script = rf"""#!/bin/bash @@ -418,153 +418,171 @@ def copy( #SBATCH --output={logs_path}/{file_path}_%j.out # Standard output log file. #SBATCH --error={logs_path}/{file_path}_%j.err # Standard error log file. #SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=4GB # Request #GB of memory. Defult 2GB. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. set -euo pipefail # Enable strict error checking. -date # Print current date/time for logging. +echo "[LOG] Job started at: $(date)" # ------------------------------------------------------------------ # Define source and destination variables. # ------------------------------------------------------------------ +echo "[LOG] Defining source and destination paths." # SOURCE_PATH: Full path of the file or directory on CFS. SOURCE_PATH="{full_cfs_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" # DEST_ROOT: Root destination on HPSS built from configuration. DEST_ROOT="{hpss_root_path}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" # FOLDER_NAME: Proposal name (project folder) derived from the file path. FOLDER_NAME="{proposal_name}" +echo "[LOG] FOLDER_NAME set to: $FOLDER_NAME" # DEST_PATH: Final HPSS destination directory. DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" +echo "[LOG] DEST_PATH set to: $DEST_PATH" # ------------------------------------------------------------------ # Create destination directory on HPSS if it doesn't exist. # ------------------------------------------------------------------ - -echo "Checking if HPSS destination directory $DEST_PATH exists." +echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." if hsi ls "$DEST_PATH"; then - echo "Destination directory $DEST_PATH already exists." + echo "[LOG] Destination directory $DEST_PATH already exists." else - echo "Destination directory $DEST_PATH does not exist. Creating it now." - # hsi mkdir "$DEST_PATH" + echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it." + hsi mkdir "$DEST_PATH" + echo "[LOG] (Simulated) Created directory $DEST_PATH." fi # ------------------------------------------------------------------ # Transfer Logic: Check if SOURCE_PATH is a file or directory. # ------------------------------------------------------------------ - +echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" if [ -f "$SOURCE_PATH" ]; then # Case: Single file detected. - echo "Single file detected. Transferring via hsi cput." + echo "[LOG] Single file detected. Transferring via hsi cput." FILE_NAME=$(basename "$SOURCE_PATH") - # Transfer the file to HPSS. 'hsi cput' will only overwrite if the source is newer. - # hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + echo "[LOG] File name: $FILE_NAME" + hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." elif [ -d "$SOURCE_PATH" ]; then # Case: Directory detected. - echo "Directory detected. Bundling scans by beam cycle." + echo "[LOG] Directory detected. Initiating bundling process." THRESHOLD=2199023255552 # 2 TB in bytes. + echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" # ------------------------------------------------------------------ - # Group files based on modification date: - # - Cycle 1: Jan 1 - Jul 15 - # - Cycle 2: Jul 16 - Dec 31 - # For each group, if total size exceeds THRESHOLD, partition into multiple tar archives. - # Naming convention: - # [proposal_name]_[year]-[cycle].tar - # [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. + # Group files based on modification date. # ------------------------------------------------------------------ - - # Create a temporary file to store list of files. + echo "[LOG] Grouping files by modification date." FILE_LIST=$(mktemp) find "$SOURCE_PATH" -type f > "$FILE_LIST" + echo "[LOG] List of files stored in temporary file: $FILE_LIST" # Declare associative arrays to hold grouped file paths and sizes. declare -A group_files declare -A group_sizes - # Read each file and determine its group based on modification date. while IFS= read -r file; do - mtime=$(stat -c %Y "$file") - year=$(date -d @"$mtime" +%Y) - month=$(date -d @"$mtime" +%m | sed 's/^0*//') - day=$(date -d @"$mtime" +%d | sed 's/^0*//') - # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. - if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then - cycle=1 - else - cycle=2 - fi - key="${{year}}-${{cycle}}" - group_files["$key"]="${{group_files["$key"]:-}} $file" - fsize=$(stat -c %s "$file") - group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) + mtime=$(stat -c %Y "$file") + year=$(date -d @"$mtime" +%Y) + month=$(date -d @"$mtime" +%m | sed 's/^0*//') + day=$(date -d @"$mtime" +%d | sed 's/^0*//') + # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then + cycle=1 + else + cycle=2 + fi + key="${{year}}-${{cycle}}" + group_files["$key"]="${{group_files["$key"]:-}} $file" + fsize=$(stat -c %s "$file") + group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) done < "$FILE_LIST" rm "$FILE_LIST" + echo "[LOG] Completed grouping files." - # Iterate over each (year,cycle) group and create tar archives. + # Print the files in each group at the end for key in "${{!group_files[@]}}"; do - files=(${{group_files["$key"]}}) - total_group_size=${{group_sizes["$key"]}} - echo "Group $key has ${{#files[@]}} files, total size $total_group_size bytes." - - part=0 - current_size=0 - current_files=() - # Bundle files until the THRESHOLD is reached, then create an archive. - for f in "${{files[@]}}"; do - fsize=$(stat -c %s "$f") - if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then - # Determine tar archive name based on whether it is the first bundle or a subsequent part. - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "Creating archive $tar_name with ${{#current_files[@]}} files, size $current_size bytes." - # htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - part=$((part+1)) - current_files=() - current_size=0 - fi - # Add the current file to the bundle and update the cumulative size. - current_files+=("$f") - current_size=$(( current_size + fsize )) - done - # Create the final archive for the group if there are remaining files. - if [ ${{#current_files[@]}} -gt 0 ]; then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "Creating final archive $tar_name with ${{#current_files[@]}} files, size $current_size bytes." - # htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - fi + echo "[LOG] Group $key contains files:" + for f in ${{group_files["$key"]}}; do + echo " $f" + done done + # ------------------------------------------------------------------ + # Bundle files into tar archives. + # ------------------------------------------------------------------ + for key in "${{!group_files[@]}}"; do + files=(${{group_files["$key"]}}) + total_group_size=${{group_sizes["$key"]}} + echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." + + part=0 + current_size=0 + current_files=() + for f in "${{files[@]}}"; do + fsize=$(stat -c %s "$f") + # If adding this file exceeds the threshold, process the current bundle. + if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Bundle reached threshold." + echo "[LOG] Files in current bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." + htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + part=$((part+1)) + echo "[DEBUG] Resetting bundle variables." + current_files=() + current_size=0 + fi + current_files+=("$f") + current_size=$(( current_size + fsize )) + done + if [ ${{#current_files[@]}} -gt 0 ]; then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Final bundle for group $key:" + echo "[LOG] Files in final bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." + htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + fi + echo "[LOG] Completed processing group $key." + done else - echo "Error: $SOURCE_PATH is neither a file nor a directory. Are you sure it exists?" + echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." exit 1 fi # ------------------------------------------------------------------ # Logging: Display directory trees for both source and destination. # ------------------------------------------------------------------ - -echo "=== Listing Source (CFS) Tree ===" +echo "[LOG] Listing Source (CFS) Tree:" if [ -d "$SOURCE_PATH" ]; then find "$SOURCE_PATH" -print else - echo "$SOURCE_PATH is a file." + echo "[LOG] $SOURCE_PATH is a file." fi -echo "=== Listing Destination (HPSS) Tree ===" -# hsi ls -R "$DEST_PATH" || echo "Failed to list HPSS tree at $DEST_PATH" +echo "[LOG] Listing Destination (HPSS) Tree:" +hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" -date +echo "[LOG] Job completed at: $(date)" """ try: logger.info("Submitting HPSS transfer job to Perlmutter.") From 17a1a52f9eb3cde906380cf9779db6f0fe49a7bc Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 18 Feb 2025 14:49:15 -0800 Subject: [PATCH 009/128] adding support for dynaconf for handling beamline configurations --- orchestration/config.py | 14 +- orchestration/transfer_controller.py | 262 ++++++++++++++------------- requirements.txt | 9 +- 3 files changed, 149 insertions(+), 136 deletions(-) diff --git a/orchestration/config.py b/orchestration/config.py index 40663aee..318b5fe3 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -1,11 +1,16 @@ from abc import ABC, abstractmethod -import collections import builtins -from pathlib import Path +import collections import os - +from pathlib import Path import yaml +from dynaconf import Dynaconf + +settings = Dynaconf( + settings_files=["config.yml"], +) + def get_config(): return read_config(config_file=Path(__file__).parent.parent / "config.yml") @@ -62,7 +67,8 @@ def __init__( beamline_id: str ) -> None: self.beamline_id = beamline_id - self.config = read_config() + # self.config = read_config() + self.config = settings self._beam_specific_config() @abstractmethod diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index b5a7720a..3e0e248f 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -448,139 +448,145 @@ def copy( # Create destination directory on HPSS if it doesn't exist. # ------------------------------------------------------------------ echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." -if hsi ls "$DEST_PATH"; then + +if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then echo "[LOG] Destination directory $DEST_PATH already exists." else echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it." - hsi mkdir "$DEST_PATH" - echo "[LOG] (Simulated) Created directory $DEST_PATH." -fi - -# ------------------------------------------------------------------ -# Transfer Logic: Check if SOURCE_PATH is a file or directory. -# ------------------------------------------------------------------ -echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" -if [ -f "$SOURCE_PATH" ]; then - # Case: Single file detected. - echo "[LOG] Single file detected. Transferring via hsi cput." - FILE_NAME=$(basename "$SOURCE_PATH") - echo "[LOG] File name: $FILE_NAME" - hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" - echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." - -elif [ -d "$SOURCE_PATH" ]; then - # Case: Directory detected. - echo "[LOG] Directory detected. Initiating bundling process." - THRESHOLD=2199023255552 # 2 TB in bytes. - echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" - - # ------------------------------------------------------------------ - # Group files based on modification date. - # ------------------------------------------------------------------ - echo "[LOG] Grouping files by modification date." - FILE_LIST=$(mktemp) - find "$SOURCE_PATH" -type f > "$FILE_LIST" - echo "[LOG] List of files stored in temporary file: $FILE_LIST" - - # Declare associative arrays to hold grouped file paths and sizes. - declare -A group_files - declare -A group_sizes - - while IFS= read -r file; do - mtime=$(stat -c %Y "$file") - year=$(date -d @"$mtime" +%Y) - month=$(date -d @"$mtime" +%m | sed 's/^0*//') - day=$(date -d @"$mtime" +%d | sed 's/^0*//') - # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. - if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then - cycle=1 - else - cycle=2 - fi - key="${{year}}-${{cycle}}" - group_files["$key"]="${{group_files["$key"]:-}} $file" - fsize=$(stat -c %s "$file") - group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) - done < "$FILE_LIST" - rm "$FILE_LIST" - echo "[LOG] Completed grouping files." - - # Print the files in each group at the end - for key in "${{!group_files[@]}}"; do - echo "[LOG] Group $key contains files:" - for f in ${{group_files["$key"]}}; do - echo " $f" - done - done - - # ------------------------------------------------------------------ - # Bundle files into tar archives. - # ------------------------------------------------------------------ - for key in "${{!group_files[@]}}"; do - files=(${{group_files["$key"]}}) - total_group_size=${{group_sizes["$key"]}} - echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." - - part=0 - current_size=0 - current_files=() - for f in "${{files[@]}}"; do - fsize=$(stat -c %s "$f") - # If adding this file exceeds the threshold, process the current bundle. - if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Bundle reached threshold." - echo "[LOG] Files in current bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." - htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - part=$((part+1)) - echo "[DEBUG] Resetting bundle variables." - current_files=() - current_size=0 - fi - current_files+=("$f") - current_size=$(( current_size + fsize )) - done - if [ ${{#current_files[@]}} -gt 0 ]; then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Final bundle for group $key:" - echo "[LOG] Files in final bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." - htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - fi - echo "[LOG] Completed processing group $key." - done -else - echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." - exit 1 -fi - -# ------------------------------------------------------------------ -# Logging: Display directory trees for both source and destination. -# ------------------------------------------------------------------ -echo "[LOG] Listing Source (CFS) Tree:" -if [ -d "$SOURCE_PATH" ]; then - find "$SOURCE_PATH" -print -else - echo "[LOG] $SOURCE_PATH is a file." + if hsi -q "mkdir $DEST_PATH" >/dev/null 2>&1; then + echo "[LOG] Created directory $DEST_PATH." + else + echo "[ERROR] Failed to create directory $DEST_PATH." + exit 1 + fi fi -echo "[LOG] Listing Destination (HPSS) Tree:" -hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" +# # ------------------------------------------------------------------ +# # Transfer Logic: Check if SOURCE_PATH is a file or directory. +# # ------------------------------------------------------------------ +# echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" +# if [ -f "$SOURCE_PATH" ]; then +# # Case: Single file detected. +# echo "[LOG] Single file detected. Transferring via hsi cput." +# FILE_NAME=$(basename "$SOURCE_PATH") +# echo "[LOG] File name: $FILE_NAME" +# hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" +# echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." + +# elif [ -d "$SOURCE_PATH" ]; then +# # Case: Directory detected. +# echo "[LOG] Directory detected. Initiating bundling process." +# THRESHOLD=2199023255552 # 2 TB in bytes. +# echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" + +# # ------------------------------------------------------------------ +# # Group files based on modification date. +# # ------------------------------------------------------------------ +# echo "[LOG] Grouping files by modification date." +# FILE_LIST=$(mktemp) +# find "$SOURCE_PATH" -type f > "$FILE_LIST" +# echo "[LOG] List of files stored in temporary file: $FILE_LIST" + +# # Declare associative arrays to hold grouped file paths and sizes. +# declare -A group_files +# declare -A group_sizes + +# while IFS= read -r file; do +# mtime=$(stat -c %Y "$file") +# year=$(date -d @"$mtime" +%Y) +# month=$(date -d @"$mtime" +%m | sed 's/^0*//') +# day=$(date -d @"$mtime" +%d | sed 's/^0*//') +# # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. +# if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then +# cycle=1 +# else +# cycle=2 +# fi +# key="${{year}}-${{cycle}}" +# group_files["$key"]="${{group_files["$key"]:-}} $file" +# fsize=$(stat -c %s "$file") +# group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) +# done < "$FILE_LIST" +# rm "$FILE_LIST" +# echo "[LOG] Completed grouping files." + +# # Print the files in each group at the end +# for key in "${{!group_files[@]}}"; do +# echo "[LOG] Group $key contains files:" +# for f in ${{group_files["$key"]}}; do +# echo " $f" +# done +# done + +# # ------------------------------------------------------------------ +# # Bundle files into tar archives. +# # ------------------------------------------------------------------ +# for key in "${{!group_files[@]}}"; do +# files=(${{group_files["$key"]}}) +# total_group_size=${{group_sizes["$key"]}} +# echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." + +# part=0 +# current_size=0 +# current_files=() +# for f in "${{files[@]}}"; do +# fsize=$(stat -c %s "$f") +# # If adding this file exceeds the threshold, process the current bundle. +# if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then +# if [ $part -eq 0 ]; then +# tar_name="${{FOLDER_NAME}}_${{key}}.tar" +# else +# tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" +# fi +# echo "[LOG] Bundle reached threshold." +# echo "[LOG] Files in current bundle:" +# for file in "${{current_files[@]}}"; do +# echo "$file" +# done +# echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." +# htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") +# part=$((part+1)) +# echo "[DEBUG] Resetting bundle variables." +# current_files=() +# current_size=0 +# fi +# current_files+=("$f") +# current_size=$(( current_size + fsize )) +# done +# if [ ${{#current_files[@]}} -gt 0 ]; then +# if [ $part -eq 0 ]; then +# tar_name="${{FOLDER_NAME}}_${{key}}.tar" +# else +# tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" +# fi +# echo "[LOG] Final bundle for group $key:" +# echo "[LOG] Files in final bundle:" +# for file in "${{current_files[@]}}"; do +# echo "$file" +# done +# echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." +# echo "[LOG] Bundle size: $current_size bytes." +# htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") +# fi +# echo "[LOG] Completed processing group $key." +# done +# else +# echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." +# exit 1 +# fi + +# # ------------------------------------------------------------------ +# # Logging: Display directory trees for both source and destination. +# # ------------------------------------------------------------------ +# echo "[LOG] Listing Source (CFS) Tree:" +# if [ -d "$SOURCE_PATH" ]; then +# find "$SOURCE_PATH" -print +# else +# echo "[LOG] $SOURCE_PATH is a file." +# fi + +# echo "[LOG] Listing Destination (HPSS) Tree:" +# hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" echo "[LOG] Job completed at: $(date)" """ diff --git a/requirements.txt b/requirements.txt index 24b8e9e4..52d6b872 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,8 @@ +authlib +dynaconf +globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk globus-sdk>=3.0 +griffe>=0.49.0,<2.0.0 h5py httpx>=0.22.0 mkdocs @@ -9,10 +13,7 @@ pillow pydantic==2.11 python-dotenv prefect==2.20.17 +prometheus_client==0.21.1 pyscicat pyyaml -authlib sfapi_client -globus-compute-sdk @ git+https://github.com/globus/globus-compute.git@d1731340074be56861ec91d732bdff44f8e2b46e#subdirectory=compute_sdk -griffe>=0.49.0,<2.0.0 -prometheus_client==0.21.1 From 343ef756b1a037aee4ee0dcbf51677b2a7c7f76d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 18 Feb 2025 16:46:20 -0800 Subject: [PATCH 010/128] Adding logic for the prune_filesystem_files flow in prune_controller.py --- orchestration/prune_controller.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 74b82a29..3b53afe2 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod import datetime import logging +import os from typing import Generic, TypeVar, Union from prefect import flow @@ -206,10 +207,33 @@ def prune_filesystem_files( Args: relative_path (str): The path of the file or directory to prune. - source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. - check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + source_endpoint (FileSystemEndpoint): The Globus source endpoint to prune from. + check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. """ - pass + logger.info(f"Running flow: prune_from_{source_endpoint.name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + + # Check if the file exists at the source endpoint + if not source_endpoint.exists(relative_path): + logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") + return + + # Check if the file exists at the check endpoint + if check_endpoint is not None and check_endpoint.exists(relative_path): + logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") + logger.info("Safe to prune.") + + # Check if it is a file or directory + if source_endpoint.is_dir(relative_path): + logger.info(f"Pruning directory {relative_path}") + source_endpoint.rmdir(relative_path) + else: + logger.info(f"Pruning file {relative_path}") + os.remove(source_endpoint.full_path(relative_path)) + else: + logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") + logger.warning("Not safe to prune.") + return @flow(name="prune_hpss_endpoint") From f2bc19b547f43665177e1db69d17a1c85c10abe1 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 19 Feb 2025 09:56:04 -0800 Subject: [PATCH 011/128] Moving specific prune implementations as private/internal methods in their respective PruneControllers, with Prefect Flow decorators. --- orchestration/prune_controller.py | 178 +++++++++++++++--------------- 1 file changed, 88 insertions(+), 90 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 3b53afe2..f5f641ac 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -82,6 +82,24 @@ def prune( ) return True + @flow(name="prune_hpss_endpoint") + def _prune_hpss_endpoint( + relative_path: str, + source_endpoint: HPSSEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None + ): + """ + Prune files from HPSS. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. + check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + # TODO: Implement HPSS pruning + pass + class FileSystemPruneController(PruneController[FileSystemEndpoint]): def __init__( @@ -114,6 +132,46 @@ def prune( ) return True + @flow(name="prune_filesystem_endpoint") + def _prune_filesystem_endpoint( + relative_path: str, + source_endpoint: FileSystemEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None + ): + """ + Prune files from a File System. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (FileSystemEndpoint): The Globus source endpoint to prune from. + check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + logger.info(f"Running flow: prune_from_{source_endpoint.name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + + # Check if the file exists at the source endpoint + if not source_endpoint.exists(relative_path): + logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") + return + + # Check if the file exists at the check endpoint + if check_endpoint is not None and check_endpoint.exists(relative_path): + logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") + logger.info("Safe to prune.") + + # Check if it is a file or directory + if source_endpoint.is_dir(relative_path): + logger.info(f"Pruning directory {relative_path}") + source_endpoint.rmdir(relative_path) + else: + logger.info(f"Pruning file {relative_path}") + os.remove(source_endpoint.full_path(relative_path)) + else: + logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") + logger.warning("Not safe to prune.") + return + class GlobusPruneController(PruneController[GlobusEndpoint]): def __init__( @@ -149,6 +207,36 @@ def prune( ) return True + @flow(name="prune_globus_endpoint") + def _prune_globus_endpoint( + relative_path: str, + source_endpoint: GlobusEndpoint, + check_endpoint: Union[GlobusEndpoint, None] = None, + config: BeamlineConfig = None + ) -> None: + """ + Prune files from a Globus endpoint. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. + check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + globus_settings = JSON.load("globus-settings").value + max_wait_seconds = globus_settings["max_wait_seconds"] + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + prune_one_safe( + file=relative_path, + if_older_than_days=0, + tranfer_client=config.tc, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + logger=logger, + max_wait_seconds=max_wait_seconds + ) + def get_prune_controller( endpoint: TransferEndpoint, @@ -162,93 +250,3 @@ def get_prune_controller( return GlobusPruneController(config) else: raise ValueError(f"Unsupported endpoint type: {type(endpoint)}") - - -@flow(name="prune_globus_endpoint") -def prune_globus_files( - relative_path: str, - source_endpoint: GlobusEndpoint, - check_endpoint: Union[GlobusEndpoint, None] = None, - config: BeamlineConfig = None -): - """ - Prune files from a source endpoint. - - Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. - check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. - """ - globus_settings = JSON.load("globus-settings").value - max_wait_seconds = globus_settings["max_wait_seconds"] - flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") - prune_one_safe( - file=relative_path, - if_older_than_days=0, - tranfer_client=config.tc, - source_endpoint=source_endpoint, - check_endpoint=check_endpoint, - logger=logger, - max_wait_seconds=max_wait_seconds - ) - - -@flow(name="prune_filesystem_endpoint") -def prune_filesystem_files( - relative_path: str, - source_endpoint: FileSystemEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, - config: BeamlineConfig = None -): - """ - Prune files from a source endpoint. - - Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (FileSystemEndpoint): The Globus source endpoint to prune from. - check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. - """ - logger.info(f"Running flow: prune_from_{source_endpoint.name}") - logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") - - # Check if the file exists at the source endpoint - if not source_endpoint.exists(relative_path): - logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") - return - - # Check if the file exists at the check endpoint - if check_endpoint is not None and check_endpoint.exists(relative_path): - logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") - logger.info("Safe to prune.") - - # Check if it is a file or directory - if source_endpoint.is_dir(relative_path): - logger.info(f"Pruning directory {relative_path}") - source_endpoint.rmdir(relative_path) - else: - logger.info(f"Pruning file {relative_path}") - os.remove(source_endpoint.full_path(relative_path)) - else: - logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") - logger.warning("Not safe to prune.") - return - - -@flow(name="prune_hpss_endpoint") -def prune_hpss_files( - relative_path: str, - source_endpoint: HPSSEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, - config: BeamlineConfig = None -): - """ - Prune files from a source endpoint. - - Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. - check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. - """ - pass From 7c686c1f9f5b3d2d7ca61b149f381669a604e094 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 19 Feb 2025 11:46:56 -0800 Subject: [PATCH 012/128] Adding documentation outlining shared infrastructure between beamline implementations --- docs/mkdocs/docs/common_infrastructure.md | 46 +++++++++++++++++++++++ docs/mkdocs/mkdocs.yml | 6 ++- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 docs/mkdocs/docs/common_infrastructure.md diff --git a/docs/mkdocs/docs/common_infrastructure.md b/docs/mkdocs/docs/common_infrastructure.md new file mode 100644 index 00000000..b462336d --- /dev/null +++ b/docs/mkdocs/docs/common_infrastructure.md @@ -0,0 +1,46 @@ +# Common Infrastructure + +## Overview +The common infrastructure for this project includes: +- **Shared Code**: There are general functions and classes used across beamline workflows to reduce code duplication. +- **Beamline Specific Implementation Patterns**: We organize each beamline's implementation in a similar way, making it easier to understand and maintain. + +## Shared Code +Shared code is organized into modules that can be imported in beamline specific implementations. Key modules include: +- **`orchestration/config.py`** + - Contains an Abstract Base Class (ABC) called `BeamlineConfig()` which serves as the base for all beamline-specific configuration classes. It uses the `Dynaconf` package to load the configuration file,`config.yml`, which contains information about endpoints, containers, and more. +- **`orchestration/transfer_endpoints.py`** + - Contains an ABC called `TransferEndpoint()`, which is extended by `FileSystemEndpoint`, `HPSSEndpoint` and `GlobusEndpoint`. These definitions are used to enforce typing and ensure the correct transfer and pruning implmentation are used. +- **`orchestration/transfer_controller.py`**: + - Contains an ABC called `TransferController()` with specific implementations for Globus, Local File Systems, and NERSC HPSS. +- **`orchestration/prune_controller.py`** + - This module is responsible for managing the pruning of data off of storage systems. It uses a configurable retention policy to determine when to remove files. It contains an ABC called `PruneController()` that is extended by specific implementations for `FileSystemEndpoint`, `GlobusEndpoint`, and `HPSSEndpoint`. +- **`orchestration/sfapi.py`**: Create an SFAPI Client to launch remote jobs at NERSC. + +## Beamline Specific Implementation Patterns +In order to balance generalizability, maintainability, and scalability of this project to multiple beamlines, we try to organize specific implementations in a similar way. We keep specific implementaqtions in the directory `orchestration/flows/bl{beamline_id}/`, which generally contains a few things: +- **`config.py`** + - Extend `BeamlineConfig()` from `orchestration/config.py` for specific implementations (e.g. `Config832`, `Config733`, etc.) This ensures only the relevant beamline specific configurations are used in each case. +- **`dispatcher.py`** + - This script is the starting point for each beamline's data transfer and analysis workflow. The Prefect Flow it contains is generally invoked by a File Watcher script on the beamline computer. The Dispatcher contains the logic for calling subflows, ensures that steps are completed in the correct order, and prevents subsequent steps from being called if there is a failure along the way. +- **`move.py`** + - This script is usually the first one the Dispatcher calls synchronously, and contains the logic for immediately moving data, scheduling pruning flows, and ingesting into SciCat. Downstream steps typically rely on this action completing first. +- **`job_controller.py`** + - For beamlines that trigger remote analysis workflows, the `JobController()` ABC allows us to define HPC or machine specific implementations, which may differ in how code can be deployed. For example, it can be extended to define how to run tomography reconstruction at ALCF and NERSC. +- **`{hpc}.py`** + - We separate HPC implementations for `JobController()` in their own files. +- **`hpss.py`** + - We define HPSS transfers for each beamline individually, as we provide different scheduling strategies based on the data throughput of each endstation. +- **`ingest.py`** + - This is where we define SciCat implementations for each beamline, as each technique will have specific metadata fields that are important to capture. + +## Testing +We write Unit Tests using [pytest](https://pytest.org/) for individual components, which can be found in `orchestration/_tests/`. We run these tests as part of our Github Actions. + +## CI/CD +The project is integrated with [GitHub Actions](https://github.com/features/actions) for continuous integration and deployment. The specifics for these can be found in `.github/workflows/`. The features we support here includes: + +- **Automated Test Execution**: All the unit tests are run automatically with every Git Push. +- **Linting**: `flake8` is used to check for syntax and styling errors. +- **MkDocs**: The documentation site is automatically updated whenever a Pull Request is merged into the main branch. +- **Docker**: A Docker image is aumatically created and registered on the Github Container Repository (ghcr.io) when a new release is made. \ No newline at end of file diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 68bcc959..6e6c84fc 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -13,8 +13,10 @@ nav: - Home: index.md - Installation and Requirements: install.md - Getting Started: getting_started.md -- Compute at ALCF: alcf832.md -- Compute at NERSC: nersc832.md +- Common Infrastructure: common_infrastructure.md +- Beamline Implementations: + - 8.3.2 Micro Tomography - Compute at ALCF: alcf832.md + - 8.3.2 Micro Tomography - Compute at NERSC: nersc832.md - Orchestration: orchestration.md - Configuration: configuration.md - HPSS Tape Archive Access: hpss.md From 3071709753b586e5e0badd03067231960ccfb296 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 19 Feb 2025 14:12:11 -0800 Subject: [PATCH 013/128] Linting and adding a few TODO comments --- orchestration/config.py | 1 + orchestration/flows/bl832/dispatcher.py | 2 +- orchestration/flows/bl832/ingest_tomo832.py | 6 ++++-- orchestration/flows/scicat/utils.py | 5 +---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/orchestration/config.py b/orchestration/config.py index 318b5fe3..a13014fc 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -7,6 +7,7 @@ from dynaconf import Dynaconf +# TODO: Add secrets management settings = Dynaconf( settings_files=["config.yml"], ) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index cd9da2f0..acb8c6bd 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -105,7 +105,7 @@ async def run_recon_flow_async(flow_name: str, parameters: dict) -> None: async def dispatcher( file_path: Optional[str] = None, is_export_control: bool = False, - config: Optional[Union[dict, Any]] = None, + config: Optional[Union[dict, Any]] = None, # TODO: Define the type of config to be BeamlineConfig ) -> None: """ Dispatcher flow that reads decision settings and launches tasks accordingly. diff --git a/orchestration/flows/bl832/ingest_tomo832.py b/orchestration/flows/bl832/ingest_tomo832.py index 096039e2..e694daaf 100644 --- a/orchestration/flows/bl832/ingest_tomo832.py +++ b/orchestration/flows/bl832/ingest_tomo832.py @@ -105,7 +105,8 @@ def ingest( file_path, dataset_id, INGEST_STORAGE_ROOT_PATH, - INGEST_SOURCE_ROOT_PATH) + INGEST_SOURCE_ROOT_PATH + ) thumbnail_file = build_thumbnail(file["/exchange/data"][0]) encoded_thumbnail = encode_image_2_thumbnail(thumbnail_file) @@ -113,7 +114,8 @@ def ingest( scicat_client, encoded_thumbnail, dataset_id, - ownable) + ownable + ) return dataset_id diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 7fc97012..6884f2d4 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -5,10 +5,8 @@ import io import json import logging -from pathlib import Path import re from typing import Dict, Optional, Union -from uuid import uuid4 import numpy as np import numpy.typing as npt @@ -17,6 +15,7 @@ logger = logging.getLogger("splash_ingest") can_debug = logger.isEnabledFor(logging.DEBUG) + class Severity(str, Enum): warning = "warning" error = "error" @@ -90,5 +89,3 @@ def build_thumbnail(image_array: npt.ArrayLike): auto_contrast_image.save(file, format="png") file.seek(0) return file - - From a68009c011820814ca489e6f3a55332f177259dd Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 19 Feb 2025 14:49:26 -0800 Subject: [PATCH 014/128] Fixed get_prune_controller() method to accept an Enum for supported types. Updated documentation. Added a new file orchestration/flows/bl832/move_refactor.py that will replace orchestration/flows/bl832/move.py --- docs/mkdocs/docs/common_infrastructure.md | 1 + orchestration/flows/bl832/move_refactor.py | 145 +++++++++++++++++++++ orchestration/prune_controller.py | 139 ++++++++++++-------- 3 files changed, 228 insertions(+), 57 deletions(-) create mode 100644 orchestration/flows/bl832/move_refactor.py diff --git a/docs/mkdocs/docs/common_infrastructure.md b/docs/mkdocs/docs/common_infrastructure.md index b462336d..6e6ca5f0 100644 --- a/docs/mkdocs/docs/common_infrastructure.md +++ b/docs/mkdocs/docs/common_infrastructure.md @@ -16,6 +16,7 @@ Shared code is organized into modules that can be imported in beamline specific - **`orchestration/prune_controller.py`** - This module is responsible for managing the pruning of data off of storage systems. It uses a configurable retention policy to determine when to remove files. It contains an ABC called `PruneController()` that is extended by specific implementations for `FileSystemEndpoint`, `GlobusEndpoint`, and `HPSSEndpoint`. - **`orchestration/sfapi.py`**: Create an SFAPI Client to launch remote jobs at NERSC. +- **`orchestration/flows/scicat/ingest.py`**: Ingests datasets into SciCat, our metadata management system. ## Beamline Specific Implementation Patterns In order to balance generalizability, maintainability, and scalability of this project to multiple beamlines, we try to organize specific implementations in a similar way. We keep specific implementaqtions in the directory `orchestration/flows/bl{beamline_id}/`, which generally contains a few things: diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py new file mode 100644 index 00000000..dbe22ea5 --- /dev/null +++ b/orchestration/flows/bl832/move_refactor.py @@ -0,0 +1,145 @@ +import datetime +import logging +import os +from pathlib import Path +import uuid + +from prefect import flow, task +from prefect.blocks.system import JSON + +from orchestration.flows.scicat.ingest import ingest_dataset +from orchestration.flows.bl832.config import Config832 +from orchestration.globus.transfer import start_transfer +from orchestration.prune_controller import get_prune_controller, PruneMethod +from orchestration.transfer_controller import get_transfer_controller, CopyMethod + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +API_KEY = os.getenv("API_KEY") +TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832" + + +@flow(name="new_832_file_flow") +def process_new_832_file( + file_path: str, + send_to_nersc=True, + config: Config832 = None +) -> None: + """ + Sends a file along a path: + - Copy from spot832 to data832 + - Copy from data832 to NERSC + - Ingest into SciCat + - Schedule a job to delete from spot832 in the future + - Schedule a job to delete from data832 in the future + + :param file_path: path to file on spot832 + :param send_to_nersc: if True, send to NERSC and ingest into SciCat + """ + + logger.info("Starting New 832 File Flow") + if not config: + config = Config832() + + # paths come in from the app on spot832 as /global/raw/... + # remove 'global' so that all paths start with 'raw', which is common + # to all 3 systems. + logger.info(f"Transferring {file_path} from spot to data") + relative_path = file_path.split("/global")[1] + + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.GLOBUS, + config=config + ) + + data832_transfer_success = transfer_controller.copy( + file_path=relative_path, + source=config.spot832, + destination=config.data832, + ) + + if send_to_nersc and data832_transfer_success: + nersc_transfer_success = transfer_controller.copy( + file_path=relative_path, + source=config.data832, + destination=config.nersc832 + ) + + if nersc_transfer_success: + logger.info( + f"File successfully transferred from data832 to NERSC {file_path}. Task {task}" + ) + logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") + try: + ingest_dataset(file_path, TOMO_INGESTOR_MODULE) + except Exception as e: + logger.error(f"SciCat ingest failed with {e}") + + bl832_settings = JSON.load("bl832-settings").value + + schedule_spot832_delete_days = bl832_settings["delete_spot832_files_after_days"] + schedule_data832_delete_days = bl832_settings["delete_data832_files_after_days"] + + prune_controller = get_prune_controller( + prune_type=PruneMethod.GLOBUS, + config=config + ) + + prune_controller.prune( + file_path=relative_path, + source_endpoint=config.spot832, + check_endpoint=config.data832, + days_from_now=datetime.timedelta(days=schedule_spot832_delete_days) + ) + logger.info( + f"Scheduled delete from spot832 at {datetime.timedelta(days=schedule_spot832_delete_days)}" + ) + + prune_controller.prune( + file_path=relative_path, + source_endpoint=config.data832, + check_endpoint=config.nersc832, + days_from_now=datetime.timedelta(days=schedule_data832_delete_days) + ) + logger.info( + f"Scheduled delete from data832 at {datetime.timedelta(days=schedule_data832_delete_days)}" + ) + + return + + +@flow(name="test_832_transfers") +def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): + config = Config832() + logger.info(f"{str(uuid.uuid4())}{file_path}") + # copy file to a uniquely-named file in the same folder + file = Path(file_path) + new_file = str(file.with_name(f"test_{str(uuid.uuid4())}.txt")) + logger.info(new_file) + + success = start_transfer( + config.tc, config.spot832, file_path, config.spot832, new_file, logger=logger + ) + + logger.info(success) + + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.GLOBUS, + config=config + ) + + dat832_success = transfer_controller.copy( + file_path=new_file, + source=config.spot832, + destination=config.data832, + ) + logger.info(f"Transferred {new_file} from spot to data. Success: {dat832_success}") + + nersc_success = transfer_controller.copy( + file_path=new_file, + source=config.data832, + destination=config.nersc832, + ) + logger.info(f"File successfully transferred from data832 to NERSC {new_file}. Success: {nersc_success}") + pass diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index f5f641ac..c49a51e0 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod import datetime +from enum import Enum import logging import os from typing import Generic, TypeVar, Union @@ -51,56 +52,6 @@ def prune( pass -class HPSSPruneController(PruneController[HPSSEndpoint]): - def __init__( - self, - config: BeamlineConfig, - ) -> None: - super().__init__(config) - - def prune( - self, - file_path: str = None, - source_endpoint: HPSSEndpoint = None, - check_endpoint: FileSystemEndpoint = None, - days_from_now: datetime.timedelta = 0 - ) -> bool: - flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - schedule_prefect_flow( - "prune_hpss_endpoint/prune_hpss_endpoint", - flow_name, - { - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - - datetime.timedelta(days=days_from_now), - ) - return True - - @flow(name="prune_hpss_endpoint") - def _prune_hpss_endpoint( - relative_path: str, - source_endpoint: HPSSEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, - config: BeamlineConfig = None - ): - """ - Prune files from HPSS. - - Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. - check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. - """ - # TODO: Implement HPSS pruning - pass - - class FileSystemPruneController(PruneController[FileSystemEndpoint]): def __init__( self, @@ -238,15 +189,89 @@ def _prune_globus_endpoint( ) +class HPSSPruneController(PruneController[HPSSEndpoint]): + def __init__( + self, + config: BeamlineConfig, + ) -> None: + super().__init__(config) + + def prune( + self, + file_path: str = None, + source_endpoint: HPSSEndpoint = None, + check_endpoint: FileSystemEndpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + schedule_prefect_flow( + "prune_hpss_endpoint/prune_hpss_endpoint", + flow_name, + { + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + + datetime.timedelta(days=days_from_now), + ) + return True + + @flow(name="prune_hpss_endpoint") + def _prune_hpss_endpoint( + relative_path: str, + source_endpoint: HPSSEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None + ): + """ + Prune files from HPSS. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. + check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + # TODO: Implement HPSS pruning + pass + + +class PruneMethod(Enum): + """ + Enum representing different prune methods. + Use enum names as strings to identify trpruneansfer methods, ensuring a standard set of values. + """ + GLOBUS = "globus" + SIMPLE = "simple" + HPSS = "hpss" + + def get_prune_controller( - endpoint: TransferEndpoint, + prune_type: PruneMethod, config: BeamlineConfig ) -> PruneController: - if isinstance(endpoint, HPSSEndpoint): - return HPSSPruneController(config) - elif isinstance(endpoint, FileSystemEndpoint): - return FileSystemPruneController(config) - elif isinstance(endpoint, GlobusEndpoint): + """ + Get the appropriate prune controller based on the prune type. + + Args: + prune_type (str): The type of transfer to perform. + config (BeamlineConfig): The configuration object. + + Returns: + PruneController: The transfer controller object. + """ + if prune_type == PruneMethod.GLOBUS: return GlobusPruneController(config) + elif prune_type == PruneMethod.SIMPLE: + return FileSystemPruneController(config) + elif prune_type == PruneMethod.CFS_TO_HPSS: + from orchestration.sfapi import create_sfapi_client + return HPSSPruneController( + client=create_sfapi_client(), + config=config + ) else: - raise ValueError(f"Unsupported endpoint type: {type(endpoint)}") + raise ValueError(f"Invalid transfer type: {prune_type}") From f5af67640fbc83dfe39856197f17d1acc6f05f9d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 21 Feb 2025 16:55:31 -0800 Subject: [PATCH 015/128] Working on a SciCat Ingestor Controller ABC, and a BL832 implementation, including how to append a new dataset location to an existing dataset in scicat. --- docs/mkdocs/docs/scicat.md | 424 ++++++++++++++++++ orchestration/flows/bl832/scicat_ingestor.py | 83 ++++ .../flows/scicat/ingestor_controller.py | 79 ++++ 3 files changed, 586 insertions(+) create mode 100644 docs/mkdocs/docs/scicat.md create mode 100644 orchestration/flows/bl832/scicat_ingestor.py create mode 100644 orchestration/flows/scicat/ingestor_controller.py diff --git a/docs/mkdocs/docs/scicat.md b/docs/mkdocs/docs/scicat.md new file mode 100644 index 00000000..75398afd --- /dev/null +++ b/docs/mkdocs/docs/scicat.md @@ -0,0 +1,424 @@ +# SciCat + +## Overview +SciCat is a data management system for scientific data. It provides tools to manage, organize, and share research data effectively. + +## Features +- **Data Management**: Efficient storage and organization of scientific datasets. +- **User Authentication**: Secure access control for users. +- **Metadata Management**: Manage metadata associated with your data. +- **Data Sharing**: Share data securely with collaborators. +- **Integration**: Integrate with other tools and workflows. + +## Workflow Diagram + +```mermaid +flowchart TD + A["Start Ingest Flow"] --> B["ingest_dataset Flow"] + B --> C["ingest_dataset_task Task"] + C --> D["Load Environment Variables"] + D --> E["Initialize SciCat Client"] + E --> F["Dynamic Import of Beamline Ingestor Module"] + F --> G["Call ingestor.ingest() in ingest_tomo832.py"] + G --> H["Open HDF5 File (h5py)"] + H --> I["Extract SciCat & Scientific Metadata"] + I --> J["Compute Access Controls & Clean Data"] + J --> K["Upload Raw Dataset (Create Dataset Object)"] + K --> L["Upload Data Block (File Info Mapping)"] + L --> M["Build Thumbnail from Data Array"] + M --> N["Encode Thumbnail to Base64"] + N --> O["Upload Attachment (Thumbnail)"] + O --> P["Return Dataset ID"] +``` + +## Getting Started + +In `splash_flows_globus`, we "ingest" our datasets into SciCat during our file movement workflows. In the directory `orchestration/flows/scicat/` there are two general scripts: `ingest.py` and `utils.py`. Since the data from each beamline is different, we define specific ingest implementations, such as `orchestration/flows/bl832/ingest_tomo832.py`. + +## SciCat Client API + +# API Documentation + +This document details the API provided by the `ScicatClient` class and its associated utility functions for interacting with the SciCat Catamel server. + +--- + +## Overview + +The `ScicatClient` class offers a comprehensive interface for communicating with the SciCat server via HTTP. It supports operations such as creating, updating, retrieving, and deleting datasets, samples, instruments, proposals, and published data. The client utilizes token-based authentication and provides helper functions to work with file metadata and image encoding. + +--- + +## ScicatClient Class + +### Initialization + + ScicatClient( + base_url: str, + token: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + timeout_seconds: Optional[int] = None + ) + +- **Parameters:** + - `base_url`: Base URL for the SciCat API (e.g., "http://localhost:3000/api/v3/"). + - `token`: (Optional) A pre-obtained authentication token. + - `username`: (Optional) Username for login. + - `password`: (Optional) Password for login. + - `timeout_seconds`: (Optional) Timeout in seconds for HTTP requests. +- **Behavior:** If no token is provided, the client attempts to log in using the provided username and password, retrieves a token, and sets the appropriate HTTP headers. +- **Raises:** An assertion error if neither a token nor valid login credentials are provided. + +--- + +### Internal Methods + +#### _send_to_scicat + + _send_to_scicat(cmd: str, endpoint: str, data: Optional[BaseModel] = None) + +- **Purpose:** Sends an HTTP request to the SciCat server. +- **Parameters:** + - `cmd`: The HTTP method (e.g., "post", "patch", "get", "delete"). + - `endpoint`: API endpoint to append to the base URL. + - `data`: (Optional) A `pydantic.BaseModel` instance representing the payload. +- **Returns:** The HTTP response object from the request. + +#### _call_endpoint + + _call_endpoint( + cmd: str, + endpoint: str, + data: Optional[BaseModel] = None, + operation: str = "" + ) -> Optional[dict] + +- **Purpose:** Calls a specific API endpoint, handles JSON parsing, and checks for errors. +- **Parameters:** + - `cmd`: The HTTP method. + - `endpoint`: The specific endpoint to call. + - `data`: (Optional) Data to include in the request body. + - `operation`: (Optional) A string identifier for the operation, used in logging. +- **Returns:** A dictionary containing the parsed JSON response. +- **Raises:** `ScicatCommError` if the server responds with an error status. + +--- + +## Dataset Operations + +### Create Dataset + + datasets_create(dataset: Dataset) -> str + +- **Purpose:** Uploads a new dataset. +- **Parameters:** + - `dataset`: An instance of the `Dataset` model. +- **Returns:** A string representing the unique identifier (PID) of the created dataset. +- **Aliases:** `upload_new_dataset`, `create_dataset`. + +### Update Dataset + + datasets_update(dataset: Dataset, pid: str) -> str + +- **Purpose:** Updates an existing dataset. +- **Parameters:** + - `dataset`: An instance of the `Dataset` model with updated fields. + - `pid`: The unique identifier of the dataset to update. +- **Returns:** A string representing the updated dataset's PID. +- **Alias:** `update_dataset`. + +### Create Dataset OrigDatablock + + datasets_origdatablock_create( + dataset_id: str, + datablockDto: CreateDatasetOrigDatablockDto + ) -> dict + +- **Purpose:** Creates an original datablock for a specified dataset. +- **Parameters:** + - `dataset_id`: The unique identifier of the dataset. + - `datablockDto`: A data transfer object containing the datablock details. +- **Returns:** A dictionary representing the created datablock. +- **Aliases:** `upload_dataset_origdatablock`, `create_dataset_origdatablock`. + +### Create Dataset Attachment + + datasets_attachment_create( + attachment: Attachment, + datasetType: str = "Datasets" + ) -> dict + +- **Purpose:** Uploads an attachment to a dataset. +- **Parameters:** + - `attachment`: An instance of the `Attachment` model. + - `datasetType`: (Optional) The type of dataset; default is "Datasets". +- **Returns:** A dictionary containing details of the uploaded attachment. +- **Aliases:** `upload_attachment`, `create_dataset_attachment`. + +### Find Datasets (Full Query) + + datasets_find( + skip: int = 0, + limit: int = 25, + query_fields: Optional[dict] = None + ) -> Optional[dict] + +- **Purpose:** Retrieves datasets using a full text search query. +- **Parameters:** + - `skip`: Number of records to skip (for pagination). + - `limit`: Maximum number of records to return. + - `query_fields`: (Optional) A dictionary specifying search criteria. +- **Returns:** A dictionary with the query results. +- **Aliases:** `get_datasets_full_query`, `find_datasets_full_query`. + +### Get Many Datasets (Simple Filter) + + datasets_get_many(filter_fields: Optional[dict] = None) -> Optional[dict] + +- **Purpose:** Retrieves datasets based on simple filtering criteria. +- **Parameters:** + - `filter_fields`: A dictionary containing the filter conditions. +- **Returns:** A dictionary with the filtered datasets. +- **Aliases:** `get_datasets`, `find_datasets`. + +### Get Single Dataset + + datasets_get_one(pid: str) -> Optional[dict] + +- **Purpose:** Retrieves a single dataset by its PID. +- **Parameters:** + - `pid`: The unique identifier of the dataset. +- **Returns:** A dictionary with the dataset details. +- **Alias:** `get_dataset_by_pid`. + +### Delete Dataset + + datasets_delete(pid: str) -> Optional[dict] + +- **Purpose:** Deletes a dataset identified by its PID. +- **Parameters:** + - `pid`: The unique identifier of the dataset to delete. +- **Returns:** A dictionary containing the server's response. +- **Alias:** `delete_dataset`. + +--- + +## Sample Operations + +### Create Sample + + samples_create(sample: Sample) -> str + +- **Purpose:** Creates a new sample. +- **Parameters:** + - `sample`: An instance of the `Sample` model. +- **Returns:** A string representing the newly created sample ID. +- **Alias:** `upload_sample`. + +### Update Sample + + samples_update(sample: Sample, sampleId: Optional[str] = None) -> str + +- **Purpose:** Updates an existing sample. +- **Parameters:** + - `sample`: An instance of the `Sample` model with updated values. + - `sampleId`: (Optional) The unique identifier of the sample; if omitted, the sampleā€™s own `sampleId` is used. +- **Returns:** A string representing the updated sample ID. + +### Get Single Sample + + samples_get_one(pid: str) -> Optional[dict] + +- **Purpose:** Retrieves a sample by its PID. +- **Parameters:** + - `pid`: The unique sample identifier. +- **Returns:** A dictionary with the sample details. +- **Alias:** `get_sample`. + +--- + +## Instrument Operations + +### Create Instrument + + instruments_create(instrument: Instrument) -> str + +- **Purpose:** Creates a new instrument. Admin rights may be required. +- **Parameters:** + - `instrument`: An instance of the `Instrument` model. +- **Returns:** A string representing the instrument's unique identifier (PID). +- **Alias:** `upload_instrument`. + +### Update Instrument + + instruments_update(instrument: Instrument, pid: Optional[str] = None) -> str + +- **Purpose:** Updates an existing instrument. +- **Parameters:** + - `instrument`: An instance of the `Instrument` model with updated fields. + - `pid`: (Optional) The unique identifier of the instrument; if omitted, the instrumentā€™s own `pid` is used. +- **Returns:** A string representing the updated instrument PID. + +### Get Single Instrument + + instruments_get_one(pid: Optional[str] = None, name: Optional[str] = None) -> Optional[dict] + +- **Purpose:** Retrieves an instrument by its PID or by name. +- **Parameters:** + - `pid`: (Optional) The unique instrument identifier. + - `name`: (Optional) The instrument name (used if PID is not provided). +- **Returns:** A dictionary with the instrument details. +- **Alias:** `get_instrument`. + +--- + +## Proposal Operations + +### Create Proposal + + proposals_create(proposal: Proposal) -> str + +- **Purpose:** Creates a new proposal. Admin rights may be required. +- **Parameters:** + - `proposal`: An instance of the `Proposal` model. +- **Returns:** A string representing the newly created proposal ID. +- **Alias:** `upload_proposal`. + +### Update Proposal + + proposals_update(proposal: Proposal, proposalId: Optional[str] = None) -> str + +- **Purpose:** Updates an existing proposal. +- **Parameters:** + - `proposal`: An instance of the `Proposal` model with updated information. + - `proposalId`: (Optional) The unique identifier of the proposal; if omitted, the proposalā€™s own `proposalId` is used. +- **Returns:** A string representing the updated proposal ID. + +### Get Single Proposal + + proposals_get_one(pid: str) -> Optional[dict] + +- **Purpose:** Retrieves a proposal by its PID. +- **Parameters:** + - `pid`: The unique proposal identifier. +- **Returns:** A dictionary with the proposal details. +- **Alias:** `get_proposal`. + +--- + +## Published Data Operations + +### Get Published Data + + published_data_get_many(filter=None) -> Optional[dict] + +- **Purpose:** Retrieves published datasets based on optional filter criteria. +- **Parameters:** + - `filter`: (Optional) A dictionary specifying filter conditions. +- **Returns:** A dictionary containing the published data. +- **Aliases:** `get_published_data`, `find_published_data`. + +--- + +## Additional Dataset Operations + +### Get Dataset OrigDatablocks + + datasets_origdatablocks_get_one(pid: str) -> Optional[dict] + +- **Purpose:** Retrieves the original datablocks associated with a dataset. +- **Parameters:** + - `pid`: The unique identifier of the dataset. +- **Returns:** A dictionary with the original datablock details. +- **Alias:** `get_dataset_origdatablocks`. + +--- + +## Utility Functions + +### File Utilities + +#### Get File Size + + get_file_size(pathobj: Path) + +- **Purpose:** Returns the size of a file in bytes. +- **Parameters:** + - `pathobj`: A `Path` object representing the file. +- **Returns:** The file size as an integer. + +#### Get Checksum + + get_checksum(pathobj: Path) + +- **Purpose:** Computes the MD5 checksum of a file. +- **Parameters:** + - `pathobj`: A `Path` object representing the file. +- **Returns:** The MD5 checksum as a hexadecimal string. + +#### Encode Thumbnail + + encode_thumbnail(filename, imType="jpg") + +- **Purpose:** Encodes an image file as a Base64 data URL, suitable for use as a thumbnail. +- **Parameters:** + - `filename`: Path to the image file. + - `imType`: (Optional) Image format (default is "jpg"). +- **Returns:** A string containing the Base64 encoded image prefixed with the appropriate data URL header. + +#### Get File Modification Time + + get_file_mod_time(pathobj: Path) + +- **Purpose:** Retrieves the last modification time of a file. +- **Parameters:** + - `pathobj`: A `Path` object representing the file. +- **Returns:** A string representation of the file's modification time. + +### Authentication Helpers + +#### Create Client from Token + + from_token(base_url: str, token: str) + +- **Purpose:** Instantiates a `ScicatClient` using an existing authentication token. +- **Parameters:** + - `base_url`: Base URL for the SciCat API. + - `token`: A valid authentication token. +- **Returns:** An instance of `ScicatClient`. + +#### Create Client from Credentials + + from_credentials(base_url: str, username: str, password: str) + +- **Purpose:** Instantiates a `ScicatClient` by logging in with username and password. +- **Parameters:** + - `base_url`: Base URL for the SciCat API. + - `username`: Login username. + - `password`: Login password. +- **Returns:** An instance of `ScicatClient`. + +#### Retrieve Token + + get_token(base_url, username, password) + +- **Purpose:** Logs in using provided credentials and retrieves an authentication token. +- **Parameters:** + - `base_url`: Base URL for the SciCat API. + - `username`: Login username. + - `password`: Login password. +- **Returns:** An authentication token as a string. +- **Behavior:** Attempts login via the `Users/login` and `auth/msad` endpoints. + +--- + +## Exception Classes + +- **ScicatLoginError** + - Raised when an error occurs during the login process. + - Contains an error message describing the issue. + +- **ScicatCommError** + - Raised when communication with the SciCat server fails (non-20x HTTP responses). + - Contains an error message describing the issue. diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py new file mode 100644 index 00000000..eb0a8320 --- /dev/null +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -0,0 +1,83 @@ +import importlib +from logging import getLogger +from typing import List + +from pyscicat.client import ScicatClient + +from orchestration.flows.bl832.config import Config832 +from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController +from orchestration.flows.scicat.utils import Issue + + +logger = getLogger(__name__) + + +class TomographyIngestorController(BeamlineIngestorController): + """ + Ingestor for Tomo832 beamline. + """ + + def __init__( + self, + config: Config832, + scicat_client: ScicatClient + ) -> None: + super().__init__(config, scicat_client) + + def ingest_new_raw_dataset( + self, + file_path: str = "", + ) -> str: + """ + Ingest a new raw dataset from the Tomo832 beamline. + + :param file_path: Path to the file to ingest. + :return: SciCat ID of the dataset. + """ + + # Ingest the dataset + ingestor_module = "orchestration.flows.bl832.ingest_tomo832" + ingestor_module = importlib.import_module(ingestor_module) + issues: List[Issue] = [] + new_dataset_id = ingestor_module.ingest( + self.scicat_client, + file_path, + issues, + ) + if len(issues) > 0: + logger.error(f"SciCat ingest failed with {len(issues)} issues") + for issue in issues: + logger.error(issue) + raise Exception("SciCat ingest failed") + return new_dataset_id + + def ingest_new_derived_dataset( + self, + file_path: str = "", + raw_dataset_id: str = "", + ) -> str: + """ + Ingest a new derived dataset from the Tomo832 beamline. + + :param file_path: Path to the file to ingest. + :return: SciCat ID of the dataset. + """ + pass + + def add_new_dataset_location( + self, + dataset_id: str, + location: str, + ) -> None: + """ + Add a new location to an existing dataset in SciCat. + + :param dataset_id: SciCat ID of the dataset. + :param location: Path to the location to add. + """ + + dataset = self.scicat_client.datasets_get_one(dataset_id) + dataset["locations"].append(location) + self.scicat_client.update_dataset(dataset, dataset_id) + logger.info(f"Added location {location} to dataset {dataset_id}") + return diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py new file mode 100644 index 00000000..64eb92a1 --- /dev/null +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -0,0 +1,79 @@ +from abc import ABC, abstractmethod + +from orchestration.config import BeamlineConfig +from pyscicat.client import ScicatClient, from_credentials + + +class BeamlineIngestorController(ABC): + """ + Abstract class for beamline ingestors. + Provides interface methods for ingesting data. + """ + + def __init__( + self, + config: BeamlineConfig, + scicat_client: ScicatClient + ) -> None: + self.config = config + self.scicat_client = scicat_client + + def _login_to_scicat( + self, + scicat_base_url: str, + scicat_user: str, + scicat_password: str + ) -> ScicatClient: + scicat_client = from_credentials( + base_url=scicat_base_url, + username=scicat_user, + password=scicat_password + ) + return scicat_client + + @abstractmethod + def ingest_new_raw_dataset( + self, + file_path: str = "", + ) -> str: + """Ingest data from the beamline. + + :param file_path: Path to the file to ingest. + :return: SciCat ID of the dataset. + """ + pass + + @abstractmethod + def ingest_new_derived_dataset( + self, + file_path: str = "", + raw_dataset_id: str = "", + ) -> str: + """Ingest data from the beamline. + + :param file_path: Path to the file to ingest. + :return: SciCat ID of the dataset. + """ + pass + + @abstractmethod + def add_new_dataset_location( + self, + dataset_id: str = "", + destination: str = "", + ) -> bool: + """ + + """ + pass + + @abstractmethod + def remove_dataset_location( + self, + dataset_id: str = "", + source: str = "", + ) -> bool: + """ + + """ + pass From 4f0170f5bacb4a709e821d504504bdf9b01d13da Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 21 Feb 2025 17:13:20 -0800 Subject: [PATCH 016/128] sourceFolder expects a string --- orchestration/flows/bl832/scicat_ingestor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index eb0a8320..715344a9 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -77,7 +77,8 @@ def add_new_dataset_location( """ dataset = self.scicat_client.datasets_get_one(dataset_id) - dataset["locations"].append(location) + # sourceFolder can only be a single string... + dataset["sourceFolder"] = location self.scicat_client.update_dataset(dataset, dataset_id) logger.info(f"Added location {location} to dataset {dataset_id}") return From fa62fa72a39df051c2459c22dd01f2e2b02ec58e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 24 Feb 2025 12:54:51 -0800 Subject: [PATCH 017/128] generalizing relevant code from ingest_tomo832.py, refactoring into a bl832/scicat_ingestor.py, udpate documentation --- docs/mkdocs/docs/scicat.md | 4 +- orchestration/flows/bl832/move_refactor.py | 11 ++-- orchestration/flows/bl832/scicat_ingestor.py | 21 +------ .../flows/scicat/ingestor_controller.py | 28 +++++++-- orchestration/flows/scicat/utils.py | 57 +++++++++++++------ orchestration/transfer_endpoints.py | 20 +++++-- 6 files changed, 86 insertions(+), 55 deletions(-) diff --git a/docs/mkdocs/docs/scicat.md b/docs/mkdocs/docs/scicat.md index 75398afd..ef456d6c 100644 --- a/docs/mkdocs/docs/scicat.md +++ b/docs/mkdocs/docs/scicat.md @@ -35,9 +35,7 @@ flowchart TD In `splash_flows_globus`, we "ingest" our datasets into SciCat during our file movement workflows. In the directory `orchestration/flows/scicat/` there are two general scripts: `ingest.py` and `utils.py`. Since the data from each beamline is different, we define specific ingest implementations, such as `orchestration/flows/bl832/ingest_tomo832.py`. -## SciCat Client API - -# API Documentation +# SciCat Client API Documentation This document details the API provided by the `ScicatClient` class and its associated utility functions for interacting with the SciCat Catamel server. diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index dbe22ea5..cdfbbc35 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -7,12 +7,14 @@ from prefect import flow, task from prefect.blocks.system import JSON -from orchestration.flows.scicat.ingest import ingest_dataset +# from orchestration.flows.scicat.ingest import ingest_dataset +from orchestration.flows.bl832.scicat_ingestor import TomographyIngestorController from orchestration.flows.bl832.config import Config832 from orchestration.globus.transfer import start_transfer from orchestration.prune_controller import get_prune_controller, PruneMethod from orchestration.transfer_controller import get_transfer_controller, CopyMethod + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -67,12 +69,11 @@ def process_new_832_file( ) if nersc_transfer_success: - logger.info( - f"File successfully transferred from data832 to NERSC {file_path}. Task {task}" - ) + logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") try: - ingest_dataset(file_path, TOMO_INGESTOR_MODULE) + ingestor = TomographyIngestorController(config, config.scicat_client) + ingestor.ingest_new_raw_dataset(file_path) except Exception as e: logger.error(f"SciCat ingest failed with {e}") diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 715344a9..07248c84 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -53,7 +53,7 @@ def ingest_new_raw_dataset( def ingest_new_derived_dataset( self, - file_path: str = "", + folder_path: str = "", raw_dataset_id: str = "", ) -> str: """ @@ -63,22 +63,3 @@ def ingest_new_derived_dataset( :return: SciCat ID of the dataset. """ pass - - def add_new_dataset_location( - self, - dataset_id: str, - location: str, - ) -> None: - """ - Add a new location to an existing dataset in SciCat. - - :param dataset_id: SciCat ID of the dataset. - :param location: Path to the location to add. - """ - - dataset = self.scicat_client.datasets_get_one(dataset_id) - # sourceFolder can only be a single string... - dataset["sourceFolder"] = location - self.scicat_client.update_dataset(dataset, dataset_id) - logger.info(f"Added location {location} to dataset {dataset_id}") - return diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 64eb92a1..22a76fcf 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod +from logging import getLogger from orchestration.config import BeamlineConfig from pyscicat.client import ScicatClient, from_credentials +logger = getLogger(__name__) + + class BeamlineIngestorController(ABC): """ Abstract class for beamline ingestors. @@ -56,18 +60,32 @@ def ingest_new_derived_dataset( """ pass - @abstractmethod def add_new_dataset_location( self, - dataset_id: str = "", - destination: str = "", + dataset_id: str, + source_folder: str, + source_folder_host: str, ) -> bool: """ + Add a new location to an existing dataset in SciCat. + + :param dataset_id: SciCat ID of the dataset. + :param source_folder: "Absolute file path on file server containing the files of this dataset, + e.g. /some/path/to/sourcefolder. In case of a single file dataset, e.g. HDF5 data, + it contains the path up to, but excluding the filename. Trailing slashes are removed.", + + :param source_folder_host: "DNS host name of file server hosting sourceFolder, + optionally including a protocol e.g. [protocol://]fileserver1.example.com", """ - pass + dataset = self.scicat_client.datasets_get_one(dataset_id) + # sourceFolder sourceFolderHost are each a string + dataset["sourceFolder"] = source_folder + dataset["sourceFolderHost"] = source_folder_host + self.scicat_client.datasets_update(dataset, dataset_id) + logger.info(f"Added location {source_folder} to dataset {dataset_id}") + return dataset_id - @abstractmethod def remove_dataset_location( self, dataset_id: str = "", diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 6884f2d4..1a04bb17 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -1,6 +1,8 @@ import base64 from dataclasses import dataclass +from datetime import datetime +from pathlib import Path from enum import Enum import io import json @@ -39,6 +41,27 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) +def build_search_terms(sample_name): + """extract search terms from sample name to provide something pleasing to search on""" + terms = re.split("[^a-zA-Z0-9]", sample_name) + description = [term.lower() for term in terms if len(term) > 0] + return " ".join(description) + + +def build_thumbnail(image_array: npt.ArrayLike): + image_array = image_array - np.min(image_array) + 1.001 + image_array = np.log(image_array) + image_array = 205 * image_array / (np.max(image_array)) + auto_contrast_image = Image.fromarray(image_array.astype("uint8")) + auto_contrast_image = ImageOps.autocontrast(auto_contrast_image, cutoff=0.1) + # filename = str(uuid4()) + ".png" + file = io.BytesIO() + # file = thumbnail_dir / Path(filename) + auto_contrast_image.save(file, format="png") + file.seek(0) + return file + + def calculate_access_controls(username, beamline, proposal) -> Dict: # make an access group list that includes the name of the proposal and the name of the beamline access_groups = [] @@ -62,11 +85,15 @@ def calculate_access_controls(username, beamline, proposal) -> Dict: return {"owner_group": owner_group, "access_groups": access_groups} -def build_search_terms(sample_name): - """extract search terms from sample name to provide something pleasing to search on""" - terms = re.split("[^a-zA-Z0-9]", sample_name) - description = [term.lower() for term in terms if len(term) > 0] - return " ".join(description) +def clean_email(email: str): + if email: + if not email or email.upper() == "NONE": + # this is a brutal case, but the beamline sometimes puts in "None" and + # the new scicat backend hates that. + unknown_email = "unknown@example.com" + return unknown_email + return email.replace(" ", "").replace(",", "").replace("'", "") + return None def encode_image_2_thumbnail(filebuffer, imType="jpg"): @@ -77,15 +104,11 @@ def encode_image_2_thumbnail(filebuffer, imType="jpg"): return header + dataStr -def build_thumbnail(image_array: npt.ArrayLike): - image_array = image_array - np.min(image_array) + 1.001 - image_array = np.log(image_array) - image_array = 205 * image_array / (np.max(image_array)) - auto_contrast_image = Image.fromarray(image_array.astype("uint8")) - auto_contrast_image = ImageOps.autocontrast(auto_contrast_image, cutoff=0.1) - # filename = str(uuid4()) + ".png" - file = io.BytesIO() - # file = thumbnail_dir / Path(filename) - auto_contrast_image.save(file, format="png") - file.seek(0) - return file +def get_file_size(file_path: Path) -> int: + """Return the size of the file in bytes.""" + return file_path.lstat().st_size + + +def get_file_mod_time(file_path: Path) -> str: + """Return the file modification time in ISO format.""" + return datetime.fromtimestamp(file_path.lstat().st_mtime).isoformat() diff --git a/orchestration/transfer_endpoints.py b/orchestration/transfer_endpoints.py index 8fbae8fa..23e5ce46 100644 --- a/orchestration/transfer_endpoints.py +++ b/orchestration/transfer_endpoints.py @@ -8,10 +8,12 @@ class TransferEndpoint(ABC): def __init__( self, name: str, - root_path: str + root_path: str, + uri: str ) -> None: self.name = name self.root_path = root_path + self.uri = uri def name(self) -> str: """ @@ -25,6 +27,12 @@ def root_path(self) -> str: """ return self.root_path + def uri(self) -> str: + """ + Root path or base directory for this endpoint. + """ + return self.uri + class FileSystemEndpoint(TransferEndpoint): """ @@ -36,9 +44,10 @@ class FileSystemEndpoint(TransferEndpoint): def __init__( self, name: str, - root_path: str + root_path: str, + uri: str ) -> None: - super().__init__(name, root_path) + super().__init__(name, root_path, uri) def full_path( self, @@ -68,9 +77,10 @@ class HPSSEndpoint(TransferEndpoint): def __init__( self, name: str, - root_path: str + root_path: str, + uri: str ) -> None: - super().__init__(name, root_path) + super().__init__(name, root_path, uri) def full_path(self, path_suffix: str) -> str: """ From 40ae3df15fa7907f5e5b3c58691d742433c96208 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 24 Feb 2025 16:28:57 -0800 Subject: [PATCH 018/128] Refactoring scicat ingestion to be more modular and structured --- orchestration/flows/bl832/ingest_tomo832.py | 2 +- orchestration/flows/bl832/scicat_ingestor.py | 417 +++++++++++++++++- .../flows/scicat/ingestor_controller.py | 2 +- orchestration/flows/scicat/utils.py | 50 +-- 4 files changed, 423 insertions(+), 48 deletions(-) diff --git a/orchestration/flows/bl832/ingest_tomo832.py b/orchestration/flows/bl832/ingest_tomo832.py index e694daaf..cbd454e5 100644 --- a/orchestration/flows/bl832/ingest_tomo832.py +++ b/orchestration/flows/bl832/ingest_tomo832.py @@ -184,7 +184,7 @@ def upload_data_block( source_root_path: str ) -> Datablock: "Creates a datablock of files" - # calcularte the path where the file will as known to SciCat + # calculate the path where the file will as known to SciCat storage_path = str(file_path).replace(source_root_path, storage_root_path) datafiles = create_data_files(file_path, storage_path) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 07248c84..efe04dd4 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -1,12 +1,35 @@ -import importlib +# import importlib +import json from logging import getLogger -from typing import List +import os +from pathlib import Path +from typing import Any, Dict, List # Optional, Union +import h5py from pyscicat.client import ScicatClient +from pyscicat.model import ( + Attachment, + CreateDatasetOrigDatablockDto, + Datablock, + DataFile, + RawDataset, + DatasetType, + Ownable, +) from orchestration.flows.bl832.config import Config832 from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController -from orchestration.flows.scicat.utils import Issue +from orchestration.flows.scicat.utils import ( + build_search_terms, + build_thumbnail, + clean_email, + encode_image_2_thumbnail, + get_file_size, + get_file_mod_time, + Issue, + NPArrayEncoder, + Severity +) logger = getLogger(__name__) @@ -16,6 +39,82 @@ class TomographyIngestorController(BeamlineIngestorController): """ Ingestor for Tomo832 beamline. """ + DEFAULT_USER = "8.3.2" # In case there's not proposal number + INGEST_SPEC = "als832_dx_3" + + DATA_SAMPLE_KEYS = [ + "/measurement/instrument/sample_motor_stack/setup/axis1pos", + "/measurement/instrument/sample_motor_stack/setup/axis2pos", + "/measurement/instrument/sample_motor_stack/setup/sample_x", + "/measurement/instrument/sample_motor_stack/setup/axis5pos", + "/measurement/instrument/camera_motor_stack/setup/camera_elevation", + "/measurement/instrument/source/current", + "/measurement/instrument/camera_motor_stack/setup/camera_distance", + "/measurement/instrument/source/beam_intensity_incident", + "/measurement/instrument/monochromator/energy", + "/measurement/instrument/detector/exposure_time", + "/measurement/instrument/time_stamp", + "/measurement/instrument/monochromator/setup/turret2", + "/measurement/instrument/monochromator/setup/turret1", + ] + + SCICAT_METADATA_KEYS = [ + "/measurement/instrument/instrument_name", + "/measurement/sample/experiment/beamline", + "/measurement/sample/experiment/experiment_lead", + "/measurement/sample/experiment/pi", + "/measurement/sample/experiment/proposal", + "/measurement/sample/experimenter/email", + "/measurement/sample/experimenter/name", + "/measurement/sample/file_name", + ] + + SCIENTIFIC_METADATA_KEYS = [ + "/measurement/instrument/attenuator/setup/filter_y", + "/measurement/instrument/camera_motor_stack/setup/tilt_motor", + "/measurement/instrument/detection_system/objective/camera_objective", + "/measurement/instrument/detection_system/scintillator/scintillator_type", + "/measurement/instrument/detector/binning_x", + "/measurement/instrument/detector/binning_y", + "/measurement/instrument/detector/dark_field_value", + "/measurement/instrument/detector/delay_time", + "/measurement/instrument/detector/dimension_x", + "/measurement/instrument/detector/dimension_y", + "/measurement/instrument/detector/model", + "/measurement/instrument/detector/pixel_size", + "/measurement/instrument/detector/temperature", + "/measurement/instrument/monochromator/setup/Z2", + "/measurement/instrument/monochromator/setup/temperature_tc2", + "/measurement/instrument/monochromator/setup/temperature_tc3", + "/measurement/instrument/slits/setup/hslits_A_Door", + "/measurement/instrument/slits/setup/hslits_A_Wall", + "/measurement/instrument/slits/setup/hslits_center", + "/measurement/instrument/slits/setup/hslits_size", + "/measurement/instrument/slits/setup/vslits_Lead_Flag", + "/measurement/instrument/source/source_name", + "/process/acquisition/dark_fields/dark_num_avg_of", + "/process/acquisition/dark_fields/num_dark_fields", + "/process/acquisition/flat_fields/i0_move_x", + "/process/acquisition/flat_fields/i0_move_y", + "/process/acquisition/flat_fields/i0cycle", + "/process/acquisition/flat_fields/num_flat_fields", + "/process/acquisition/flat_fields/usebrightexpose", + "/process/acquisition/mosaic/tile_xmovedist", + "/process/acquisition/mosaic/tile_xnumimg", + "/process/acquisition/mosaic/tile_xorig", + "/process/acquisition/mosaic/tile_xoverlap", + "/process/acquisition/mosaic/tile_ymovedist", + "/process/acquisition/mosaic/tile_ynumimg", + "/process/acquisition/mosaic/tile_yorig", + "/process/acquisition/mosaic/tile_yoverlap", + "/process/acquisition/name", + "/process/acquisition/rotation/blur_limit", + "/process/acquisition/rotation/blur_limit", + "/process/acquisition/rotation/multiRev", + "/process/acquisition/rotation/nhalfCir", + "/process/acquisition/rotation/num_angles", + "/process/acquisition/rotation/range", + ] def __init__( self, @@ -29,27 +128,90 @@ def ingest_new_raw_dataset( file_path: str = "", ) -> str: """ - Ingest a new raw dataset from the Tomo832 beamline. + Ingest a new raw tomography dataset from the 8.3.2 beamline. + + This method integrates the full ingestion process: + - Reading and parsing the HDF5 file. + - Extracting SciCat and scientific metadata. + - Calculating access controls. + - Creating and uploading the RawDataset and datablock. + - Generating and uploading a thumbnail attachment. :param file_path: Path to the file to ingest. - :return: SciCat ID of the dataset. + :return: SciCat dataset ID. + :raises ValueError: If required environment variables are missing. + :raises Exception: If any issues are encountered during ingestion. """ - - # Ingest the dataset - ingestor_module = "orchestration.flows.bl832.ingest_tomo832" - ingestor_module = importlib.import_module(ingestor_module) issues: List[Issue] = [] - new_dataset_id = ingestor_module.ingest( - self.scicat_client, - file_path, - issues, - ) - if len(issues) > 0: - logger.error(f"SciCat ingest failed with {len(issues)} issues") + logger.setLevel("INFO") + + # Retrieve required environment variables for storage paths + INGEST_STORAGE_ROOT_PATH = os.getenv("INGEST_STORAGE_ROOT_PATH") + INGEST_SOURCE_ROOT_PATH = os.getenv("INGEST_SOURCE_ROOT_PATH") + if not INGEST_STORAGE_ROOT_PATH or not INGEST_SOURCE_ROOT_PATH: + raise ValueError( + "INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set" + ) + + file_path_obj = Path(file_path) + with h5py.File(file_path, "r") as file: + # Extract metadata from the HDF5 file using beamline-specific keys + scicat_metadata = self._extract_fields(file, self.SCICAT_METADATA_KEYS, issues) + scientific_metadata = self._extract_fields(file, self.SCIENTIFIC_METADATA_KEYS, issues) + scientific_metadata["data_sample"] = self._get_data_sample(file) + + # Encode scientific metadata using NPArrayEncoder + encoded_scientific_metadata = json.loads( + json.dumps(scientific_metadata, cls=NPArrayEncoder) + ) + + # Calculate access controls + access_controls = self._calculate_access_controls( + self.DEFAULT_USER, + scicat_metadata.get("/measurement/sample/experiment/beamline"), + scicat_metadata.get("/measurement/sample/experiment/proposal"), + ) + logger.info( + f"Access controls for {file_path_obj} - access_groups: {access_controls.get('access_groups')} " + f"owner_group: {access_controls.get('owner_group')}" + ) + + ownable = Ownable( + ownerGroup=access_controls["owner_group"], + accessGroups=access_controls["access_groups"], + ) + + # Create and upload the raw dataset + dataset_id = self._upload_raw_dataset( + file_path_obj, + scicat_metadata, + encoded_scientific_metadata, + ownable, + ) + + # Upload the data block (associated files) + self._upload_data_block( + file_path_obj, + dataset_id, + INGEST_STORAGE_ROOT_PATH, + INGEST_SOURCE_ROOT_PATH, + ) + + # Generate and upload a thumbnail attachment + # The "/exchange/data" key is specific to the Tomo832 HDF5 file structure. + thumbnail_file = build_thumbnail(file["/exchange/data"][0]) + encoded_thumbnail = encode_image_2_thumbnail(thumbnail_file) + self._upload_attachment( + encoded_thumbnail, + dataset_id, + ownable, + ) + + if issues: for issue in issues: logger.error(issue) - raise Exception("SciCat ingest failed") - return new_dataset_id + raise Exception(f"SciCat ingest failed with {len(issues)} issues") + return dataset_id def ingest_new_derived_dataset( self, @@ -63,3 +225,222 @@ def ingest_new_derived_dataset( :return: SciCat ID of the dataset. """ pass + + def _calculate_access_controls( + username, + beamline, + proposal + ) -> Dict: + """Calculate access controls for a dataset.""" + + # make an access group list that includes the name of the proposal and the name of the beamline + access_groups = [] + # sometimes the beamline name is super dirty " '8.3.2', "" '8.3.2', " + beamline = beamline.replace(" '", "").replace("', ", "") if beamline else None + # set owner_group to username so that at least someone has access in case no proposal number is found + owner_group = username + if beamline: + access_groups.append(beamline) + # username lets the user see the Dataset in order to ingest objects after the Dataset + access_groups.append(username) + # temporary mapping while beamline controls process request to match beamline name with what comes + # from ALSHub + if beamline == "bl832" and "8.3.2" not in access_groups: + access_groups.append("8.3.2") + + if proposal and proposal != "None": + owner_group = proposal + + # this is a bit of a kludge. Add 8.3.2 into the access groups so that staff will be able to see it + return {"owner_group": owner_group, "access_groups": access_groups} + + def _create_data_files( + self, + file_path: Path, + storage_path: str + ) -> List[DataFile]: + "Collects all fits files" + datafiles = [] + datafile = DataFile( + path=storage_path, + size=get_file_size(file_path), + time=get_file_mod_time(file_path), + type="RawDatasets", + ) + datafiles.append(datafile) + return datafiles + + def _extract_fields( + self, + file, + keys, + issues + ) -> Dict[str, Any]: + metadata = {} + for md_key in keys: + dataset = file.get(md_key) + if not dataset: + issues.append( + Issue(msg=f"dataset not found {md_key}", severity=Severity.warning) + ) + continue + metadata[md_key] = self._get_dataset_value(file[md_key]) + return metadata + + def _get_dataset_value( + self, + data_set + ): + """ + Extracts the value of a dataset from an HDF5 file. + """ + logger.debug(f"{data_set} {data_set.dtype}") + try: + if "S" in data_set.dtype.str: + if data_set.shape == (1,): + return data_set.asstr()[0] + elif data_set.shape == (): + return data_set[()].decode("utf-8") + else: + return list(data_set.asstr()) + else: + if data_set.maxshape == (1,): + logger.debug(f"{data_set} {data_set[()][0]}") + return data_set[()][0] + else: + logger.debug(f"{data_set} {data_set[()]}") + return data_set[()] + except Exception: + logger.exception("Exception extracting dataset value") + return None + + def _get_data_sample( + self, + file, + sample_size=10 + ): + """ Extracts a sample of the data from the HDF5 file. """ + data_sample = {} + for key in self.DATA_SAMPLE_KEYS: + data_array = file.get(key) + if not data_array: + continue + step_size = int(len(data_array) / sample_size) + if step_size == 0: + step_size = 1 + sample = data_array[0::step_size] + data_sample[key] = sample + + return data_sample + + def _upload_data_block( + self, + file_path: Path, + dataset_id: str, + storage_root_path: str, + source_root_path: str + ) -> Datablock: + "Creates a datablock of files" + # calculate the path where the file will as known to SciCat + storage_path = str(file_path).replace(source_root_path, storage_root_path) + datafiles = self._create_data_files(file_path, storage_path) + + datablock = CreateDatasetOrigDatablockDto( + size=get_file_size(file_path), + dataFileList=datafiles + ) + return self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) + + def _upload_attachment( + self, + encoded_thumnbnail: str, + dataset_id: str, + ownable: Ownable, + ) -> Attachment: + "Creates a thumbnail png" + attachment = Attachment( + datasetId=dataset_id, + thumbnail=encoded_thumnbnail, + caption="raw image", + **ownable.dict(), + ) + self.scicat_client.upload_attachment(attachment) + + def _upload_raw_dataset( + self, + file_path: Path, + scicat_metadata: Dict, + scientific_metadata: Dict, + ownable: Ownable, + ) -> str: + """ + Create and upload a new raw dataset to SciCat. + + :param file_path: Path to the file to ingest. + :param scicat_metadata: SciCat metadata. + :param scientific_metadata: Scientific metadata. + :param ownable: Ownable object. + :return: SciCat ID of the dataset + """ + file_size = get_file_size(file_path) + file_mod_time = get_file_mod_time(file_path) + file_name = scicat_metadata.get("/measurement/sample/file_name") + description = build_search_terms(file_name) + appended_keywords = description.split() + + dataset = RawDataset( + owner=scicat_metadata.get("/measurement/sample/experiment/pi") or "Unknown", + contactEmail=clean_email(scicat_metadata.get("/measurement/sample/experimenter/email")) + or "Unknown", + creationLocation=scicat_metadata.get("/measurement/instrument/instrument_name") + or "Unknown", + datasetName=file_name, + type=DatasetType.raw, + instrumentId=scicat_metadata.get("/measurement/instrument/instrument_name") + or "Unknown", + proposalId=scicat_metadata.get("/measurement/sample/experiment/proposal"), + dataFormat="DX", + principalInvestigator=scicat_metadata.get("/measurement/sample/experiment/pi") + or "Unknown", + sourceFolder=str(file_path.parent), + size=file_size, + scientificMetadata=scientific_metadata, + sampleId=description, + isPublished=False, + description=description, + keywords=appended_keywords, + creationTime=file_mod_time, + **ownable.dict(), + ) + logger.debug(f"dataset: {dataset}") + dataset_id = self.scicat_client.upload_new_dataset(dataset) + return dataset_id + + # def ingest_new_raw_dataset( + # self, + # file_path: str = "", + # ) -> str: + # """ + # Ingest a new raw dataset from the Tomo832 beamline. + + # :param file_path: Path to the file to ingest. + # :return: SciCat ID of the dataset. + # """ + + # # Ingest the dataset + # ingestor_module = "orchestration.flows.bl832.ingest_tomo832" + # ingestor_module = importlib.import_module(ingestor_module) + # issues: List[Issue] = [] + # new_dataset_id = ingestor_module.ingest( + # self.scicat_client, + # file_path, + # issues, + # ) + # if len(issues) > 0: + # logger.error(f"SciCat ingest failed with {len(issues)} issues") + # for issue in issues: + # logger.error(issue) + # raise Exception("SciCat ingest failed") + # return new_dataset_id + + # Generalizable from ingest_tomo832.py: diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 22a76fcf..f5c25896 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -10,7 +10,7 @@ class BeamlineIngestorController(ABC): """ - Abstract class for beamline ingestors. + Abstract class for beamline SciCat ingestors. Provides interface methods for ingesting data. """ diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 1a04bb17..43096b38 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -8,7 +8,7 @@ import json import logging import re -from typing import Dict, Optional, Union +from typing import Optional, Union import numpy as np import numpy.typing as npt @@ -18,19 +18,25 @@ can_debug = logger.isEnabledFor(logging.DEBUG) -class Severity(str, Enum): +class Severity( + str, + Enum +): + """Enum for issue severity.""" warning = "warning" error = "error" @dataclass class Issue: + """Dataclass for issues.""" severity: Severity msg: str exception: Optional[Union[str, None]] = None class NPArrayEncoder(json.JSONEncoder): + """Custom JSON encoder for numpy types.""" def default(self, obj): if isinstance(obj, np.integer): return int(obj) @@ -41,14 +47,19 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) -def build_search_terms(sample_name): +def build_search_terms( + sample_name: str +) -> str: """extract search terms from sample name to provide something pleasing to search on""" terms = re.split("[^a-zA-Z0-9]", sample_name) description = [term.lower() for term in terms if len(term) > 0] return " ".join(description) -def build_thumbnail(image_array: npt.ArrayLike): +def build_thumbnail( + image_array: npt.ArrayLike +) -> io.BytesIO: + """Create a thumbnail from an image array.""" image_array = image_array - np.min(image_array) + 1.001 image_array = np.log(image_array) image_array = 205 * image_array / (np.max(image_array)) @@ -62,30 +73,9 @@ def build_thumbnail(image_array: npt.ArrayLike): return file -def calculate_access_controls(username, beamline, proposal) -> Dict: - # make an access group list that includes the name of the proposal and the name of the beamline - access_groups = [] - # sometimes the beamline name is super dirty " '8.3.2', "" '8.3.2', " - beamline = beamline.replace(" '", "").replace("', ", "") if beamline else None - # set owner_group to username so that at least someone has access in case no proposal number is found - owner_group = username - if beamline: - access_groups.append(beamline) - # username lets the user see the Dataset in order to ingest objects after the Dataset - access_groups.append(username) - # temporary mapping while beamline controls process request to match beamline name with what comes - # from ALSHub - if beamline == "bl832" and "8.3.2" not in access_groups: - access_groups.append("8.3.2") - - if proposal and proposal != "None": - owner_group = proposal - - # this is a bit of a kludge. Add 8.3.2 into the access groups so that staff will be able to see it - return {"owner_group": owner_group, "access_groups": access_groups} - - def clean_email(email: str): + """Clean up email addresses.""" + if email: if not email or email.upper() == "NONE": # this is a brutal case, but the beamline sometimes puts in "None" and @@ -96,7 +86,11 @@ def clean_email(email: str): return None -def encode_image_2_thumbnail(filebuffer, imType="jpg"): +def encode_image_2_thumbnail( + filebuffer, + imType="jpg" +) -> str: + """Encode an image file to a base 64 string for use as a thumbnail.""" logging.info("Creating thumbnail for dataset") header = "data:image/{imType};base64,".format(imType=imType) dataBytes = base64.b64encode(filebuffer.read()) From 69414386a30c685c34f279b49a98f540b968eda4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 24 Feb 2025 16:30:00 -0800 Subject: [PATCH 019/128] Adding a custom Prefect Block to keep track of pending project names to be transferred to HPSS Tape. --- orchestration/flows/bl832/dispatcher.py | 47 +++++++++++++++++ orchestration/{flows/bl832 => }/hpss.py | 68 ++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) rename orchestration/{flows/bl832 => }/hpss.py (52%) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index acb8c6bd..b0850ac3 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -1,4 +1,5 @@ import asyncio +from pathlib import Path from prefect import flow, task, get_run_logger from prefect.blocks.system import JSON from prefect.deployments.deployments import run_deployment @@ -140,6 +141,18 @@ async def dispatcher( # Optionally, raise a specific ValueError raise ValueError("new_file_832 task Failed") from e + # TODO: Track the project name in a list of what needs to move + # to HPSS from NERSC CFS, for a scheduled run every 6 months. + # Maybe in a Prefect JSON Block? + try: + # Add project to tape archive queue, if file_path is provided + project_path = str(Path(file_path).parent) + archive_queue = TapeArchiveQueue("TAPE_ARCHIVE_QUEUE") + archive_queue.enqueue_project(project_path) + logger.info(f"Project '{project_path}' added to tape archive queue.") + except Exception as e: + logger.error(f"Failed to add project to tape archive queue: {e}") + # Prepare ALCF and NERSC flows to run asynchronously, based on settings tasks = [] if decision_settings.value.get("alcf_recon_flow/alcf_recon_flow"): @@ -163,6 +176,40 @@ async def dispatcher( return None +# --------------------------------------------------------------------------- +# Tape Transfer Flow: Process pending projects +# --------------------------------------------------------------------------- +# Runs every 6 months to process tape transfers for pending projects. +# --------------------------------------------------------------------------- +@flow(name="tape_transfer_dispatcher") +async def tape_transfer_dispatcher(config) -> None: + """ + Scheduled flow to process tape transfers. + It should call the CFStoHPSSTransferController (not shown) and, upon success, mark projects as moved. + """ + logger = get_run_logger() + try: + block = TapeArchiveQueue.load("TAPE_ARCHIVE_QUEUE_BLOCK") + except Exception: + logger.info("No project status block found.") + return + + for project in block.pending_projects.copy(): + logger.info(f"Transferring project '{project}' to tape...") + + # Run the CFS to HPSS transfer flow for the project + # params = { + # "file_path": project, + # "source": nersc_cfs, + # "destination": hpss, + # "config": Config832() + # } + # run_specific_flow("cfs_to_hpss_flow/cfs_to_hpss_flow", {"file_path": project}) + + TapeArchiveQueue.mark_project_as_moved(project) + logger.info(f"Project '{project}' marked as moved.") + + if __name__ == "__main__": """ This script defines the flow for the decision making process of the BL832 beamline. diff --git a/orchestration/flows/bl832/hpss.py b/orchestration/hpss.py similarity index 52% rename from orchestration/flows/bl832/hpss.py rename to orchestration/hpss.py index 2142490f..6c404371 100644 --- a/orchestration/flows/bl832/hpss.py +++ b/orchestration/hpss.py @@ -5,16 +5,82 @@ from typing import List, Optional from prefect import flow +from prefect.blocks.core import Block +from pydantic import BaseModel, Field from orchestration.config import BeamlineConfig -from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint from orchestration.flows.bl832.config import Config832 from orchestration.transfer_controller import get_transfer_controller, CopyMethod +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +class TapeArchiveQueueBlock(Block, BaseModel): + """ + Custom Prefect block for tracking projects pending tape archiving and those already archived. + """ + pending_projects: List[str] = Field(default_factory=list) + moved_projects: List[str] = Field(default_factory=list) + + def add_pending(self, project: str) -> None: + """Add a project to the pending list if not already added or archived.""" + if project and project not in self.pending_projects and project not in self.moved_projects: + self.pending_projects.append(project) + + def mark_moved(self, project: str) -> None: + """Remove a project from pending and add it to the moved (archived) list.""" + if project in self.pending_projects: + self.pending_projects.remove(project) + if project and project not in self.moved_projects: + self.moved_projects.append(project) + + +class TapeArchiveQueue: + """ + Reusable controller for managing tape archive project queues. + + Attributes: + block_name (str): The name used to persist the tape archive queue block. + """ + def __init__(self, block_name: str = "TAPE_ARCHIVE_QUEUE_BLOCK"): + self.block_name = block_name + + def load_block(self) -> TapeArchiveQueueBlock: + """ + Load the persisted tape archive queue block, or create a new one if it doesn't exist. + """ + try: + block = TapeArchiveQueueBlock.load(self.block_name) + except Exception: + block = TapeArchiveQueueBlock() + return block + + def save_block(self, block: TapeArchiveQueueBlock) -> None: + """Persist the tape archive queue block.""" + block.save(self.block_name, overwrite=True) + + def enqueue_project(self, project: str) -> None: + """Add a project to the pending (to be archived) list.""" + block = self.load_block() + block.add_pending(project) + self.save_block(block) + logger.info(f"Enqueued project '{project}' for tape archiving.") + + def mark_project_as_moved(self, project: str) -> None: + """Mark a project as moved (archived) by removing it from pending and adding it to moved.""" + block = self.load_block() + block.mark_moved(project) + self.save_block(block) + logger.info(f"Project '{project}' marked as archived.") + + def get_pending_projects(self) -> List[str]: + """Retrieve the list of projects pending tape archiving.""" + block = self.load_block() + return block.pending_projects + + @flow(name="cfs_to_hpss_flow") def cfs_to_hpss_flow( file_path: str = None, From 49c5b794f8609c559d968a86eaf1630c75320b68 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 25 Feb 2025 13:59:44 -0800 Subject: [PATCH 020/128] Added three dispatchers for bl832 to handle archiving data to HPSS: 1) specify project(s) to copy, 2) schedule a flow every 6 months to copy the previous cycle's data, and 3) archive everything. Also fixing pytests to handle recent changes. --- create_deployment_832_dispatcher.sh | 12 +- orchestration/_tests/test_scicat.py | 2 + .../_tests/test_transfer_controller.py | 23 +- orchestration/flows/bl832/dispatcher.py | 203 +++++++++++++++--- orchestration/flows/scicat/utils.py | 31 ++- orchestration/hpss.py | 86 ++------ 6 files changed, 237 insertions(+), 120 deletions(-) diff --git a/create_deployment_832_dispatcher.sh b/create_deployment_832_dispatcher.sh index afaae432..6ab20cb5 100755 --- a/create_deployment_832_dispatcher.sh +++ b/create_deployment_832_dispatcher.sh @@ -3,4 +3,14 @@ export $(grep -v '^#' .env | xargs) prefect work-pool create 'dispatcher_pool' prefect deployment build ./orchestration/flows/bl832/dispatcher.py:dispatcher -n run_832_dispatcher -q bl832 -p dispatcher_pool -prefect deployment apply dispatcher-deployment.yaml \ No newline at end of file +prefect deployment apply dispatcher-deployment.yaml + +prefect work-pool create 'hpss_pool' +prefect deployment build ./orchestration/flows/bl832/dispatcher.py:archive_832_project_dispatcher -n run_archive_832_project_dispatcher -q hpss_dispatcher_queue -p hpss_pool +prefect deployment apply archive_832_project_dispatcher-deployment.yaml + +prefect deployment build ./orchestration/flows/bl832/dispatcher.py:archive_832_projects_from_previous_cycle_dispatcher -n run_archive_832_projects_from_previous_cycle_dispatcher -q hpss_dispatcher_queue -p hpss_pool +prefect deployment apply archive_832_projects_from_previous_cycle_dispatcher-deployment.yaml + +prefect deployment build ./orchestration/flows/bl832/dispatcher.py:archive_all_832_raw_projects_dispatcher -n run_archive_all_832_raw_projects_dispatcher -q hpss_dispatcher_queue -p hpss_pool +prefect deployment apply archive_all_832_raw_projects_dispatcher-deployment.yaml \ No newline at end of file diff --git a/orchestration/_tests/test_scicat.py b/orchestration/_tests/test_scicat.py index 6cc68cdb..b204fa85 100644 --- a/orchestration/_tests/test_scicat.py +++ b/orchestration/_tests/test_scicat.py @@ -60,6 +60,8 @@ def test_np_encoder(): test_dict = {"dont_panic": np.full((1, 1), np.inf)} encoded_np = json.loads(json.dumps(test_dict, cls=NPArrayEncoder)) + # requests doesn't allow strings that have np.inf or np.nan + # so the NPArrayEncoder needs to return both as None assert json.dumps(encoded_np, allow_nan=False) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index 5f356442..f47be9be 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -118,7 +118,8 @@ def mock_file_system_endpoint(transfer_controller_module): FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] endpoint = FileSystemEndpoint( name="mock_filesystem_endpoint", - root_path="/mock_fs_root" + root_path="/mock_fs_root", + uri="mock_uri" ) return endpoint @@ -380,8 +381,8 @@ def test_cfs_to_hpss_transfer_controller_success(mock_config832, transfer_contro FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] # Create mock endpoints for source (CFS) and destination (HPSS) - source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source") - destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest") + source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source", "mock.uri") + destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest", "mock.uri") # Create a fake job object that simulates successful completion. fake_job = MagicMock() @@ -416,8 +417,8 @@ def test_cfs_to_hpss_transfer_controller_failure(mock_config832, transfer_contro HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] - source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source") - destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest") + source_endpoint = FileSystemEndpoint("mock_cfs_source", "/mock_cfs_source", "mock.uri") + destination_endpoint = HPSSEndpoint("mock_hpss_dest", "/mock_hpss_dest", "mock.uri") # Create a fake client whose compute().submit_job raises an exception. fake_client = MagicMock() @@ -448,8 +449,8 @@ def test_hpss_to_cfs_transfer_controller_success(mock_config832, transfer_contro HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] - source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") - destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source", "mock.uri") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest", "mock.uri") # Create a fake job object for a successful transfer. fake_job = MagicMock() @@ -495,8 +496,8 @@ def test_hpss_to_cfs_transfer_controller_job_failure(mock_config832, transfer_co HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] - source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") - destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source", "mock.uri") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest", "mock.uri") fake_job = MagicMock() fake_job.jobid = "67891" @@ -529,8 +530,8 @@ def test_hpss_to_cfs_transfer_controller_recovery(mock_config832, transfer_contr HPSSEndpoint = transfer_controller_module["HPSSEndpoint"] FileSystemEndpoint = transfer_controller_module["FileSystemEndpoint"] - source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source") - destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest") + source_endpoint = HPSSEndpoint("mock_hpss_source", "/mock_hpss_source", "mock.uri") + destination_endpoint = FileSystemEndpoint("mock_cfs_dest", "/mock_cfs_dest", "mock.uri") # Fake job that fails initially with a "Job not found:" error. fake_job_initial = MagicMock() diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index b0850ac3..afdb8c8e 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -1,14 +1,26 @@ import asyncio -from pathlib import Path +from datetime import datetime +from dateutil.parser import isoparse + from prefect import flow, task, get_run_logger from prefect.blocks.system import JSON from prefect.deployments.deployments import run_deployment from pydantic import BaseModel, ValidationError, Field -from typing import Any, Optional, Union +from typing import Any, List, Optional, Union from orchestration.flows.bl832.move import process_new_832_file_task +from orchestration.flows.bl832.config import Config832 +# ------------------------------------------------------------------------------------------------------------------------ +# Decision Flow: Dispatcher +# ------------------------------------------------------------------------------------------------------------------------ +# This flow reads decision settings and launches tasks accordingly. +# ------------------------------------------------------------------------------------------------------------------------ +# The dispatcher flow reads decision settings and launches tasks accordingly. +# It first runs the new_832_file_flow/new_file_832 flow synchronously. +# Then, it prepares the ALCF and NERSC flows to run asynchronously based on the decision settings. +# ------------------------------------------------------------------------------------------------------------------------ class FlowParameterMapper: """ Class to define and map the parameters required for each flow. @@ -106,7 +118,7 @@ async def run_recon_flow_async(flow_name: str, parameters: dict) -> None: async def dispatcher( file_path: Optional[str] = None, is_export_control: bool = False, - config: Optional[Union[dict, Any]] = None, # TODO: Define the type of config to be BeamlineConfig + config: Optional[Union[dict, Any]] = None ) -> None: """ Dispatcher flow that reads decision settings and launches tasks accordingly. @@ -141,18 +153,6 @@ async def dispatcher( # Optionally, raise a specific ValueError raise ValueError("new_file_832 task Failed") from e - # TODO: Track the project name in a list of what needs to move - # to HPSS from NERSC CFS, for a scheduled run every 6 months. - # Maybe in a Prefect JSON Block? - try: - # Add project to tape archive queue, if file_path is provided - project_path = str(Path(file_path).parent) - archive_queue = TapeArchiveQueue("TAPE_ARCHIVE_QUEUE") - archive_queue.enqueue_project(project_path) - logger.info(f"Project '{project_path}' added to tape archive queue.") - except Exception as e: - logger.error(f"Failed to add project to tape archive queue: {e}") - # Prepare ALCF and NERSC flows to run asynchronously, based on settings tasks = [] if decision_settings.value.get("alcf_recon_flow/alcf_recon_flow"): @@ -176,38 +176,171 @@ async def dispatcher( return None +# --------------------------------------------------------------------------- +# Tape Transfer Flow: Archive a single 832 project (raw) +# --------------------------------------------------------------------------- +@flow(name="archive_832_project_dispatcher") +def archive_832_project_dispatcher( + config: Config832, + file_path: Union[str, List[str]] = None, +) -> None: + """ + Flow to archive one or more beamline 832 projects to tape. + Accepts a single file path (str) or a list of file paths, and for each one, + calls the CFStoHPSSTransferController via run_specific_flow. + + Parameters + ---------- + file_path : Union[str, List[str]] + A single file path or a list of file paths to be archived. + config : Config832 + Configuration object containing endpoint details. + """ + + # Normalize file_path into a list if it's a single string. + if isinstance(file_path, str): + file_paths = [file_path] + else: + file_paths = file_path + + for fp in file_paths: + try: + run_specific_flow( + "cfs_to_hpss_flow/cfs_to_hpss_flow", + { + "file_path": fp, + "source": config.nersc832, # NERSC FileSystem Endpoint + "destination": config.hpss_alsdev, # HPSS Endpoint + "config": config + } + ) + logger.info(f"Scheduled tape transfer for project: {fp}") + except Exception as e: + logger.error(f"Error scheduling transfer for {fp}: {e}") + + # --------------------------------------------------------------------------- # Tape Transfer Flow: Process pending projects # --------------------------------------------------------------------------- -# Runs every 6 months to process tape transfers for pending projects. +# Scheduled to run every 6 months to process tape transfers. # --------------------------------------------------------------------------- -@flow(name="tape_transfer_dispatcher") -async def tape_transfer_dispatcher(config) -> None: +@flow(name="archive_832_projects_from_previous_cycle_dispatcher") +def archive_832_projects_from_previous_cycle_dispatcher( + config: Config832, +) -> None: """ - Scheduled flow to process tape transfers. - It should call the CFStoHPSSTransferController (not shown) and, upon success, mark projects as moved. + Archives the previous cycle's projects from the NERSC / CFS / 8.3.2 / SCRATCH directory. + + The schedule is as follows: + - On January 2: Archive projects with modification dates between January 1 and July 15 (previous year) + - On July 4: Archive projects with modification dates between July 16 and December 31 (previous year) + + The flow lists projects via Globus Transfer's operation_ls, filters them based on modification times, + and then calls the cfs_to_hpss_flow for each eligible project. """ logger = get_run_logger() + now = datetime.now() + + # Validate that today is a scheduled trigger day and set the archive window accordingly. + if now.month == 1 and now.day == 2: + archive_start = datetime(now.year - 1, 1, 1, 0, 0, 0) + archive_end = datetime(now.year - 1, 7, 15, 23, 59, 59) + logger.info(f"Archiving Cycle 1 ({archive_start.strftime('%b %d, %Y %H:%M:%S')} - " + f"{archive_end.strftime('%b %d, %Y %H:%M:%S')})") + elif now.month == 7 and now.day == 4: + archive_start = datetime(now.year - 1, 7, 16, 0, 0, 0) + archive_end = datetime(now.year - 1, 12, 31, 23, 59, 59) + logger.info(f"Archiving Cycle 2 ({archive_start.strftime('%b %d, %Y %H:%M:%S')} - " + f"{archive_end.strftime('%b %d, %Y %H:%M:%S')})") + else: + logger.info("Today is not a scheduled day for archiving.") + return + + logger.info(f"Archive window: {archive_start} to {archive_end}") + + # List projects using Globus Transfer's operation_ls. try: - block = TapeArchiveQueue.load("TAPE_ARCHIVE_QUEUE_BLOCK") - except Exception: - logger.info("No project status block found.") + # config.tc: configured Globus Transfer client. + # config.nersc832.endpoint_id: the NERSC endpoint ID. + # config.nersc832_alsdev_scratch.path: the SCRATCH directory path. + projects = config.tc.operation_ls( + endpoint_id=config.nersc832.endpoint_id, + path=config.nersc832_alsdev_scratch.path, + orderby=["name", "last_modified"], + ).get("DATA", []) + except Exception as e: + logger.error(f"Failed to list projects: {e}") return - for project in block.pending_projects.copy(): - logger.info(f"Transferring project '{project}' to tape...") + logger.info(f"Found {len(projects)} items in the SCRATCH directory.") + + # Process each project: check its modification time and trigger transfer if within the archive window. + for project in projects: + project_name = project.get("name") + last_mod_str = project.get("last_modified") + if not project_name or not last_mod_str: + logger.warning(f"Skipping project due to missing name or last_modified: {project}") + continue - # Run the CFS to HPSS transfer flow for the project - # params = { - # "file_path": project, - # "source": nersc_cfs, - # "destination": hpss, - # "config": Config832() - # } - # run_specific_flow("cfs_to_hpss_flow/cfs_to_hpss_flow", {"file_path": project}) + try: + last_mod = isoparse(last_mod_str) + except Exception as e: + logger.error(f"Error parsing modification time for project {project_name}: {e}") + continue + + if archive_start <= last_mod <= archive_end: + logger.info(f"Project {project_name} last modified at {last_mod} is within the archive window.") + try: + # Call the transfer flow for this project. + run_specific_flow( + "cfs_to_hpss_flow/cfs_to_hpss_flow", + { + "project": project, + "source_endpoint": config.nersc832, + "destination_endpoint": config.hpss_alsdev, + "config": config + } + ) + except Exception as e: + logger.error(f"Error archiving project {project_name}: {e}") + else: + logger.info(f"Project {project_name} last modified at {last_mod} is outside the archive window.") - TapeArchiveQueue.mark_project_as_moved(project) - logger.info(f"Project '{project}' marked as moved.") + +# --------------------------------------------------------------------------- +# Tape Transfer Flow: Archive all 832 projects (raw) +# --------------------------------------------------------------------------- +@flow(name="archive_all_832_raw_projects_dispatcher") +def archive_all_832_projects_dispatcher( + config: Config832, +) -> None: + """ + Scheduled flow to process tape transfers. + It should call the CFStoHPSSTransferController (not shown) and, upon success, mark projects as moved. + """ + logger = get_run_logger() + + logger.info(f"Checking for projects at {config.nersc832_alsdev_scratch.path} to archive to tape...") + + # ARCHIVE ALL PROJECTS IN THE NERSC / CFS / 8.3.2 / SCRATCH DIRECTORY + for project in config.tc.operation_ls( + endpoint_id=config.nersc832.endpoint_id, + path=config.nersc832_alsdev_scratch.path, + orderby=["name", "last_modified"], + ): + logger.info(f"Found project: {project}") + try: + run_specific_flow( + "cfs_to_hpss_flow/cfs_to_hpss_flow", + { + "file_path": project, + "source": config.nersc832, # NERSC FileSystem Endpoint (not globus) + "destination": config.hpss_alsdev, # HPSS Endpoint + "config": config + } + ) + except Exception as e: + logger.error(e) if __name__ == "__main__": diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 43096b38..86ef283a 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -8,7 +8,7 @@ import json import logging import re -from typing import Optional, Union +from typing import Dict, Optional, Union import numpy as np import numpy.typing as npt @@ -73,6 +73,35 @@ def build_thumbnail( return file +def calculate_access_controls( + username, + beamline, + proposal +) -> Dict: + """Calculate access controls for a dataset.""" + + # make an access group list that includes the name of the proposal and the name of the beamline + access_groups = [] + # sometimes the beamline name is super dirty " '8.3.2', "" '8.3.2', " + beamline = beamline.replace(" '", "").replace("', ", "") if beamline else None + # set owner_group to username so that at least someone has access in case no proposal number is found + owner_group = username + if beamline: + access_groups.append(beamline) + # username lets the user see the Dataset in order to ingest objects after the Dataset + access_groups.append(username) + # temporary mapping while beamline controls process request to match beamline name with what comes + # from ALSHub + if beamline == "bl832" and "8.3.2" not in access_groups: + access_groups.append("8.3.2") + + if proposal and proposal != "None": + owner_group = proposal + + # this is a bit of a kludge. Add 8.3.2 into the access groups so that staff will be able to see it + return {"owner_group": owner_group, "access_groups": access_groups} + + def clean_email(email: str): """Clean up email addresses.""" diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 6c404371..de3c9213 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -2,11 +2,9 @@ This module contains the HPSS flow for BL832. """ import logging -from typing import List, Optional +from typing import List, Optional, Union from prefect import flow -from prefect.blocks.core import Block -from pydantic import BaseModel, Field from orchestration.config import BeamlineConfig from orchestration.flows.bl832.config import Config832 @@ -17,88 +15,31 @@ logger.setLevel(logging.INFO) -class TapeArchiveQueueBlock(Block, BaseModel): - """ - Custom Prefect block for tracking projects pending tape archiving and those already archived. - """ - pending_projects: List[str] = Field(default_factory=list) - moved_projects: List[str] = Field(default_factory=list) - - def add_pending(self, project: str) -> None: - """Add a project to the pending list if not already added or archived.""" - if project and project not in self.pending_projects and project not in self.moved_projects: - self.pending_projects.append(project) - - def mark_moved(self, project: str) -> None: - """Remove a project from pending and add it to the moved (archived) list.""" - if project in self.pending_projects: - self.pending_projects.remove(project) - if project and project not in self.moved_projects: - self.moved_projects.append(project) - - -class TapeArchiveQueue: - """ - Reusable controller for managing tape archive project queues. - - Attributes: - block_name (str): The name used to persist the tape archive queue block. - """ - def __init__(self, block_name: str = "TAPE_ARCHIVE_QUEUE_BLOCK"): - self.block_name = block_name - - def load_block(self) -> TapeArchiveQueueBlock: - """ - Load the persisted tape archive queue block, or create a new one if it doesn't exist. - """ - try: - block = TapeArchiveQueueBlock.load(self.block_name) - except Exception: - block = TapeArchiveQueueBlock() - return block - - def save_block(self, block: TapeArchiveQueueBlock) -> None: - """Persist the tape archive queue block.""" - block.save(self.block_name, overwrite=True) - - def enqueue_project(self, project: str) -> None: - """Add a project to the pending (to be archived) list.""" - block = self.load_block() - block.add_pending(project) - self.save_block(block) - logger.info(f"Enqueued project '{project}' for tape archiving.") - - def mark_project_as_moved(self, project: str) -> None: - """Mark a project as moved (archived) by removing it from pending and adding it to moved.""" - block = self.load_block() - block.mark_moved(project) - self.save_block(block) - logger.info(f"Project '{project}' marked as archived.") - - def get_pending_projects(self) -> List[str]: - """Retrieve the list of projects pending tape archiving.""" - block = self.load_block() - return block.pending_projects - - @flow(name="cfs_to_hpss_flow") def cfs_to_hpss_flow( - file_path: str = None, + file_path: Union[str, List[str]] = None, source: FileSystemEndpoint = None, destination: HPSSEndpoint = None, - config: BeamlineConfig = Config832() + config: BeamlineConfig = None ) -> bool: """ The CFS to HPSS flow for BL832. Parameters ---------- - file_path : str - The path of the file to transfer. + file_path : Union[str, List[str]] + A single file path or a list of file paths to transfer. source : FileSystemEndpoint The source endpoint. destination : HPSSEndpoints The destination endpoint. + config : BeamlineConfig + The beamline configuration. + + Returns + ------- + bool + True if all transfers succeeded, False otherwise. """ logger.info("Running cfs_to_hpss_flow") @@ -110,7 +51,8 @@ def cfs_to_hpss_flow( config=config ) - logger.info("CFSToHPSSTransferController selected. Initiating transfer.") + logger.info("CFSToHPSSTransferController selected. Initiating transfer for all file paths.") + result = transfer_controller.copy( file_path=file_path, source=source, From a7496b7abe83a2c4adc994c9e7a476f6f9ef0e65 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 13:47:04 -0800 Subject: [PATCH 021/128] Updating documentation and comments --- docs/mkdocs/docs/common_infrastructure.md | 4 +- docs/mkdocs/docs/hpss.md | 57 +++++++++++++++++--- orchestration/flows/bl832/move_refactor.py | 2 - orchestration/flows/bl832/scicat_ingestor.py | 29 ---------- orchestration/hpss.py | 15 ++++-- 5 files changed, 61 insertions(+), 46 deletions(-) diff --git a/docs/mkdocs/docs/common_infrastructure.md b/docs/mkdocs/docs/common_infrastructure.md index 6e6ca5f0..3f448d56 100644 --- a/docs/mkdocs/docs/common_infrastructure.md +++ b/docs/mkdocs/docs/common_infrastructure.md @@ -17,6 +17,8 @@ Shared code is organized into modules that can be imported in beamline specific - This module is responsible for managing the pruning of data off of storage systems. It uses a configurable retention policy to determine when to remove files. It contains an ABC called `PruneController()` that is extended by specific implementations for `FileSystemEndpoint`, `GlobusEndpoint`, and `HPSSEndpoint`. - **`orchestration/sfapi.py`**: Create an SFAPI Client to launch remote jobs at NERSC. - **`orchestration/flows/scicat/ingest.py`**: Ingests datasets into SciCat, our metadata management system. +- **`orchestration/hpss.py`**: Schedule a Prefect Flow to copy data between NERSC CFS and HPSS. These call the relevant TransferControllers for HPSS, which handle the underlying tape-safe logic. + ## Beamline Specific Implementation Patterns In order to balance generalizability, maintainability, and scalability of this project to multiple beamlines, we try to organize specific implementations in a similar way. We keep specific implementaqtions in the directory `orchestration/flows/bl{beamline_id}/`, which generally contains a few things: @@ -30,8 +32,6 @@ In order to balance generalizability, maintainability, and scalability of this p - For beamlines that trigger remote analysis workflows, the `JobController()` ABC allows us to define HPC or machine specific implementations, which may differ in how code can be deployed. For example, it can be extended to define how to run tomography reconstruction at ALCF and NERSC. - **`{hpc}.py`** - We separate HPC implementations for `JobController()` in their own files. -- **`hpss.py`** - - We define HPSS transfers for each beamline individually, as we provide different scheduling strategies based on the data throughput of each endstation. - **`ingest.py`** - This is where we define SciCat implementations for each beamline, as each technique will have specific metadata fields that are important to capture. diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index ab5ddeb2..0eb64571 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -2,11 +2,18 @@ HPSS is the tape-based data storage system we use for long term storage of experimental data at the ALS. Tape storage, while it may seem antiquated, is still a very economical and secure medium for infrequently accessed data as tape does not need to be powered except for reading and writing. This requires certain considerations when working with this system. +## Overview + +**Purpose:** Archive and retrieve large experimental datasets using HPSS. +**Approach:** Use HPSS tools (hsi and htar) within a structured transfer framework orchestrated via SFAPI and SLURM jobs. +**Key Considerations:** File sizes should typically be between 100 GB and 2 TB. Larger projects are segmented into multiple archives. + + In `orchestration/transfer_controller.py` we have included two transfer classes for moving data from CFS to HPSS and vice versa (HPSS to CFS). We are following the [HPSS best practices](https://docs.nersc.gov/filesystems/HPSS-best-practices/) outlined in the NERSC documentation. HPSS is intended for long-term storage of data that is not frequently accessed, and users should aim for file sizes between 100 GB and 2 TB. Since HPSS is a tape system, we need to ensure storage and retrieval commands are done efficiently, as it is a mechanical process to load in a tape and then scroll to the correct region on the tape. -While there are Globus endpoints for HPSS, the NERSC documentation recommends against it as there are certain conditions (i.e. network disconnection) that are not as robust as their recommended HPSS tools `hsi` and `htar`, which they say is the fastest approach. Together, these tools allow us to work with the HPSS filesystem and carefully bundle our projects into `tar` archives that are built directly on HPSS. +While there are Globus endpoints for HPSS, the NERSC documentation recommends against it as there are certain conditions (i.e. network disconnection) that are not as robust as their recommended HPSS tools `hsi` and `htar`, which they say is the fastest approach. Together, these tools allow us to work with the HPSS filesystem and carefully bundle our projects into `tar` archives that are built directly on HPSS. Another couple of drawbacks to using Globus here is 1) if you have small files, you need to tar them regardless before transferring, and 2) HPSS does not support collab accounts (i.e. alsdev). ## Working with `hsi` @@ -161,9 +168,14 @@ htar -xvf als_user_project_folder.tar cool_scan1.h5 Most of the time we expect transfers to occur from CFS to HPSS on a scheduled basis, after users have completed scanning during their alotted beamtime. ### Transfer to HPSS Implementation -**`orchestration/transfer_controller.py`: `CFSToHPSSTransferController()`** +**`orchestration/transfer_controller.py`:** + - **`CFSToHPSSTransferController()`**: This controller uses a Slurm Job Script and SFAPI to launch the tape transfer job. The Slurm script handles the specific logic for handling single and multiple files, on a project by project basis. It reads the files sizes, and creates bundles that are <= 2TB. The groups within each tar archive are saved in a log on NERSC CFS for posterity. + +**`orchestration/hpss.py`:** +- **`cfs_to_hpss_flow()`** This Prefect Flow sets up the CFSToHPSSTransferController() and calls the copy command. By registering this Flow, the HPSS transfers can be easily scheduled. +**HPSS SFAPI/Slurm Job Logic**: ```mermaid flowchart TD @@ -223,13 +235,42 @@ flowchart TD ``` ### Transfer to CFS Implementation -**`orchestration/transfer_controller.py`: `HPSSToCFSTransferController()`** -Input -Output +**`orchestration/transfer_controller.py`:** + - **`CFSToHPSSTransferController()`**: This controller uses a Slurm Job Script and SFAPI to copy data from tape to NERSC CFS. The Slurm script handles the specific logic for handling single and multiple files, on a project by project basis. Based on the file path, the Slurm job determines whether a single file or a tar archive has been requested (or even specific files within a tar archive), and run the correct routine to copy the data to CFS. + +**`orchestration/hpss.py`:** +- **`cfs_to_hpss_flow()`** This Prefect Flow sets up the HPSSToCFSTransferController() and calls the copy command. By registering this Flow, the HPSS transfers to CFS can be easily scheduled. While copying from CFS to HPSS is likely not going to be automated, it is still helpful to have this as a Prefect Flow to simplify data access in low-code manner. -## Update SciCat with HPSS file paths - -TBD +## Update SciCat with HPSS file paths +`BeamlineIngestorController()` in `orchestration/flows/scicat/ingestor_controller.py` contains a method `add_new_dataset_location()` that can be used to update the source folder and host metadata in SciCat with new HPSS location: + +```python + def add_new_dataset_location( + self, + dataset_id: str, + source_folder: str, + source_folder_host: str, + ) -> bool: + """ + Add a new location to an existing dataset in SciCat. + + :param dataset_id: SciCat ID of the dataset. + :param source_folder: "Absolute file path on file server containing the files of this dataset, + e.g. /some/path/to/sourcefolder. In case of a single file dataset, e.g. HDF5 data, + it contains the path up to, but excluding the filename. Trailing slashes are removed.", + + :param source_folder_host: "DNS host name of file server hosting sourceFolder, + optionally including a protocol e.g. [protocol://]fileserver1.example.com", + + """ + dataset = self.scicat_client.datasets_get_one(dataset_id) + # sourceFolder sourceFolderHost are each a string + dataset["sourceFolder"] = source_folder + dataset["sourceFolderHost"] = source_folder_host + self.scicat_client.datasets_update(dataset, dataset_id) + logger.info(f"Added location {source_folder} to dataset {dataset_id}") + return dataset_id +``` \ No newline at end of file diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index cdfbbc35..b53d32eb 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -19,7 +19,6 @@ logger.setLevel(logging.INFO) API_KEY = os.getenv("API_KEY") -TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832" @flow(name="new_832_file_flow") @@ -70,7 +69,6 @@ def process_new_832_file( if nersc_transfer_success: logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") - logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") try: ingestor = TomographyIngestorController(config, config.scicat_client) ingestor.ingest_new_raw_dataset(file_path) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index efe04dd4..c3d9ad27 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -415,32 +415,3 @@ def _upload_raw_dataset( logger.debug(f"dataset: {dataset}") dataset_id = self.scicat_client.upload_new_dataset(dataset) return dataset_id - - # def ingest_new_raw_dataset( - # self, - # file_path: str = "", - # ) -> str: - # """ - # Ingest a new raw dataset from the Tomo832 beamline. - - # :param file_path: Path to the file to ingest. - # :return: SciCat ID of the dataset. - # """ - - # # Ingest the dataset - # ingestor_module = "orchestration.flows.bl832.ingest_tomo832" - # ingestor_module = importlib.import_module(ingestor_module) - # issues: List[Issue] = [] - # new_dataset_id = ingestor_module.ingest( - # self.scicat_client, - # file_path, - # issues, - # ) - # if len(issues) > 0: - # logger.error(f"SciCat ingest failed with {len(issues)} issues") - # for issue in issues: - # logger.error(issue) - # raise Exception("SciCat ingest failed") - # return new_dataset_id - - # Generalizable from ingest_tomo832.py: diff --git a/orchestration/hpss.py b/orchestration/hpss.py index de3c9213..47e0afda 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -7,7 +7,6 @@ from prefect import flow from orchestration.config import BeamlineConfig -from orchestration.flows.bl832.config import Config832 from orchestration.transfer_controller import get_transfer_controller, CopyMethod from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint @@ -23,7 +22,7 @@ def cfs_to_hpss_flow( config: BeamlineConfig = None ) -> bool: """ - The CFS to HPSS flow for BL832. + The CFS to HPSS flow. Parameters ---------- @@ -68,10 +67,10 @@ def hpss_to_cfs_flow( source: HPSSEndpoint = None, destination: FileSystemEndpoint = None, files_to_extract: Optional[List[str]] = None, - config: BeamlineConfig = Config832() + config: BeamlineConfig = None ) -> bool: """ - The HPSS to CFS flow for BL832. + The HPSS to CFS flow. Parameters ---------- @@ -83,11 +82,17 @@ def hpss_to_cfs_flow( The destination endpoint. """ + logger.info("Running hpss_to_cfs_flow") + logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") + + logger.info("Configuring transfer controller for HPSS_TO_CFS.") transfer_controller = get_transfer_controller( transfer_type=CopyMethod.HPSS_TO_CFS, config=config ) + logger.info("HPSSToCFSTransferController selected. Initiating transfer for all file paths.") + result = transfer_controller.copy( file_path=file_path, source=source, @@ -99,7 +104,7 @@ def hpss_to_cfs_flow( if __name__ == "__main__": - + from orchestration.flows.bl832.config import Config832 config = Config832() project_name = "ALS-11193_nbalsara" source = FileSystemEndpoint( From 8e384da24d68994822c46522f3944396d1d58bac Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 14:32:09 -0800 Subject: [PATCH 022/128] Testing HPSS flow, verified that I can create new directories on HPSS from Slurm/SFAPI. --- orchestration/flows/bl832/scicat_ingestor.py | 2 +- orchestration/hpss.py | 6 ++-- orchestration/transfer_controller.py | 35 +++++++++++++++----- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index c3d9ad27..7b6f0df7 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -40,7 +40,7 @@ class TomographyIngestorController(BeamlineIngestorController): Ingestor for Tomo832 beamline. """ DEFAULT_USER = "8.3.2" # In case there's not proposal number - INGEST_SPEC = "als832_dx_3" + INGEST_SPEC = "als832_dx_3" # Where is this spec defined? DATA_SAMPLE_KEYS = [ "/measurement/instrument/sample_motor_stack/setup/axis1pos", diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 47e0afda..364324b0 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -109,11 +109,13 @@ def hpss_to_cfs_flow( project_name = "ALS-11193_nbalsara" source = FileSystemEndpoint( name="CFS", - root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/" + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", + uri="nersc.gov" ) destination = HPSSEndpoint( name="HPSS", - root_path=config.hpss_alsdev["root_path"] + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] ) cfs_to_hpss_flow( file_path=f"{project_name}", diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 3e0e248f..2a4827fc 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -445,22 +445,39 @@ def copy( echo "[LOG] DEST_PATH set to: $DEST_PATH" # ------------------------------------------------------------------ -# Create destination directory on HPSS if it doesn't exist. +# Create destination directory on HPSS recursively using hsi mkdir. # ------------------------------------------------------------------ echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." - if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then echo "[LOG] Destination directory $DEST_PATH already exists." else - echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it." - if hsi -q "mkdir $DEST_PATH" >/dev/null 2>&1; then - echo "[LOG] Created directory $DEST_PATH." - else - echo "[ERROR] Failed to create directory $DEST_PATH." - exit 1 - fi + echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." + current="" + # Split the DEST_PATH on '/' and iterate over each directory component. + IFS='/' read -ra parts <<< "$DEST_PATH" + for part in "${{parts[@]}}"; do + if [ -z "$part" ]; then + continue + fi + current="$current/$part" + if ! hsi -q "ls $current" >/dev/null 2>&1; then + if hsi "mkdir $current" >/dev/null 2>&1; then + echo "[LOG] Created directory $current." + else + echo "[ERROR] Failed to create directory $current." + exit 1 + fi + else + echo "[LOG] Directory $current already exists." + fi + done fi +# List the final HPSS directory tree for logging purposes. +# For some reason this gets logged in the project.err file, not the .out file. +hsi ls $DEST_PATH +echo "[LOG] Job completed at: $(date)" + # # ------------------------------------------------------------------ # # Transfer Logic: Check if SOURCE_PATH is a file or directory. # # ------------------------------------------------------------------ From c1e652e2601cdada8623a5e9f47517aa4a6fc381 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 14:36:04 -0800 Subject: [PATCH 023/128] Updating documentation: --- docs/mkdocs/docs/hpss.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index 0eb64571..006e7728 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -171,6 +171,23 @@ Most of the time we expect transfers to occur from CFS to HPSS on a scheduled ba **`orchestration/transfer_controller.py`:** - **`CFSToHPSSTransferController()`**: This controller uses a Slurm Job Script and SFAPI to launch the tape transfer job. The Slurm script handles the specific logic for handling single and multiple files, on a project by project basis. It reads the files sizes, and creates bundles that are <= 2TB. The groups within each tar archive are saved in a log on NERSC CFS for posterity. + Here is a high level overview of the steps taken within the SFAPI Slurm Job: + 1. Define the source (CFS) and destination (HPSS) paths. + 2. Create the destination directory on HPSS if it doesn't exist. + - Recursively check each part of the incoming file path if the folder exists + - If the folder does not exist, use `hsi mkdir` + - Repeat until the file path is built + 3. Determine if the source is a file or a directory. + - If a file, transfer it using 'hsi cput'. + - If a directory, group files by beam cycle and archive them. + * Cycle 1: Jan 1 - Jul 15 + * Cycle 2: Jul 16 - Dec 31 + * If a group exceeds 2 TB, it is partitioned into multiple tar archives. + * Archive names: + [proposal_name]_[year]-[cycle].tar + [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. + + **`orchestration/hpss.py`:** - **`cfs_to_hpss_flow()`** This Prefect Flow sets up the CFSToHPSSTransferController() and calls the copy command. By registering this Flow, the HPSS transfers can be easily scheduled. From 9cd6ddc20a458caa3be19a2a6d29175c5dd6d2fe Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 15:00:00 -0800 Subject: [PATCH 024/128] Verified that htar bundling and building on HPSS works --- orchestration/transfer_controller.py | 304 ++++++++++++++------------- 1 file changed, 162 insertions(+), 142 deletions(-) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 2a4827fc..2a504ca9 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -446,164 +446,184 @@ def copy( # ------------------------------------------------------------------ # Create destination directory on HPSS recursively using hsi mkdir. +# This section ensures that the entire directory tree specified in DEST_PATH +# exists on HPSS. Since HPSS hsi does not support a recursive mkdir option, +# we split the path into its components and create each directory one by one. # ------------------------------------------------------------------ + echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." + +# Use 'hsi ls' to verify if the destination directory exists. +# The '-q' flag is used for quiet mode, and any output or errors are discarded. if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then echo "[LOG] Destination directory $DEST_PATH already exists." else + # If the directory does not exist, begin the process to create it. echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." + + # Initialize an empty variable 'current' that will store the path built so far. current="" - # Split the DEST_PATH on '/' and iterate over each directory component. + + # Split the DEST_PATH using '/' as the delimiter. + # This creates an array 'parts' where each element is a directory level in the path. IFS='/' read -ra parts <<< "$DEST_PATH" + + # Iterate over each directory component in the 'parts' array. for part in "${{parts[@]}}"; do - if [ -z "$part" ]; then - continue - fi - current="$current/$part" - if ! hsi -q "ls $current" >/dev/null 2>&1; then - if hsi "mkdir $current" >/dev/null 2>&1; then - echo "[LOG] Created directory $current." - else - echo "[ERROR] Failed to create directory $current." - exit 1 - fi - else - echo "[LOG] Directory $current already exists." - fi + # Skip any empty parts. An empty string may occur if the path starts with a '/'. + if [ -z "$part" ]; then + continue + fi + + # Append the current part to the 'current' path variable. + # This step incrementally reconstructs the full path one directory at a time. + current="$current/$part" + + # Check if the current directory exists on HPSS using 'hsi ls'. + if ! hsi -q "ls $current" >/dev/null 2>&1; then + # If the directory does not exist, attempt to create it using 'hsi mkdir'. + if hsi "mkdir $current" >/dev/null 2>&1; then + echo "[LOG] Created directory $current." + else + echo "[ERROR] Failed to create directory $current." + exit 1 + fi + else + echo "[LOG] Directory $current already exists." + fi done fi # List the final HPSS directory tree for logging purposes. # For some reason this gets logged in the project.err file, not the .out file. hsi ls $DEST_PATH -echo "[LOG] Job completed at: $(date)" -# # ------------------------------------------------------------------ -# # Transfer Logic: Check if SOURCE_PATH is a file or directory. -# # ------------------------------------------------------------------ -# echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" -# if [ -f "$SOURCE_PATH" ]; then -# # Case: Single file detected. -# echo "[LOG] Single file detected. Transferring via hsi cput." -# FILE_NAME=$(basename "$SOURCE_PATH") -# echo "[LOG] File name: $FILE_NAME" -# hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" -# echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." - -# elif [ -d "$SOURCE_PATH" ]; then -# # Case: Directory detected. -# echo "[LOG] Directory detected. Initiating bundling process." -# THRESHOLD=2199023255552 # 2 TB in bytes. -# echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" - -# # ------------------------------------------------------------------ -# # Group files based on modification date. -# # ------------------------------------------------------------------ -# echo "[LOG] Grouping files by modification date." -# FILE_LIST=$(mktemp) -# find "$SOURCE_PATH" -type f > "$FILE_LIST" -# echo "[LOG] List of files stored in temporary file: $FILE_LIST" - -# # Declare associative arrays to hold grouped file paths and sizes. -# declare -A group_files -# declare -A group_sizes - -# while IFS= read -r file; do -# mtime=$(stat -c %Y "$file") -# year=$(date -d @"$mtime" +%Y) -# month=$(date -d @"$mtime" +%m | sed 's/^0*//') -# day=$(date -d @"$mtime" +%d | sed 's/^0*//') -# # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. -# if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then -# cycle=1 -# else -# cycle=2 -# fi -# key="${{year}}-${{cycle}}" -# group_files["$key"]="${{group_files["$key"]:-}} $file" -# fsize=$(stat -c %s "$file") -# group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) -# done < "$FILE_LIST" -# rm "$FILE_LIST" -# echo "[LOG] Completed grouping files." - -# # Print the files in each group at the end -# for key in "${{!group_files[@]}}"; do -# echo "[LOG] Group $key contains files:" -# for f in ${{group_files["$key"]}}; do -# echo " $f" -# done -# done - -# # ------------------------------------------------------------------ -# # Bundle files into tar archives. -# # ------------------------------------------------------------------ -# for key in "${{!group_files[@]}}"; do -# files=(${{group_files["$key"]}}) -# total_group_size=${{group_sizes["$key"]}} -# echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." - -# part=0 -# current_size=0 -# current_files=() -# for f in "${{files[@]}}"; do -# fsize=$(stat -c %s "$f") -# # If adding this file exceeds the threshold, process the current bundle. -# if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then -# if [ $part -eq 0 ]; then -# tar_name="${{FOLDER_NAME}}_${{key}}.tar" -# else -# tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" -# fi -# echo "[LOG] Bundle reached threshold." -# echo "[LOG] Files in current bundle:" -# for file in "${{current_files[@]}}"; do -# echo "$file" -# done -# echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." -# htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") -# part=$((part+1)) -# echo "[DEBUG] Resetting bundle variables." -# current_files=() -# current_size=0 -# fi -# current_files+=("$f") -# current_size=$(( current_size + fsize )) -# done -# if [ ${{#current_files[@]}} -gt 0 ]; then -# if [ $part -eq 0 ]; then -# tar_name="${{FOLDER_NAME}}_${{key}}.tar" -# else -# tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" -# fi -# echo "[LOG] Final bundle for group $key:" -# echo "[LOG] Files in final bundle:" -# for file in "${{current_files[@]}}"; do -# echo "$file" -# done -# echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." -# echo "[LOG] Bundle size: $current_size bytes." -# htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") -# fi -# echo "[LOG] Completed processing group $key." -# done -# else -# echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." -# exit 1 -# fi - -# # ------------------------------------------------------------------ -# # Logging: Display directory trees for both source and destination. -# # ------------------------------------------------------------------ -# echo "[LOG] Listing Source (CFS) Tree:" -# if [ -d "$SOURCE_PATH" ]; then -# find "$SOURCE_PATH" -print -# else -# echo "[LOG] $SOURCE_PATH is a file." -# fi - -# echo "[LOG] Listing Destination (HPSS) Tree:" -# hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" +# ------------------------------------------------------------------ +# Transfer Logic: Check if SOURCE_PATH is a file or directory. +# ------------------------------------------------------------------ +echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" +if [ -f "$SOURCE_PATH" ]; then + # Case: Single file detected. + echo "[LOG] Single file detected. Transferring via hsi cput." + FILE_NAME=$(basename "$SOURCE_PATH") + echo "[LOG] File name: $FILE_NAME" + hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." + +elif [ -d "$SOURCE_PATH" ]; then + # Case: Directory detected. + echo "[LOG] Directory detected. Initiating bundling process." + THRESHOLD=2199023255552 # 2 TB in bytes. + echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" + + # ------------------------------------------------------------------ + # Group files based on modification date. + # ------------------------------------------------------------------ + echo "[LOG] Grouping files by modification date." + FILE_LIST=$(mktemp) + find "$SOURCE_PATH" -type f > "$FILE_LIST" + echo "[LOG] List of files stored in temporary file: $FILE_LIST" + + # Declare associative arrays to hold grouped file paths and sizes. + declare -A group_files + declare -A group_sizes + + while IFS= read -r file; do + mtime=$(stat -c %Y "$file") + year=$(date -d @"$mtime" +%Y) + month=$(date -d @"$mtime" +%m | sed 's/^0*//') + day=$(date -d @"$mtime" +%d | sed 's/^0*//') + # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then + cycle=1 + else + cycle=2 + fi + key="${{year}}-${{cycle}}" + group_files["$key"]="${{group_files["$key"]:-}} $file" + fsize=$(stat -c %s "$file") + group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) + done < "$FILE_LIST" + rm "$FILE_LIST" + echo "[LOG] Completed grouping files." + + # Print the files in each group at the end + for key in "${{!group_files[@]}}"; do + echo "[LOG] Group $key contains files:" + for f in ${{group_files["$key"]}}; do + echo " $f" + done + done + + # ------------------------------------------------------------------ + # Bundle files into tar archives. + # ------------------------------------------------------------------ + for key in "${{!group_files[@]}}"; do + files=(${{group_files["$key"]}}) + total_group_size=${{group_sizes["$key"]}} + echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." + + part=0 + current_size=0 + current_files=() + for f in "${{files[@]}}"; do + fsize=$(stat -c %s "$f") + # If adding this file exceeds the threshold, process the current bundle. + if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Bundle reached threshold." + echo "[LOG] Files in current bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." + htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + part=$((part+1)) + echo "[DEBUG] Resetting bundle variables." + current_files=() + current_size=0 + fi + current_files+=("$f") + current_size=$(( current_size + fsize )) + done + if [ ${{#current_files[@]}} -gt 0 ]; then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Final bundle for group $key:" + echo "[LOG] Files in final bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." + echo "[LOG] Bundle size: $current_size bytes." + htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + fi + echo "[LOG] Completed processing group $key." + done +else + echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." + exit 1 +fi + +# ------------------------------------------------------------------ +# Logging: Display directory trees for both source and destination. +# ------------------------------------------------------------------ +echo "[LOG] Listing Source (CFS) Tree:" +if [ -d "$SOURCE_PATH" ]; then + find "$SOURCE_PATH" -print +else + echo "[LOG] $SOURCE_PATH is a file." +fi + +echo "[LOG] Listing Destination (HPSS) Tree:" +hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" echo "[LOG] Job completed at: $(date)" """ From 0e7ff47f92655f5fc1a2a6b2d375baeaf51d031b Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 15:58:35 -0800 Subject: [PATCH 025/128] Updating HPSS documentation typo --- docs/mkdocs/docs/hpss.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index 006e7728..d56607ce 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -184,8 +184,8 @@ Most of the time we expect transfers to occur from CFS to HPSS on a scheduled ba * Cycle 2: Jul 16 - Dec 31 * If a group exceeds 2 TB, it is partitioned into multiple tar archives. * Archive names: - [proposal_name]_[year]-[cycle].tar - [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. + `[proposal_name]_[year]-[cycle].tar` + `[proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc.` **`orchestration/hpss.py`:** From c78dc9689d17e06bfc80f1ce7094e81e552bc3ff Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 26 Feb 2025 16:02:01 -0800 Subject: [PATCH 026/128] Adding a _find_dataset() method to ingestor_controller base class, in case a SciCat ID is not known, but the proposal_id and file_name are known. I anticipate a case where we move data later, but do not have the SciCat ID immediately available. --- .../flows/scicat/ingestor_controller.py | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index f5c25896..ac820278 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from logging import getLogger +from typing import Optional from orchestration.config import BeamlineConfig from pyscicat.client import ScicatClient, from_credentials @@ -62,10 +63,12 @@ def ingest_new_derived_dataset( def add_new_dataset_location( self, - dataset_id: str, - source_folder: str, - source_folder_host: str, - ) -> bool: + dataset_id: Optional[str] = None, + proposal_id: Optional[str] = None, + file_name: Optional[str] = None, + source_folder: str = None, + source_folder_host: str = None + ) -> str: """ Add a new location to an existing dataset in SciCat. @@ -78,7 +81,13 @@ def add_new_dataset_location( optionally including a protocol e.g. [protocol://]fileserver1.example.com", """ + # If dataset_id is not provided, we need to find it using proposal_id and file_name. + # Otherwise, we use the provided dataset_id directly. + if dataset_id is None and proposal_id and file_name: + dataset_id = self._find_dataset(proposal_id=proposal_id, file_name=file_name) + dataset = self.scicat_client.datasets_get_one(dataset_id) + # sourceFolder sourceFolderHost are each a string dataset["sourceFolder"] = source_folder dataset["sourceFolderHost"] = source_folder_host @@ -95,3 +104,54 @@ def remove_dataset_location( """ pass + + def _find_dataset( + self, + proposal_id: Optional[str] = None, # The ALS proposal ID, not the SciCat ID + file_name: Optional[str] = None + ) -> str: + """ + Find a dataset in SciCat and return the ID based on proposal ID and file name. + This method is used when a dataset ID is not provided. + If more than one dataset is found, an error is raised, and the user is advised to check the logs. + If no dataset is found, an error is raised. + If exactly one dataset is found, its ID is returned. + This method is intended to be used internally within the class. + + Parameters: + self, + proposal_id (Optional[str]): The proposal identifier used in ingestion. + file_name (Optional[str]): The dataset name (derived from file name). + + Raises: + ValueError: If insufficient search parameters are provided, + no dataset is found, or multiple datasets match. + """ + # Require both search terms if no dataset_id is given. + if not (proposal_id and file_name): + raise ValueError("Either a dataset ID must be provided or both proposal_id and file_name must be given.") + + query_fields = { + "proposalId": proposal_id, + "datasetName": file_name + } + results = self.scicat_client.datasets_find(query_fields=query_fields) + count = results.get("count", 0) + + if count == 0: + raise ValueError(f"No dataset found for proposal '{proposal_id}' with name '{file_name}'.") + elif count > 1: + # Log all found dataset IDs for human review. + dataset_ids = [d.get("pid", "N/A") for d in results["data"]] + logger.error( + f"Multiple datasets found for proposal '{proposal_id}' with name '{file_name}': {dataset_ids}. Please verify." + ) + raise ValueError( + f"Multiple datasets found for proposal '{proposal_id}' with name '{file_name}'. See log for details." + ) + dataset = results["data"][0] + dataset_id = dataset.get("pid") + if not dataset_id: + raise ValueError("The dataset returned does not have a valid 'pid' field.") + + return dataset_id From 3278a1fee6eb1ae556b78b9baf55f64ca7e13271 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 27 Feb 2025 13:19:00 -0800 Subject: [PATCH 027/128] Successfully extracted files from a .tar archive on HPSS back to CFS! One thing to change is the file_path stored in the .tar (currently each file in the tar has the full cfs path, which seems silly. I think it should just be the relative path within the project). --- orchestration/hpss.py | 76 +++++++++--- orchestration/transfer_controller.py | 171 +++++++++++++++++++-------- 2 files changed, 179 insertions(+), 68 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 364324b0..c8678245 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -104,22 +104,60 @@ def hpss_to_cfs_flow( if __name__ == "__main__": - from orchestration.flows.bl832.config import Config832 - config = Config832() - project_name = "ALS-11193_nbalsara" - source = FileSystemEndpoint( - name="CFS", - root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", - uri="nersc.gov" - ) - destination = HPSSEndpoint( - name="HPSS", - root_path=config.hpss_alsdev["root_path"], - uri=config.hpss_alsdev["uri"] - ) - cfs_to_hpss_flow( - file_path=f"{project_name}", - source=source, - destination=destination, - config=config - ) + TEST_CFS_TO_HPSS = False + TEST_HPSS_TO_CFS = True + + # ------------------------------------------------------ + # Test transfer from CFS to HPSS + # ------------------------------------------------------ + if TEST_CFS_TO_HPSS: + from orchestration.flows.bl832.config import Config832 + config = Config832() + project_name = "ALS-11193_nbalsara" + source = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", + uri="nersc.gov" + ) + destination = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + cfs_to_hpss_flow( + file_path=f"{project_name}", + source=source, + destination=destination, + config=config + ) + + # ------------------------------------------------------ + # Test transfer from HPSS to CFS + # ------------------------------------------------------ + if TEST_HPSS_TO_CFS: + from orchestration.flows.bl832.config import Config832 + config = Config832() + relative_file_path = f"{config.beamline_id}/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" + source = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], # root_path: /home/a/alsdev/data_mover + uri=config.hpss_alsdev["uri"] + ) + destination = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/retrieved_from_tape", + uri="nersc.gov" + ) + + files_to_extract = [ + "/global/cfs/cdirs/als/data_mover/8.3.2/raw/ALS-11193_nbalsara/20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", + "/global/cfs/cdirs/als/data_mover/8.3.2/raw/ALS-11193_nbalsara/20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", + ] + + hpss_to_cfs_flow( + file_path=f"{relative_file_path}", + source=source, + destination=destination, + files_to_extract=files_to_extract, + config=config + ) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 2a504ca9..4728fd6b 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -371,6 +371,17 @@ def copy( source: FileSystemEndpoint = None, destination: HPSSEndpoint = None ) -> bool: + """ + Copy a file or directory from a CFS source endpoint to an HPSS destination endpoint. + + Args: + file_path (str): Path to the file or directory on CFS. + source (FileSystemEndpoint): The CFS source endpoint. + destination (HPSSEndpoint): The HPSS destination endpoint. + + Returns: + bool: True if the transfer job completes successfully, False otherwise. + """ logger.info("Transferring data from CFS to HPSS") if not file_path or not source or not destination: logger.error("Missing required parameters for CFSToHPSSTransferController.") @@ -414,9 +425,9 @@ def copy( #SBATCH -A als # Specify the account. #SBATCH -C cron # Use the 'cron' constraint. #SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_to_HPSS_{file_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/{file_path}_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{file_path}_%j.err # Standard error log file. +#SBATCH --job-name=transfer_to_HPSS_{proposal_name} # Set a descriptive job name. +#SBATCH --output={logs_path}/{proposal_name}_to_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{proposal_name}_to_hpss_%j.err # Standard error log file. #SBATCH --licenses=SCRATCH # Request the SCRATCH license. #SBATCH --mem=20GB # Request #GB of memory. Default 2GB. @@ -666,25 +677,16 @@ def copy( class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): """ - Use SFAPI to move data between HPSS and CFS at NERSC. + Use SFAPI, Slurm, hsi and htar to move data between HPSS and CFS at NERSC. This controller retrieves data from an HPSS source endpoint and places it on a CFS destination endpoint. - It supports: - - Single file retrieval via hsi get. - - Full tar archive extraction via htar -xvf. - - Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), + It supports the following modes: + - "single": Single file retrieval via hsi get. + - "tar": Full tar archive extraction via htar -xvf. + - "partial": Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), only the specified files will be extracted. A single SLURM job script is generated that branches based on the mode. - Args: - file_path (str): Path to the file or tar archive on HPSS. - source (HPSSEndpoint): The HPSS source endpoint. - destination (FileSystemEndpoint): The CFS destination endpoint. - files_to_extract (List[str], optional): Specific files to extract from the tar archive. - If provided (and file_path ends with '.tar'), only these files will be extracted. - - Returns: - bool: True if the transfer job completes successfully, False otherwise. """ def __init__( @@ -702,18 +704,34 @@ def copy( destination: FileSystemEndpoint = None, files_to_extract: Optional[List[str]] = None, ) -> bool: + """ + Copy a file from an HPSS source endpoint to a CFS destination endpoint. + + Args: + file_path (str): Path to the file or tar archive on HPSS. + source (HPSSEndpoint): The HPSS source endpoint. + destination (FileSystemEndpoint): The CFS destination endpoint. + files_to_extract (List[str], optional): Specific files to extract from the tar archive. + If provided (and file_path ends with '.tar'), only these files will be extracted. + If not provided, the entire tar archive will be extracted. + If file_path is a single file, this parameter is ignored. + + Returns: + bool: True if the transfer job completes successfully, False otherwise. + """ logger.info("Starting HPSS to CFS transfer.") if not file_path or not source or not destination: logger.error("Missing required parameters: file_path, source, or destination.") return False - # Set the job name suffix based on the file name (or archive stem) - job_name_suffix = Path(file_path).stem - # Compute the full HPSS path from the source endpoint. hpss_path = source.full_path(file_path) dest_root = destination.root_path - logs_path = "/global/cfs/cdirs/als/data_mover/hpss_transfer_logs" + + # Get the beamline_id from the configuration. + beamline_id = self.config.beamline_id + + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" # If files_to_extract is provided, join them as a spaceā€separated string. files_to_extract_str = " ".join(files_to_extract) if files_to_extract else "" @@ -724,26 +742,56 @@ def copy( # else MODE is "tar". # - Otherwise, MODE is "single" and hsi get is used. job_script = fr"""#!/bin/bash -#SBATCH -q xfer -#SBATCH -A als -#SBATCH -C cron -#SBATCH --time=12:00:00 -#SBATCH --job-name=transfer_from_HPSS_{job_name_suffix} -#SBATCH --output={logs_path}/%j.out -#SBATCH --error={logs_path}/%j.err -#SBATCH --licenses=SCRATCH -#SBATCH --mem=100GB - -set -euo pipefail -date +#SBATCH -q xfer # Specify the SLURM queue to u +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/{file_path}_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{file_path}_from_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" -# Environment variables provided by Python. -HPSS_PATH="{hpss_path}" +# ------------------------------------------------------------------- +# Define source and destination variables. +# ------------------------------------------------------------------- + +echo "[LOG] Defining source and destination paths." + +# SOURCE_PATH: Full path of the file or directory on HPSS. +SOURCE_PATH="{hpss_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" + +# DEST_ROOT: Root destination on CFS built from configuration. DEST_ROOT="{dest_root}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" + +# FILES_TO_EXTRACT: Specific files to extract from the tar archive, if any. +# If not provided, this will be empty. FILES_TO_EXTRACT="{files_to_extract_str}" +echo "[LOG] FILES_TO_EXTRACT set to: $FILES_TO_EXTRACT" + +# ------------------------------------------------------------------- +# Verify that SOURCE_PATH exists on HPSS using hsi ls. +# ------------------------------------------------------------------- + +echo "[LOG] Verifying file existence with hsi ls." +if ! hsi ls "$SOURCE_PATH" >/dev/null 2>&1; then + echo "[ERROR] File not found on HPSS: $SOURCE_PATH" + exit 1 +fi + +# ------------------------------------------------------------------- +# Determine the transfer mode based on the type (file vs tar). +# ------------------------------------------------------------------- -# Determine the transfer mode in bash. -if [[ "$HPSS_PATH" =~ \.tar$ ]]; then +echo "[LOG] Determining transfer mode based on the type (file vs tar)." + +# Check if SOURCE_PATH ends with .tar +if [[ "$SOURCE_PATH" =~ \.tar$ ]]; then + # If FILES_TO_EXTRACT is nonempty, MODE becomes "partial", else MODE is "tar". if [ -n "${{FILES_TO_EXTRACT}}" ]; then MODE="partial" else @@ -754,27 +802,52 @@ def copy( fi echo "Transfer mode: $MODE" + +# ------------------------------------------------------------------- +# Transfer Logic: Based on the mode, perform the appropriate transfer. +# ------------------------------------------------------------------- + if [ "$MODE" = "single" ]; then - echo "Single file detected. Using hsi get." - mkdir -p "$DEST_ROOT" - hsi get "$HPSS_PATH" "$DEST_ROOT/" + echo "[LOG] Single file detected. Using hsi get." + # mkdir -p "$DEST_ROOT" + # hsi get "$SOURCE_PATH" "$DEST_ROOT/" elif [ "$MODE" = "tar" ]; then - echo "Tar archive detected. Extracting entire archive using htar." - ARCHIVE_BASENAME=$(basename "$HPSS_PATH") + echo "[LOG] Tar archive detected. Extracting entire archive using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" - mkdir -p "$DEST_PATH" - htar -xvf "$HPSS_PATH" -C "$DEST_PATH" + echo "[LOG] Extracting to: $DEST_PATH" + # mkdir -p "$DEST_PATH" + # htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" elif [ "$MODE" = "partial" ]; then - echo "Partial extraction detected. Extracting selected files using htar." - ARCHIVE_BASENAME=$(basename "$HPSS_PATH") + echo "[LOG] Partial extraction detected. Extracting selected files using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + + # Verify that each requested file exists in the tar archive. + echo "[LOG] Verifying requested files are in the tar archive." + ARCHIVE_CONTENTS=$(htar -tvf "$SOURCE_PATH") + echo "[LOG] List: $ARCHIVE_CONTENTS" + for file in $FILES_TO_EXTRACT; do + echo "[LOG] Checking for file: $file" + if ! echo "$ARCHIVE_CONTENTS" | grep -q "$file"; then + echo "[ERROR] Requested file '$file' not found in archive $SOURCE_PATH" + exit 1 + else + echo "[LOG] File '$file' found in archive." + fi + done + + echo "[LOG] All requested files verified. Proceeding with extraction." mkdir -p "$DEST_PATH" - echo "Files to extract: $FILES_TO_EXTRACT" - htar -xvf "$HPSS_PATH" -C "$DEST_PATH" $FILES_TO_EXTRACT + (cd "$DEST_PATH" && htar -xvf "$SOURCE_PATH" -Hnostage $FILES_TO_EXTRACT) + + echo "[LOG] Extraction complete. Listing contents of $DEST_PATH:" + ls -l "$DEST_PATH" + else - echo "Error: Unknown mode: $MODE" + echo "[ERROR]: Unknown mode: $MODE" exit 1 fi From 6618d99ffd1450abf44708dfe4f9493057ef8585 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 27 Feb 2025 14:57:08 -0800 Subject: [PATCH 028/128] Moving all HPSS related transfer/prune implementations into orchestration/hpss.py. Working on an HPSS pruning implementation --- orchestration/hpss.py | 725 ++++++++++++++++++++++++++- orchestration/prune_controller.py | 55 +- orchestration/transfer_controller.py | 568 +-------------------- 3 files changed, 728 insertions(+), 620 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index c8678245..d2c0804a 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -1,19 +1,31 @@ """ -This module contains the HPSS flow for BL832. +This module contains HPSS related functions and classes. """ +import datetime import logging +from pathlib import Path +import re +import time from typing import List, Optional, Union from prefect import flow +from sfapi_client import Client +from sfapi_client.compute import Machine from orchestration.config import BeamlineConfig -from orchestration.transfer_controller import get_transfer_controller, CopyMethod +from orchestration.prefect import schedule_prefect_flow +from orchestration.prune_controller import PruneController +from orchestration.transfer_controller import get_transfer_controller, CopyMethod, TransferController from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +# --------------------------------- +# HPSS Prefect Flows +# --------------------------------- + @flow(name="cfs_to_hpss_flow") def cfs_to_hpss_flow( file_path: Union[str, List[str]] = None, @@ -103,6 +115,711 @@ def hpss_to_cfs_flow( return result +# ---------------------------------- +# HPSS Prune Controller +# ---------------------------------- + +class HPSSPruneController(PruneController[HPSSEndpoint]): + def __init__( + self, + client: Client, + config: BeamlineConfig, + ) -> None: + super().__init__(config) + self.client = client + + def prune( + self, + file_path: str = None, + source_endpoint: HPSSEndpoint = None, + check_endpoint: FileSystemEndpoint = None, + days_from_now: datetime.timedelta = 0 + ) -> bool: + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + schedule_prefect_flow( + "prune_hpss_endpoint/prune_hpss_endpoint", + flow_name, + { + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + + datetime.timedelta(days=days_from_now), + ) + return True + + @flow(name="prune_hpss_endpoint") + def _prune_hpss_endpoint( + self, + relative_path: str, + source_endpoint: HPSSEndpoint, + check_endpoint: Union[FileSystemEndpoint, None] = None, + config: BeamlineConfig = None + ) -> None: + """ + Prune files from HPSS. + + Args: + relative_path (str): The path of the file or directory to prune. + source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. + check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + """ + + beamline_id = config.beamline_id + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + + job_script = rf"""#!/bin/bash +# ------------------------------------------------------------------ +# SLURM Job Script for Pruning Data from HPSS +# ------------------------------------------------------------------ + +#SBATCH -q xfer # Specify the SLURM queue to use. +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_to_HPSS_{relative_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/{relative_path}_prune_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{relative_path}_prune_from_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. + +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# Check if the file exists on HPSS +if hsi -c "ls {source_endpoint.full_path(relative_path)}" &> /dev/null; then + echo "[LOG] File {relative_path} exists on HPSS. Proceeding to prune." + # Prune the file from HPSS + hsi -c "rm {source_endpoint.full_path(relative_path)}" + echo "[LOG] File {relative_path} has been pruned from HPSS." +fi +echo "[LOG] Job completed at: $(date)" + +""" + try: + logger.info("Submitting HPSS transfer job to Perlmutter.") + perlmutter = self.client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + logger.info(f"Submitted job ID: {job.jobid}") + + try: + job.update() + except Exception as update_err: + logger.warning(f"Initial job update failed, continuing: {update_err}") + + time.sleep(60) + logger.info(f"Job {job.jobid} current state: {job.state}") + + job.complete() # Wait until the job completes. + logger.info("Transfer job completed successfully.") + return True + + except Exception as e: + logger.error(f"Error during job submission or completion: {e}") + match = re.search(r"Job not found:\s*(\d+)", str(e)) + if match: + jobid = match.group(1) + logger.info(f"Attempting to recover job {jobid}.") + try: + job = self.client.perlmutter.job(jobid=jobid) + time.sleep(30) + job.complete() + logger.info("Transfer job completed successfully after recovery.") + return True + except Exception as recovery_err: + logger.error(f"Failed to recover job {jobid}: {recovery_err}") + return False + else: + return False + + +# ---------------------------------- +# HPSS Transfer Controllers +# ---------------------------------- + +class CFSToHPSSTransferController(TransferController[HPSSEndpoint]): + """ + Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. + + This controller requires the source to be a FileSystemEndpoint on CFS and the + destination to be an HPSSEndpoint. For a single file, the transfer is done using hsi (via hsi cput). + For a directory, the transfer is performed with htar. In this updated version, if the source is a + directory then the files are bundled into tar archives based on their modification dates as follows: + - Files with modification dates between Jan 1 and Jul 15 (inclusive) are grouped together + (Cycle 1 for that year). + - Files with modification dates between Jul 16 and Dec 31 are grouped together (Cycle 2). + + Within each group, if the total size exceeds 2 TB the files are partitioned into multiple tar bundles. + The resulting naming convention on HPSS is: + + /home/a/alsdev/data_mover/[beamline]/raw/[proposal_name]/ + [proposal_name]_[year]-[cycle].tar + [proposal_name]_[year]-[cycle]_part0.tar + [proposal_name]_[year]-[cycle]_part1.tar + ... + + At the end of the SLURM script, the directory tree for both the source (CFS) and destination (HPSS) + is echoed for logging purposes. + """ + + def __init__( + self, + client: Client, + config: BeamlineConfig + ) -> None: + super().__init__(config) + self.client = client + + def copy( + self, + file_path: str = None, + source: FileSystemEndpoint = None, + destination: HPSSEndpoint = None + ) -> bool: + """ + Copy a file or directory from a CFS source endpoint to an HPSS destination endpoint. + + Args: + file_path (str): Path to the file or directory on CFS. + source (FileSystemEndpoint): The CFS source endpoint. + destination (HPSSEndpoint): The HPSS destination endpoint. + + Returns: + bool: True if the transfer job completes successfully, False otherwise. + """ + logger.info("Transferring data from CFS to HPSS") + if not file_path or not source or not destination: + logger.error("Missing required parameters for CFSToHPSSTransferController.") + return False + + # Compute the full path on CFS for the file/directory. + full_cfs_path = source.full_path(file_path) + # Get the beamline_id from the configuration. + beamline_id = self.config.beamline_id + # Build the HPSS destination root path using the convention: [destination.root_path]/[beamline_id]/raw + hpss_root_path = f"{destination.root_path.rstrip('/')}/{beamline_id}/raw" + + # Determine the proposal (project) folder name from the file_path. + path = Path(file_path) + proposal_name = path.parent.name + if not proposal_name or proposal_name == ".": # if file_path is in the root directory + proposal_name = file_path + + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + + # Build the SLURM job script with detailed inline comments for clarity. + job_script = rf"""#!/bin/bash +# ------------------------------------------------------------------ +# SLURM Job Script for Transferring Data from CFS to HPSS +# This script will: +# 1. Define the source (CFS) and destination (HPSS) paths. +# 2. Create the destination directory on HPSS if it doesn't exist. +# 3. Determine if the source is a file or a directory. +# - If a file, transfer it using 'hsi cput'. +# - If a directory, group files by beam cycle and archive them. +# * Cycle 1: Jan 1 - Jul 15 +# * Cycle 2: Jul 16 - Dec 31 +# * If a group exceeds 2 TB, it is partitioned into multiple tar archives. +# * Archive names: +# [proposal_name]_[year]-[cycle].tar +# [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. +# 4. Echo directory trees for both source and destination for logging. +# ------------------------------------------------------------------ + +#SBATCH -q xfer # Specify the SLURM queue to use. +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_to_HPSS_{proposal_name} # Set a descriptive job name. +#SBATCH --output={logs_path}/{proposal_name}_to_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{proposal_name}_to_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. + +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# ------------------------------------------------------------------ +# Define source and destination variables. +# ------------------------------------------------------------------ +echo "[LOG] Defining source and destination paths." + +# SOURCE_PATH: Full path of the file or directory on CFS. +SOURCE_PATH="{full_cfs_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" + +# DEST_ROOT: Root destination on HPSS built from configuration. +DEST_ROOT="{hpss_root_path}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" + +# FOLDER_NAME: Proposal name (project folder) derived from the file path. +FOLDER_NAME="{proposal_name}" +echo "[LOG] FOLDER_NAME set to: $FOLDER_NAME" + +# DEST_PATH: Final HPSS destination directory. +DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" +echo "[LOG] DEST_PATH set to: $DEST_PATH" + +# ------------------------------------------------------------------ +# Create destination directory on HPSS recursively using hsi mkdir. +# This section ensures that the entire directory tree specified in DEST_PATH +# exists on HPSS. Since HPSS hsi does not support a recursive mkdir option, +# we split the path into its components and create each directory one by one. +# ------------------------------------------------------------------ + +echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." + +# Use 'hsi ls' to verify if the destination directory exists. +# The '-q' flag is used for quiet mode, and any output or errors are discarded. +if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then + echo "[LOG] Destination directory $DEST_PATH already exists." +else + # If the directory does not exist, begin the process to create it. + echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." + + # Initialize an empty variable 'current' that will store the path built so far. + current="" + + # Split the DEST_PATH using '/' as the delimiter. + # This creates an array 'parts' where each element is a directory level in the path. + IFS='/' read -ra parts <<< "$DEST_PATH" + + # Iterate over each directory component in the 'parts' array. + for part in "${{parts[@]}}"; do + # Skip any empty parts. An empty string may occur if the path starts with a '/'. + if [ -z "$part" ]; then + continue + fi + + # Append the current part to the 'current' path variable. + # This step incrementally reconstructs the full path one directory at a time. + current="$current/$part" + + # Check if the current directory exists on HPSS using 'hsi ls'. + if ! hsi -q "ls $current" >/dev/null 2>&1; then + # If the directory does not exist, attempt to create it using 'hsi mkdir'. + if hsi "mkdir $current" >/dev/null 2>&1; then + echo "[LOG] Created directory $current." + else + echo "[ERROR] Failed to create directory $current." + exit 1 + fi + else + echo "[LOG] Directory $current already exists." + fi + done +fi + +# List the final HPSS directory tree for logging purposes. +# For some reason this gets logged in the project.err file, not the .out file. +hsi ls $DEST_PATH + +# ------------------------------------------------------------------ +# Transfer Logic: Check if SOURCE_PATH is a file or directory. +# ------------------------------------------------------------------ +echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" +if [ -f "$SOURCE_PATH" ]; then + # Case: Single file detected. + echo "[LOG] Single file detected. Transferring via hsi cput." + FILE_NAME=$(basename "$SOURCE_PATH") + echo "[LOG] File name: $FILE_NAME" + hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." + +elif [ -d "$SOURCE_PATH" ]; then + # Case: Directory detected. + echo "[LOG] Directory detected. Initiating bundling process." + THRESHOLD=2199023255552 # 2 TB in bytes. + echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" + + # ------------------------------------------------------------------ + # Generate a list of relative file paths in the project directory. + # This list will be used to group files by their modification date. + # ------------------------------------------------------------------ + # Create a temporary file to store the list of relative file paths. + # Explanation: + # 1. FILE_LIST=$(mktemp) + # - mktemp creates a unique temporary file and its path is stored in FILE_LIST. + # + # 2. (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') + # - The parentheses run the commands in a subshell, so the directory change does not affect the current shell. + # - cd "$SOURCE_PATH": Changes the working directory to the source directory. + # - find . -type f: Recursively finds all files starting from the current directory (which is now SOURCE_PATH), + # outputting paths prefixed with "./". + # - sed 's|^\./||': Removes the leading "./" from each file path, resulting in relative paths without the prefix. + # + # 3. The output is then redirected into the temporary file specified by FILE_LIST. + # ------------------------------------------------------------------ + echo "[LOG] Grouping files by modification date." + + FILE_LIST=$(mktemp) + (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') > "$FILE_LIST" + + echo "[LOG] List of files stored in temporary file: $FILE_LIST" + + # Declare associative arrays to hold grouped file paths and sizes. + declare -A group_files + declare -A group_sizes + + # ------------------------------------------------------------------ + # Group files by modification date. + # ------------------------------------------------------------------ + + while IFS= read -r file; do + mtime=$(stat -c %Y "$file") + year=$(date -d @"$mtime" +%Y) + month=$(date -d @"$mtime" +%m | sed 's/^0*//') + day=$(date -d @"$mtime" +%d | sed 's/^0*//') + # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then + cycle=1 + else + cycle=2 + fi + key="${{year}}-${{cycle}}" + group_files["$key"]="${{group_files["$key"]:-}} $file" + fsize=$(stat -c %s "$file") + group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) + done < "$FILE_LIST" + rm "$FILE_LIST" + echo "[LOG] Completed grouping files." + + # Print the files in each group at the end + for key in "${{!group_files[@]}}"; do + echo "[LOG] Group $key contains files:" + for f in ${{group_files["$key"]}}; do + echo " $f" + done + done + + # ------------------------------------------------------------------ + # Bundle files into tar archives. + # ------------------------------------------------------------------ + for key in "${{!group_files[@]}}"; do + files=(${{group_files["$key"]}}) + total_group_size=${{group_sizes["$key"]}} + echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." + + part=0 + current_size=0 + current_files=() + for f in "${{files[@]}}"; do + fsize=$(stat -c %s "$f") + # If adding this file exceeds the threshold, process the current bundle. + if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Bundle reached threshold." + echo "[LOG] Files in current bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + part=$((part+1)) + echo "[DEBUG] Resetting bundle variables." + current_files=() + current_size=0 + fi + current_files+=("$f") + current_size=$(( current_size + fsize )) + done + if [ ${{#current_files[@]}} -gt 0 ]; then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Final bundle for group $key:" + echo "[LOG] Files in final bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." + echo "[LOG] Bundle size: $current_size bytes." + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + fi + echo "[LOG] Completed processing group $key." + done +else + echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." + exit 1 +fi + +# ------------------------------------------------------------------ +# Logging: Display directory trees for both source and destination. +# ------------------------------------------------------------------ +echo "[LOG] Listing Source (CFS) Tree:" +if [ -d "$SOURCE_PATH" ]; then + find "$SOURCE_PATH" -print +else + echo "[LOG] $SOURCE_PATH is a file." +fi + +echo "[LOG] Listing Destination (HPSS) Tree:" +hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" + +echo "[LOG] Job completed at: $(date)" +""" + try: + logger.info("Submitting HPSS transfer job to Perlmutter.") + perlmutter = self.client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + logger.info(f"Submitted job ID: {job.jobid}") + + try: + job.update() + except Exception as update_err: + logger.warning(f"Initial job update failed, continuing: {update_err}") + + time.sleep(60) + logger.info(f"Job {job.jobid} current state: {job.state}") + + job.complete() # Wait until the job completes. + logger.info("Transfer job completed successfully.") + return True + + except Exception as e: + logger.error(f"Error during job submission or completion: {e}") + match = re.search(r"Job not found:\s*(\d+)", str(e)) + if match: + jobid = match.group(1) + logger.info(f"Attempting to recover job {jobid}.") + try: + job = self.client.perlmutter.job(jobid=jobid) + time.sleep(30) + job.complete() + logger.info("Transfer job completed successfully after recovery.") + return True + except Exception as recovery_err: + logger.error(f"Failed to recover job {jobid}: {recovery_err}") + return False + else: + return False + + +class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): + """ + Use SFAPI, Slurm, hsi and htar to move data between HPSS and CFS at NERSC. + + This controller retrieves data from an HPSS source endpoint and places it on a CFS destination endpoint. + It supports the following modes: + - "single": Single file retrieval via hsi get. + - "tar": Full tar archive extraction via htar -xvf. + - "partial": Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), + only the specified files will be extracted. + + A single SLURM job script is generated that branches based on the mode. + """ + + def __init__( + self, + client: Client, + config: BeamlineConfig + ) -> None: + super().__init__(config) + self.client = client + + def copy( + self, + file_path: str = None, + source: HPSSEndpoint = None, + destination: FileSystemEndpoint = None, + files_to_extract: Optional[List[str]] = None, + ) -> bool: + """ + Copy a file from an HPSS source endpoint to a CFS destination endpoint. + + Args: + file_path (str): Path to the file or tar archive on HPSS. + source (HPSSEndpoint): The HPSS source endpoint. + destination (FileSystemEndpoint): The CFS destination endpoint. + files_to_extract (List[str], optional): Specific files to extract from the tar archive. + If provided (and file_path ends with '.tar'), only these files will be extracted. + If not provided, the entire tar archive will be extracted. + If file_path is a single file, this parameter is ignored. + + Returns: + bool: True if the transfer job completes successfully, False otherwise. + """ + logger.info("Starting HPSS to CFS transfer.") + if not file_path or not source or not destination: + logger.error("Missing required parameters: file_path, source, or destination.") + return False + + # Compute the full HPSS path from the source endpoint. + hpss_path = source.full_path(file_path) + dest_root = destination.root_path + + # Get the beamline_id from the configuration. + beamline_id = self.config.beamline_id + + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + + # If files_to_extract is provided, join them as a spaceā€separated string. + files_to_extract_str = " ".join(files_to_extract) if files_to_extract else "" + + # The following SLURM script contains all logic to decide the transfer mode. + # It determines: + # - if HPSS_PATH ends with .tar, then if FILES_TO_EXTRACT is nonempty, MODE becomes "partial", + # else MODE is "tar". + # - Otherwise, MODE is "single" and hsi get is used. + job_script = fr"""#!/bin/bash +#SBATCH -q xfer # Specify the SLURM queue to u +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/{file_path}_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{file_path}_from_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# ------------------------------------------------------------------- +# Define source and destination variables. +# ------------------------------------------------------------------- + +echo "[LOG] Defining source and destination paths." + +# SOURCE_PATH: Full path of the file or directory on HPSS. +SOURCE_PATH="{hpss_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" + +# DEST_ROOT: Root destination on CFS built from configuration. +DEST_ROOT="{dest_root}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" + +# FILES_TO_EXTRACT: Specific files to extract from the tar archive, if any. +# If not provided, this will be empty. +FILES_TO_EXTRACT="{files_to_extract_str}" +echo "[LOG] FILES_TO_EXTRACT set to: $FILES_TO_EXTRACT" + +# ------------------------------------------------------------------- +# Verify that SOURCE_PATH exists on HPSS using hsi ls. +# ------------------------------------------------------------------- + +echo "[LOG] Verifying file existence with hsi ls." +if ! hsi ls "$SOURCE_PATH" >/dev/null 2>&1; then + echo "[ERROR] File not found on HPSS: $SOURCE_PATH" + exit 1 +fi + +# ------------------------------------------------------------------- +# Determine the transfer mode based on the type (file vs tar). +# ------------------------------------------------------------------- + +echo "[LOG] Determining transfer mode based on the type (file vs tar)." + +# Check if SOURCE_PATH ends with .tar +if [[ "$SOURCE_PATH" =~ \.tar$ ]]; then + # If FILES_TO_EXTRACT is nonempty, MODE becomes "partial", else MODE is "tar". + if [ -n "${{FILES_TO_EXTRACT}}" ]; then + MODE="partial" + else + MODE="tar" + fi +else + MODE="single" +fi + +echo "Transfer mode: $MODE" + +# ------------------------------------------------------------------- +# Transfer Logic: Based on the mode, perform the appropriate transfer. +# ------------------------------------------------------------------- + +if [ "$MODE" = "single" ]; then + echo "[LOG] Single file detected. Using hsi get." + # mkdir -p "$DEST_ROOT" + # hsi get "$SOURCE_PATH" "$DEST_ROOT/" +elif [ "$MODE" = "tar" ]; then + echo "[LOG] Tar archive detected. Extracting entire archive using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + echo "[LOG] Extracting to: $DEST_PATH" + # mkdir -p "$DEST_PATH" + # htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" +elif [ "$MODE" = "partial" ]; then + echo "[LOG] Partial extraction detected. Extracting selected files using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + + # Verify that each requested file exists in the tar archive. + echo "[LOG] Verifying requested files are in the tar archive." + ARCHIVE_CONTENTS=$(htar -tvf "$SOURCE_PATH") + echo "[LOG] List: $ARCHIVE_CONTENTS" + for file in $FILES_TO_EXTRACT; do + echo "[LOG] Checking for file: $file" + if ! echo "$ARCHIVE_CONTENTS" | grep -q "$file"; then + echo "[ERROR] Requested file '$file' not found in archive $SOURCE_PATH" + exit 1 + else + echo "[LOG] File '$file' found in archive." + fi + done + + echo "[LOG] All requested files verified. Proceeding with extraction." + mkdir -p "$DEST_PATH" + (cd "$DEST_PATH" && htar -xvf "$SOURCE_PATH" -Hnostage $FILES_TO_EXTRACT) + + echo "[LOG] Extraction complete. Listing contents of $DEST_PATH:" + ls -l "$DEST_PATH" + +else + echo "[ERROR]: Unknown mode: $MODE" + exit 1 +fi + +date +""" + logger.info("Submitting HPSS to CFS transfer job to Perlmutter.") + try: + perlmutter = self.client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + logger.info(f"Submitted job ID: {job.jobid}") + + try: + job.update() + except Exception as update_err: + logger.warning(f"Initial job update failed, continuing: {update_err}") + + time.sleep(60) + logger.info(f"Job {job.jobid} current state: {job.state}") + + job.complete() # Wait until the job completes. + logger.info("HPSS to CFS transfer job completed successfully.") + return True + + except Exception as e: + logger.error(f"Error during job submission or completion: {e}") + match = re.search(r"Job not found:\s*(\d+)", str(e)) + if match: + jobid = match.group(1) + logger.info(f"Attempting to recover job {jobid}.") + try: + job = self.client.perlmutter.job(jobid=jobid) + time.sleep(30) + job.complete() + logger.info("HPSS to CFS transfer job completed successfully after recovery.") + return True + except Exception as recovery_err: + logger.error(f"Failed to recover job {jobid}: {recovery_err}") + return False + else: + return False + + if __name__ == "__main__": TEST_CFS_TO_HPSS = False TEST_HPSS_TO_CFS = True @@ -150,8 +867,8 @@ def hpss_to_cfs_flow( ) files_to_extract = [ - "/global/cfs/cdirs/als/data_mover/8.3.2/raw/ALS-11193_nbalsara/20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", - "/global/cfs/cdirs/als/data_mover/8.3.2/raw/ALS-11193_nbalsara/20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", + "ALS-11193_nbalsara/20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", + "ALS-11193_nbalsara/20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", ] hpss_to_cfs_flow( diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index c49a51e0..3a80ba26 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -11,7 +11,7 @@ from orchestration.config import BeamlineConfig from orchestration.globus.transfer import GlobusEndpoint, prune_one_safe from orchestration.prefect import schedule_prefect_flow -from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint, TransferEndpoint +from orchestration.transfer_endpoints import FileSystemEndpoint, TransferEndpoint logger = logging.getLogger(__name__) @@ -189,56 +189,6 @@ def _prune_globus_endpoint( ) -class HPSSPruneController(PruneController[HPSSEndpoint]): - def __init__( - self, - config: BeamlineConfig, - ) -> None: - super().__init__(config) - - def prune( - self, - file_path: str = None, - source_endpoint: HPSSEndpoint = None, - check_endpoint: FileSystemEndpoint = None, - days_from_now: datetime.timedelta = 0 - ) -> bool: - flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - schedule_prefect_flow( - "prune_hpss_endpoint/prune_hpss_endpoint", - flow_name, - { - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - - datetime.timedelta(days=days_from_now), - ) - return True - - @flow(name="prune_hpss_endpoint") - def _prune_hpss_endpoint( - relative_path: str, - source_endpoint: HPSSEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, - config: BeamlineConfig = None - ): - """ - Prune files from HPSS. - - Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. - check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. - """ - # TODO: Implement HPSS pruning - pass - - class PruneMethod(Enum): """ Enum representing different prune methods. @@ -267,7 +217,8 @@ def get_prune_controller( return GlobusPruneController(config) elif prune_type == PruneMethod.SIMPLE: return FileSystemPruneController(config) - elif prune_type == PruneMethod.CFS_TO_HPSS: + elif prune_type == PruneMethod.HPSS: + from orchestration.prune_controller import HPSSPruneController from orchestration.sfapi import create_sfapi_client return HPSSPruneController( client=create_sfapi_client(), diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 4728fd6b..b4e7757d 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -4,20 +4,16 @@ import datetime import logging import os -from pathlib import Path -import re import time -from typing import Generic, List, Optional, TypeVar +from typing import Generic, Optional, TypeVar import globus_sdk -from sfapi_client import Client -from sfapi_client.compute import Machine # Import the generic Beamline configuration class. from orchestration.config import BeamlineConfig from orchestration.globus.transfer import GlobusEndpoint, start_transfer from orchestration.prometheus_utils import PrometheusMetrics -from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint, TransferEndpoint +from orchestration.transfer_endpoints import FileSystemEndpoint, TransferEndpoint logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -332,564 +328,6 @@ def copy( logger.info(f"Transfer process took {elapsed_time:.2f} seconds.") -class CFSToHPSSTransferController(TransferController[HPSSEndpoint]): - """ - Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. - - This controller requires the source to be a FileSystemEndpoint on CFS and the - destination to be an HPSSEndpoint. For a single file, the transfer is done using hsi (via hsi cput). - For a directory, the transfer is performed with htar. In this updated version, if the source is a - directory then the files are bundled into tar archives based on their modification dates as follows: - - Files with modification dates between Jan 1 and Jul 15 (inclusive) are grouped together - (Cycle 1 for that year). - - Files with modification dates between Jul 16 and Dec 31 are grouped together (Cycle 2). - - Within each group, if the total size exceeds 2 TB the files are partitioned into multiple tar bundles. - The resulting naming convention on HPSS is: - - /home/a/alsdev/data_mover/[beamline]/raw/[proposal_name]/ - [proposal_name]_[year]-[cycle].tar - [proposal_name]_[year]-[cycle]_part0.tar - [proposal_name]_[year]-[cycle]_part1.tar - ... - - At the end of the SLURM script, the directory tree for both the source (CFS) and destination (HPSS) - is echoed for logging purposes. - """ - - def __init__( - self, - client: Client, - config: BeamlineConfig - ) -> None: - super().__init__(config) - self.client = client - - def copy( - self, - file_path: str = None, - source: FileSystemEndpoint = None, - destination: HPSSEndpoint = None - ) -> bool: - """ - Copy a file or directory from a CFS source endpoint to an HPSS destination endpoint. - - Args: - file_path (str): Path to the file or directory on CFS. - source (FileSystemEndpoint): The CFS source endpoint. - destination (HPSSEndpoint): The HPSS destination endpoint. - - Returns: - bool: True if the transfer job completes successfully, False otherwise. - """ - logger.info("Transferring data from CFS to HPSS") - if not file_path or not source or not destination: - logger.error("Missing required parameters for CFSToHPSSTransferController.") - return False - - # Compute the full path on CFS for the file/directory. - full_cfs_path = source.full_path(file_path) - # Get the beamline_id from the configuration. - beamline_id = self.config.beamline_id - # Build the HPSS destination root path using the convention: [destination.root_path]/[beamline_id]/raw - hpss_root_path = f"{destination.root_path.rstrip('/')}/{beamline_id}/raw" - - # Determine the proposal (project) folder name from the file_path. - path = Path(file_path) - proposal_name = path.parent.name - if not proposal_name or proposal_name == ".": # if file_path is in the root directory - proposal_name = file_path - - logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" - - # Build the SLURM job script with detailed inline comments for clarity. - job_script = rf"""#!/bin/bash -# ------------------------------------------------------------------ -# SLURM Job Script for Transferring Data from CFS to HPSS -# This script will: -# 1. Define the source (CFS) and destination (HPSS) paths. -# 2. Create the destination directory on HPSS if it doesn't exist. -# 3. Determine if the source is a file or a directory. -# - If a file, transfer it using 'hsi cput'. -# - If a directory, group files by beam cycle and archive them. -# * Cycle 1: Jan 1 - Jul 15 -# * Cycle 2: Jul 16 - Dec 31 -# * If a group exceeds 2 TB, it is partitioned into multiple tar archives. -# * Archive names: -# [proposal_name]_[year]-[cycle].tar -# [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. -# 4. Echo directory trees for both source and destination for logging. -# ------------------------------------------------------------------ - -#SBATCH -q xfer # Specify the SLURM queue to use. -#SBATCH -A als # Specify the account. -#SBATCH -C cron # Use the 'cron' constraint. -#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_to_HPSS_{proposal_name} # Set a descriptive job name. -#SBATCH --output={logs_path}/{proposal_name}_to_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{proposal_name}_to_hpss_%j.err # Standard error log file. -#SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. - -set -euo pipefail # Enable strict error checking. -echo "[LOG] Job started at: $(date)" - -# ------------------------------------------------------------------ -# Define source and destination variables. -# ------------------------------------------------------------------ -echo "[LOG] Defining source and destination paths." - -# SOURCE_PATH: Full path of the file or directory on CFS. -SOURCE_PATH="{full_cfs_path}" -echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" - -# DEST_ROOT: Root destination on HPSS built from configuration. -DEST_ROOT="{hpss_root_path}" -echo "[LOG] DEST_ROOT set to: $DEST_ROOT" - -# FOLDER_NAME: Proposal name (project folder) derived from the file path. -FOLDER_NAME="{proposal_name}" -echo "[LOG] FOLDER_NAME set to: $FOLDER_NAME" - -# DEST_PATH: Final HPSS destination directory. -DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" -echo "[LOG] DEST_PATH set to: $DEST_PATH" - -# ------------------------------------------------------------------ -# Create destination directory on HPSS recursively using hsi mkdir. -# This section ensures that the entire directory tree specified in DEST_PATH -# exists on HPSS. Since HPSS hsi does not support a recursive mkdir option, -# we split the path into its components and create each directory one by one. -# ------------------------------------------------------------------ - -echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." - -# Use 'hsi ls' to verify if the destination directory exists. -# The '-q' flag is used for quiet mode, and any output or errors are discarded. -if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then - echo "[LOG] Destination directory $DEST_PATH already exists." -else - # If the directory does not exist, begin the process to create it. - echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." - - # Initialize an empty variable 'current' that will store the path built so far. - current="" - - # Split the DEST_PATH using '/' as the delimiter. - # This creates an array 'parts' where each element is a directory level in the path. - IFS='/' read -ra parts <<< "$DEST_PATH" - - # Iterate over each directory component in the 'parts' array. - for part in "${{parts[@]}}"; do - # Skip any empty parts. An empty string may occur if the path starts with a '/'. - if [ -z "$part" ]; then - continue - fi - - # Append the current part to the 'current' path variable. - # This step incrementally reconstructs the full path one directory at a time. - current="$current/$part" - - # Check if the current directory exists on HPSS using 'hsi ls'. - if ! hsi -q "ls $current" >/dev/null 2>&1; then - # If the directory does not exist, attempt to create it using 'hsi mkdir'. - if hsi "mkdir $current" >/dev/null 2>&1; then - echo "[LOG] Created directory $current." - else - echo "[ERROR] Failed to create directory $current." - exit 1 - fi - else - echo "[LOG] Directory $current already exists." - fi - done -fi - -# List the final HPSS directory tree for logging purposes. -# For some reason this gets logged in the project.err file, not the .out file. -hsi ls $DEST_PATH - -# ------------------------------------------------------------------ -# Transfer Logic: Check if SOURCE_PATH is a file or directory. -# ------------------------------------------------------------------ -echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" -if [ -f "$SOURCE_PATH" ]; then - # Case: Single file detected. - echo "[LOG] Single file detected. Transferring via hsi cput." - FILE_NAME=$(basename "$SOURCE_PATH") - echo "[LOG] File name: $FILE_NAME" - hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" - echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." - -elif [ -d "$SOURCE_PATH" ]; then - # Case: Directory detected. - echo "[LOG] Directory detected. Initiating bundling process." - THRESHOLD=2199023255552 # 2 TB in bytes. - echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" - - # ------------------------------------------------------------------ - # Group files based on modification date. - # ------------------------------------------------------------------ - echo "[LOG] Grouping files by modification date." - FILE_LIST=$(mktemp) - find "$SOURCE_PATH" -type f > "$FILE_LIST" - echo "[LOG] List of files stored in temporary file: $FILE_LIST" - - # Declare associative arrays to hold grouped file paths and sizes. - declare -A group_files - declare -A group_sizes - - while IFS= read -r file; do - mtime=$(stat -c %Y "$file") - year=$(date -d @"$mtime" +%Y) - month=$(date -d @"$mtime" +%m | sed 's/^0*//') - day=$(date -d @"$mtime" +%d | sed 's/^0*//') - # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. - if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then - cycle=1 - else - cycle=2 - fi - key="${{year}}-${{cycle}}" - group_files["$key"]="${{group_files["$key"]:-}} $file" - fsize=$(stat -c %s "$file") - group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) - done < "$FILE_LIST" - rm "$FILE_LIST" - echo "[LOG] Completed grouping files." - - # Print the files in each group at the end - for key in "${{!group_files[@]}}"; do - echo "[LOG] Group $key contains files:" - for f in ${{group_files["$key"]}}; do - echo " $f" - done - done - - # ------------------------------------------------------------------ - # Bundle files into tar archives. - # ------------------------------------------------------------------ - for key in "${{!group_files[@]}}"; do - files=(${{group_files["$key"]}}) - total_group_size=${{group_sizes["$key"]}} - echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." - - part=0 - current_size=0 - current_files=() - for f in "${{files[@]}}"; do - fsize=$(stat -c %s "$f") - # If adding this file exceeds the threshold, process the current bundle. - if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Bundle reached threshold." - echo "[LOG] Files in current bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." - htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - part=$((part+1)) - echo "[DEBUG] Resetting bundle variables." - current_files=() - current_size=0 - fi - current_files+=("$f") - current_size=$(( current_size + fsize )) - done - if [ ${{#current_files[@]}} -gt 0 ]; then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Final bundle for group $key:" - echo "[LOG] Files in final bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." - echo "[LOG] Bundle size: $current_size bytes." - htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") - fi - echo "[LOG] Completed processing group $key." - done -else - echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." - exit 1 -fi - -# ------------------------------------------------------------------ -# Logging: Display directory trees for both source and destination. -# ------------------------------------------------------------------ -echo "[LOG] Listing Source (CFS) Tree:" -if [ -d "$SOURCE_PATH" ]; then - find "$SOURCE_PATH" -print -else - echo "[LOG] $SOURCE_PATH is a file." -fi - -echo "[LOG] Listing Destination (HPSS) Tree:" -hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" - -echo "[LOG] Job completed at: $(date)" -""" - try: - logger.info("Submitting HPSS transfer job to Perlmutter.") - perlmutter = self.client.compute(Machine.perlmutter) - job = perlmutter.submit_job(job_script) - logger.info(f"Submitted job ID: {job.jobid}") - - try: - job.update() - except Exception as update_err: - logger.warning(f"Initial job update failed, continuing: {update_err}") - - time.sleep(60) - logger.info(f"Job {job.jobid} current state: {job.state}") - - job.complete() # Wait until the job completes. - logger.info("Transfer job completed successfully.") - return True - - except Exception as e: - logger.error(f"Error during job submission or completion: {e}") - match = re.search(r"Job not found:\s*(\d+)", str(e)) - if match: - jobid = match.group(1) - logger.info(f"Attempting to recover job {jobid}.") - try: - job = self.client.perlmutter.job(jobid=jobid) - time.sleep(30) - job.complete() - logger.info("Transfer job completed successfully after recovery.") - return True - except Exception as recovery_err: - logger.error(f"Failed to recover job {jobid}: {recovery_err}") - return False - else: - return False - - -class HPSSToCFSTransferController(TransferController[HPSSEndpoint]): - """ - Use SFAPI, Slurm, hsi and htar to move data between HPSS and CFS at NERSC. - - This controller retrieves data from an HPSS source endpoint and places it on a CFS destination endpoint. - It supports the following modes: - - "single": Single file retrieval via hsi get. - - "tar": Full tar archive extraction via htar -xvf. - - "partial": Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), - only the specified files will be extracted. - - A single SLURM job script is generated that branches based on the mode. - """ - - def __init__( - self, - client: Client, - config: BeamlineConfig - ) -> None: - super().__init__(config) - self.client = client - - def copy( - self, - file_path: str = None, - source: HPSSEndpoint = None, - destination: FileSystemEndpoint = None, - files_to_extract: Optional[List[str]] = None, - ) -> bool: - """ - Copy a file from an HPSS source endpoint to a CFS destination endpoint. - - Args: - file_path (str): Path to the file or tar archive on HPSS. - source (HPSSEndpoint): The HPSS source endpoint. - destination (FileSystemEndpoint): The CFS destination endpoint. - files_to_extract (List[str], optional): Specific files to extract from the tar archive. - If provided (and file_path ends with '.tar'), only these files will be extracted. - If not provided, the entire tar archive will be extracted. - If file_path is a single file, this parameter is ignored. - - Returns: - bool: True if the transfer job completes successfully, False otherwise. - """ - logger.info("Starting HPSS to CFS transfer.") - if not file_path or not source or not destination: - logger.error("Missing required parameters: file_path, source, or destination.") - return False - - # Compute the full HPSS path from the source endpoint. - hpss_path = source.full_path(file_path) - dest_root = destination.root_path - - # Get the beamline_id from the configuration. - beamline_id = self.config.beamline_id - - logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" - - # If files_to_extract is provided, join them as a spaceā€separated string. - files_to_extract_str = " ".join(files_to_extract) if files_to_extract else "" - - # The following SLURM script contains all logic to decide the transfer mode. - # It determines: - # - if HPSS_PATH ends with .tar, then if FILES_TO_EXTRACT is nonempty, MODE becomes "partial", - # else MODE is "tar". - # - Otherwise, MODE is "single" and hsi get is used. - job_script = fr"""#!/bin/bash -#SBATCH -q xfer # Specify the SLURM queue to u -#SBATCH -A als # Specify the account. -#SBATCH -C cron # Use the 'cron' constraint. -#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/{file_path}_from_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{file_path}_from_hpss_%j.err # Standard error log file. -#SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. -set -euo pipefail # Enable strict error checking. -echo "[LOG] Job started at: $(date)" - -# ------------------------------------------------------------------- -# Define source and destination variables. -# ------------------------------------------------------------------- - -echo "[LOG] Defining source and destination paths." - -# SOURCE_PATH: Full path of the file or directory on HPSS. -SOURCE_PATH="{hpss_path}" -echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" - -# DEST_ROOT: Root destination on CFS built from configuration. -DEST_ROOT="{dest_root}" -echo "[LOG] DEST_ROOT set to: $DEST_ROOT" - -# FILES_TO_EXTRACT: Specific files to extract from the tar archive, if any. -# If not provided, this will be empty. -FILES_TO_EXTRACT="{files_to_extract_str}" -echo "[LOG] FILES_TO_EXTRACT set to: $FILES_TO_EXTRACT" - -# ------------------------------------------------------------------- -# Verify that SOURCE_PATH exists on HPSS using hsi ls. -# ------------------------------------------------------------------- - -echo "[LOG] Verifying file existence with hsi ls." -if ! hsi ls "$SOURCE_PATH" >/dev/null 2>&1; then - echo "[ERROR] File not found on HPSS: $SOURCE_PATH" - exit 1 -fi - -# ------------------------------------------------------------------- -# Determine the transfer mode based on the type (file vs tar). -# ------------------------------------------------------------------- - -echo "[LOG] Determining transfer mode based on the type (file vs tar)." - -# Check if SOURCE_PATH ends with .tar -if [[ "$SOURCE_PATH" =~ \.tar$ ]]; then - # If FILES_TO_EXTRACT is nonempty, MODE becomes "partial", else MODE is "tar". - if [ -n "${{FILES_TO_EXTRACT}}" ]; then - MODE="partial" - else - MODE="tar" - fi -else - MODE="single" -fi - -echo "Transfer mode: $MODE" - -# ------------------------------------------------------------------- -# Transfer Logic: Based on the mode, perform the appropriate transfer. -# ------------------------------------------------------------------- - -if [ "$MODE" = "single" ]; then - echo "[LOG] Single file detected. Using hsi get." - # mkdir -p "$DEST_ROOT" - # hsi get "$SOURCE_PATH" "$DEST_ROOT/" -elif [ "$MODE" = "tar" ]; then - echo "[LOG] Tar archive detected. Extracting entire archive using htar." - ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") - ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" - DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" - echo "[LOG] Extracting to: $DEST_PATH" - # mkdir -p "$DEST_PATH" - # htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" -elif [ "$MODE" = "partial" ]; then - echo "[LOG] Partial extraction detected. Extracting selected files using htar." - ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") - ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" - DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" - - # Verify that each requested file exists in the tar archive. - echo "[LOG] Verifying requested files are in the tar archive." - ARCHIVE_CONTENTS=$(htar -tvf "$SOURCE_PATH") - echo "[LOG] List: $ARCHIVE_CONTENTS" - for file in $FILES_TO_EXTRACT; do - echo "[LOG] Checking for file: $file" - if ! echo "$ARCHIVE_CONTENTS" | grep -q "$file"; then - echo "[ERROR] Requested file '$file' not found in archive $SOURCE_PATH" - exit 1 - else - echo "[LOG] File '$file' found in archive." - fi - done - - echo "[LOG] All requested files verified. Proceeding with extraction." - mkdir -p "$DEST_PATH" - (cd "$DEST_PATH" && htar -xvf "$SOURCE_PATH" -Hnostage $FILES_TO_EXTRACT) - - echo "[LOG] Extraction complete. Listing contents of $DEST_PATH:" - ls -l "$DEST_PATH" - -else - echo "[ERROR]: Unknown mode: $MODE" - exit 1 -fi - -date -""" - logger.info("Submitting HPSS to CFS transfer job to Perlmutter.") - try: - perlmutter = self.client.compute(Machine.perlmutter) - job = perlmutter.submit_job(job_script) - logger.info(f"Submitted job ID: {job.jobid}") - - try: - job.update() - except Exception as update_err: - logger.warning(f"Initial job update failed, continuing: {update_err}") - - time.sleep(60) - logger.info(f"Job {job.jobid} current state: {job.state}") - - job.complete() # Wait until the job completes. - logger.info("HPSS to CFS transfer job completed successfully.") - return True - - except Exception as e: - logger.error(f"Error during job submission or completion: {e}") - match = re.search(r"Job not found:\s*(\d+)", str(e)) - if match: - jobid = match.group(1) - logger.info(f"Attempting to recover job {jobid}.") - try: - job = self.client.perlmutter.job(jobid=jobid) - time.sleep(30) - job.complete() - logger.info("HPSS to CFS transfer job completed successfully after recovery.") - return True - except Exception as recovery_err: - logger.error(f"Failed to recover job {jobid}: {recovery_err}") - return False - else: - return False - - class CopyMethod(Enum): """ Enum representing different transfer methods. @@ -921,12 +359,14 @@ def get_transfer_controller( elif transfer_type == CopyMethod.SIMPLE: return SimpleTransferController(config) elif transfer_type == CopyMethod.CFS_TO_HPSS: + from orchestration.hpss import CFSToHPSSTransferController from orchestration.sfapi import create_sfapi_client return CFSToHPSSTransferController( client=create_sfapi_client(), config=config ) elif transfer_type == CopyMethod.HPSS_TO_CFS: + from orchestration.hpss import HPSSToCFSTransferController from orchestration.sfapi import create_sfapi_client return HPSSToCFSTransferController( client=create_sfapi_client(), From 4e9215c9de9e85af6445a4f094da92709a8b4c03 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 27 Feb 2025 15:48:40 -0800 Subject: [PATCH 029/128] Adjusting hpss imports in the transfer_controller pytest --- orchestration/_tests/test_transfer_controller.py | 6 ++++-- orchestration/hpss.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index f47be9be..f7ffd7e9 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -54,11 +54,13 @@ def transfer_controller_module(): FileSystemEndpoint, GlobusTransferController, SimpleTransferController, + get_transfer_controller, + CopyMethod, + ) + from orchestration.hpss import ( CFSToHPSSTransferController, HPSSToCFSTransferController, HPSSEndpoint, - get_transfer_controller, - CopyMethod, ) return { "FileSystemEndpoint": FileSystemEndpoint, diff --git a/orchestration/hpss.py b/orchestration/hpss.py index d2c0804a..dc1d23a4 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -120,6 +120,12 @@ def hpss_to_cfs_flow( # ---------------------------------- class HPSSPruneController(PruneController[HPSSEndpoint]): + """ + Use SFAPI, Slurm, and hsi to prune data from HPSS at NERSC. + This controller requires the source to be an HPSSEndpoint and the + optional destination to be a FileSystemEndpoint. It uses "hsi rm" to prune + files from HPSS. + """ def __init__( self, client: Client, From 58eeec5e329ba3a5d0956ead1bfb0b0618a1e10e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 27 Feb 2025 16:27:35 -0800 Subject: [PATCH 030/128] Verified that the HPSSPruneController successfully pruned from HPSS --- orchestration/hpss.py | 82 ++++++++++++++++++++-------- orchestration/prune_controller.py | 2 +- orchestration/transfer_controller.py | 2 +- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index dc1d23a4..76e5b3d2 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -14,7 +14,7 @@ from orchestration.config import BeamlineConfig from orchestration.prefect import schedule_prefect_flow -from orchestration.prune_controller import PruneController +from orchestration.prune_controller import get_prune_controller, PruneController, PruneMethod from orchestration.transfer_controller import get_transfer_controller, CopyMethod, TransferController from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint @@ -138,24 +138,31 @@ def prune( self, file_path: str = None, source_endpoint: HPSSEndpoint = None, - check_endpoint: FileSystemEndpoint = None, + check_endpoint: Optional[FileSystemEndpoint] = None, days_from_now: datetime.timedelta = 0 ) -> bool: flow_name = f"prune_from_{source_endpoint.name}" logger.info(f"Running flow: {flow_name}") logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - schedule_prefect_flow( - "prune_hpss_endpoint/prune_hpss_endpoint", - flow_name, - { - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - - datetime.timedelta(days=days_from_now), + + self._prune_hpss_endpoint( + self, + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, ) + # Uncomment the following lines to schedule the flow with Prefect. + # schedule_prefect_flow( + # deployment_name="prune_hpss_endpoint/prune_hpss_endpoint", + # flow_run_name=flow_name, + # parameters={ + # "relative_path": file_path, + # "source_endpoint": source_endpoint, + # "check_endpoint": check_endpoint, + # "config": self.config + # }, + # duration_from_now=days_from_now + # ) return True @flow(name="prune_hpss_endpoint") @@ -163,19 +170,20 @@ def _prune_hpss_endpoint( self, relative_path: str, source_endpoint: HPSSEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, - config: BeamlineConfig = None + check_endpoint: Optional[Union[FileSystemEndpoint, None]] = None, ) -> None: """ Prune files from HPSS. Args: - relative_path (str): The path of the file or directory to prune. + relative_path (str): The HPSS path of the file or directory to prune. source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. """ + logger.info("Pruning files from HPSS") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") - beamline_id = config.beamline_id + beamline_id = self.config.beamline_id logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" job_script = rf"""#!/bin/bash @@ -197,14 +205,17 @@ def _prune_hpss_endpoint( echo "[LOG] Job started at: $(date)" # Check if the file exists on HPSS -if hsi -c "ls {source_endpoint.full_path(relative_path)}" &> /dev/null; then +if hsi "ls {source_endpoint.full_path(relative_path)}" &> /dev/null; then echo "[LOG] File {relative_path} exists on HPSS. Proceeding to prune." # Prune the file from HPSS - hsi -c "rm {source_endpoint.full_path(relative_path)}" + hsi "rm {source_endpoint.full_path(relative_path)}" echo "[LOG] File {relative_path} has been pruned from HPSS." + hsi ls -R {source_endpoint.full_path(relative_path)} +else + echo "[LOG] Could not find File {relative_path} does not on HPSS. Check your file path again." + exit 0 fi echo "[LOG] Job completed at: $(date)" - """ try: logger.info("Submitting HPSS transfer job to Perlmutter.") @@ -284,7 +295,8 @@ def copy( self, file_path: str = None, source: FileSystemEndpoint = None, - destination: HPSSEndpoint = None + destination: HPSSEndpoint = None, + days_from_now: datetime.timedelta = 0 ) -> bool: """ Copy a file or directory from a CFS source endpoint to an HPSS destination endpoint. @@ -827,9 +839,35 @@ def copy( if __name__ == "__main__": + TEST_HPSS_PRUNE = True TEST_CFS_TO_HPSS = False - TEST_HPSS_TO_CFS = True + TEST_HPSS_TO_CFS = False + # ------------------------------------------------------ + # Test pruning from HPSS + # ------------------------------------------------------ + if TEST_HPSS_PRUNE: + from orchestration.flows.bl832.config import Config832 + config = Config832() + file_name = "8.3.2/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" + source = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + + days_from_now = datetime.timedelta(days=0) # Prune immediately + + prune_controller = get_prune_controller( + prune_type=PruneMethod.HPSS, + config=config + ) + prune_controller.prune( + file_path=f"{file_name}", + source_endpoint=source, + check_endpoint=None, + days_from_now=days_from_now + ) # ------------------------------------------------------ # Test transfer from CFS to HPSS # ------------------------------------------------------ diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 3a80ba26..9b1d72ea 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -218,7 +218,7 @@ def get_prune_controller( elif prune_type == PruneMethod.SIMPLE: return FileSystemPruneController(config) elif prune_type == PruneMethod.HPSS: - from orchestration.prune_controller import HPSSPruneController + from orchestration.hpss import HPSSPruneController from orchestration.sfapi import create_sfapi_client return HPSSPruneController( client=create_sfapi_client(), diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index b4e7757d..09970398 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -349,7 +349,7 @@ def get_transfer_controller( Args: transfer_type (str): The type of transfer to perform. - config (Config832): The configuration object. + config (BeamlineConfig): The configuration object. Returns: TransferController: The transfer controller object. From 33fce11cc907f3426eeac387e3aec5ac65d96377 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 27 Feb 2025 17:00:45 -0800 Subject: [PATCH 031/128] Simplified file paths within the .tar archives on HPSS so it references only the file name, not the entire cfs path --- orchestration/hpss.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 76e5b3d2..bd8bcd26 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -447,7 +447,6 @@ def copy( echo "[LOG] File name: $FILE_NAME" hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." - elif [ -d "$SOURCE_PATH" ]; then # Case: Directory detected. echo "[LOG] Directory detected. Initiating bundling process." @@ -487,6 +486,7 @@ def copy( # Group files by modification date. # ------------------------------------------------------------------ + cd "$SOURCE_PATH" && \ while IFS= read -r file; do mtime=$(stat -c %Y "$file") year=$(date -d @"$mtime" +%Y) @@ -540,7 +540,7 @@ def copy( echo "$file" done echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." - (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) part=$((part+1)) echo "[DEBUG] Resetting bundle variables." current_files=() @@ -562,7 +562,7 @@ def copy( done echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." echo "[LOG] Bundle size: $current_size bytes." - (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}") + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) fi echo "[LOG] Completed processing group $key." done @@ -839,7 +839,7 @@ def copy( if __name__ == "__main__": - TEST_HPSS_PRUNE = True + TEST_HPSS_PRUNE = False TEST_CFS_TO_HPSS = False TEST_HPSS_TO_CFS = False @@ -911,8 +911,8 @@ def copy( ) files_to_extract = [ - "ALS-11193_nbalsara/20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", - "ALS-11193_nbalsara/20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", + "20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", + "20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", ] hpss_to_cfs_flow( From 09e340386858297dc7f4b33396475f51d101fb45 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 4 Mar 2025 14:27:37 -0800 Subject: [PATCH 032/128] Updated the HPSS flows in bl832/dispatcher.py to include updating the filepath and filesystem location in SciCat. To be tested... --- orchestration/flows/bl832/dispatcher.py | 41 +++++++++++++++++++++++++ orchestration/hpss.py | 37 +++++++++++----------- 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index afdb8c8e..1867519a 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -10,6 +10,7 @@ from orchestration.flows.bl832.move import process_new_832_file_task from orchestration.flows.bl832.config import Config832 +from orchestration.flows.bl832.scicat_ingestor import TomographyIngestorController # ------------------------------------------------------------------------------------------------------------------------ @@ -215,6 +216,7 @@ def archive_832_project_dispatcher( } ) logger.info(f"Scheduled tape transfer for project: {fp}") + except Exception as e: logger.error(f"Error scheduling transfer for {fp}: {e}") @@ -301,8 +303,27 @@ def archive_832_projects_from_previous_cycle_dispatcher( "config": config } ) + except Exception as e: logger.error(f"Error archiving project {project_name}: {e}") + try: + # Ingest the project into SciCat. + logger.info("Ingesting new file path into SciCat...") + ingestor = TomographyIngestorController( + config=config, + scicat_client=config.scicat + ) + scicat_id = ingestor._find_dataset( + proposal_id="proposal_id", + file_name="file_name" + ) + ingestor.add_new_dataset_location( + dataset_id=scicat_id, + source_folder="source_folder", + source_folder_host="source_folder_host" + ) + except Exception as e: + logger.error(f"Error ingesting project {project_name} into SciCat: {e}") else: logger.info(f"Project {project_name} last modified at {last_mod} is outside the archive window.") @@ -339,6 +360,26 @@ def archive_all_832_projects_dispatcher( "config": config } ) + + try: + # Ingest the project into SciCat. + logger.info("Ingesting new file path into SciCat...") + ingestor = TomographyIngestorController( + config=config, + scicat_client=config.scicat + ) + scicat_id = ingestor._find_dataset( + proposal_id="proposal_id", + file_name="file_name" + ) + ingestor.add_new_dataset_location( + dataset_id=scicat_id, + source_folder="source_folder", + source_folder_host="source_folder_host" + ) + except Exception as e: + logger.error(f"Error ingesting project {project} into SciCat: {e}") + except Exception as e: logger.error(e) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index bd8bcd26..b4e21504 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -145,24 +145,25 @@ def prune( logger.info(f"Running flow: {flow_name}") logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - self._prune_hpss_endpoint( - self, - relative_path=file_path, - source_endpoint=source_endpoint, - check_endpoint=check_endpoint, - ) - # Uncomment the following lines to schedule the flow with Prefect. - # schedule_prefect_flow( - # deployment_name="prune_hpss_endpoint/prune_hpss_endpoint", - # flow_run_name=flow_name, - # parameters={ - # "relative_path": file_path, - # "source_endpoint": source_endpoint, - # "check_endpoint": check_endpoint, - # "config": self.config - # }, - # duration_from_now=days_from_now - # ) + if days_from_now == 0: + self._prune_hpss_endpoint( + self, + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + ) + else: + schedule_prefect_flow( + deployment_name="prune_hpss_endpoint/prune_hpss_endpoint", + flow_run_name=flow_name, + parameters={ + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + duration_from_now=days_from_now + ) return True @flow(name="prune_hpss_endpoint") From abe84feb8be258a69621ae23f034320cca685778 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 4 Mar 2025 16:24:19 -0800 Subject: [PATCH 033/128] Updated docstrings, logging, error handling --- orchestration/transfer_controller.py | 149 ++++++++++++++++++++------- 1 file changed, 110 insertions(+), 39 deletions(-) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 09970398..a019c7ac 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -25,16 +25,20 @@ class TransferController(Generic[Endpoint], ABC): """ - Abstract class for transferring data. + Abstract base class for transferring data between endpoints. + + This class defines the common interface that all transfer controllers must implement, + regardless of the specific transfer mechanism they use. Args: - ABC: Abstract Base Class + config (BeamlineConfig): Configuration object containing endpoints and credentials """ def __init__( self, config: BeamlineConfig ) -> None: self.config = config + logger.debug(f"Initialized {self.__class__.__name__} with config for beamline {config.beamline_id}") @abstractmethod def copy( @@ -47,9 +51,9 @@ def copy( Copy a file from a source endpoint to a destination endpoint. Args: - file_path (str): The path of the file to copy. - source (Endpoint): The source endpoint. - destination (Endpoint): The destination endpoint. + file_path (str): The path of the file to copy, relative to the endpoint's root path + source (Endpoint): The source endpoint from which to copy the file + destination (Endpoint): The destination endpoint to which to copy the file Returns: bool: True if the transfer was successful, False otherwise. @@ -58,6 +62,16 @@ def copy( class GlobusTransferController(TransferController[GlobusEndpoint]): + """ + Use Globus Transfer to move data between Globus endpoints. + + This controller handles the transfer of files between Globus endpoints using the + Globus Transfer API. It manages authentication, transfer submissions, and status tracking. + + Args: + config (BeamlineConfig): Configuration object containing Globus endpoints and credentials + """ + def __init__( self, config: BeamlineConfig, @@ -65,7 +79,7 @@ def __init__( ) -> None: super().__init__(config) self.prometheus_metrics = prometheus_metrics - + logger.debug(f"Initialized GlobusTransferController for beamline {config.beamline_id}") """ Use Globus Transfer to move data between endpoints. @@ -179,16 +193,29 @@ def copy( destination: GlobusEndpoint = None, ) -> bool: """ - Copy a file from a source endpoint to a destination endpoint. + Copy a file from a source Globus endpoint to a destination Globus endpoint. + + This method handles the full transfer process, including path normalization, + submission to the Globus Transfer API, and waiting for completion or error. Args: - file_path (str): The path of the file to copy. - source (GlobusEndpoint): The source endpoint. - destination (GlobusEndpoint): The destination endpoint. + file_path (str): The path of the file to copy, relative to the endpoint's root path + source (GlobusEndpoint): The source Globus endpoint from which to copy the file + destination (GlobusEndpoint): The destination Globus endpoint to which to copy the file Returns: - bool: True if the transfer was successful, False otherwise. + bool: True if the transfer was successful, False otherwise + + Raises: + globus_sdk.services.transfer.errors.TransferAPIError: If there are issues with the Globus API """ + if not file_path: + logger.error("No file path provided for transfer") + return False + + if not source or not destination: + logger.error("Missing source or destination endpoint for transfer") + return False if not file_path: logger.error("No file_path provided") @@ -200,10 +227,12 @@ def copy( logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") - # Remove leading slash if present + # Normalize the file path by removing leading slashes if present if file_path[0] == "/": file_path = file_path[1:] + logger.debug(f"Normalized file path to '{file_path}'") + # Build full paths for source and destination source_path = os.path.join(source.root_path, file_path) dest_path = os.path.join(destination.root_path, file_path) logger.info(f"Transferring {source_path} to {dest_path}") @@ -215,6 +244,7 @@ def copy( file_size = 0 # Initialize file_size here as well try: + logger.info(f"Submitting Globus transfer task from {source.uuid} to {destination.uuid}") success, task_id = start_transfer( transfer_client=self.config.tc, source_endpoint=source, @@ -231,7 +261,13 @@ def copy( logger.error("Transfer failed.") except globus_sdk.services.transfer.errors.TransferAPIError as e: - logger.error(f"Failed to submit transfer: {e}") + logger.error(f"Globus Transfer API error: {e}") + logger.error(f"Status code: {e.status_code if hasattr(e, 'status_code') else 'unknown'}") + logger.error(f"Error details: {e.data if hasattr(e, 'data') else e}") + return False + except Exception as e: + logger.error(f"Unexpected error during transfer: {str(e)}", exc_info=True) + return False finally: # Stop the timer and calculate the duration @@ -263,17 +299,21 @@ def copy( class SimpleTransferController(TransferController[FileSystemEndpoint]): - def __init__( - self, - config: BeamlineConfig - ) -> None: - super().__init__(config) """ Use a simple 'cp' command to move data within the same system. + This controller is suitable for transfers between directories on the same + file system, where network transfer protocols are not needed. + Args: - TransferController: Abstract class for transferring data. + config (BeamlineConfig): Configuration object containing file system paths """ + def __init__( + self, + config: BeamlineConfig + ) -> None: + super().__init__(config) + logger.debug(f"Initialized SimpleTransferController for beamline {config.beamline_id}") def copy( self, @@ -282,28 +322,34 @@ def copy( destination: FileSystemEndpoint = None, ) -> bool: """ - Copy a file from a source endpoint to a destination endpoint using the 'cp' command. + Copy a file from a source directory to a destination directory using the 'cp' command. + + This method handles local file copying through the system's cp command, + including path normalization and status tracking. Args: - file_path (str): The path of the file to copy. - source (FileSystemEndpoint): The source endpoint. - destination (FileSystemEndpoint): The destination endpoint. + file_path (str): The path of the file to copy, relative to the endpoint's root path + source (FileSystemEndpoint): The source file system location + destination (FileSystemEndpoint): The destination file system location Returns: bool: True if the transfer was successful, False otherwise. """ if not file_path: - logger.error("No file_path provided.") + logger.error("No file_path provided for local copy operation") return False if not source or not destination: - logger.error("Source or destination endpoint not provided.") + logger.error("Source or destination endpoint not provided for local copy operation") return False logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") + # Normalize file path by removing leading slash if present if file_path.startswith("/"): file_path = file_path[1:] + logger.debug(f"Normalized file path to '{file_path}'") + # Build full paths for source and destination source_path = os.path.join(source.root_path, file_path) dest_path = os.path.join(destination.root_path, file_path) logger.info(f"Transferring {source_path} to {dest_path}") @@ -312,31 +358,45 @@ def copy( start_time = time.time() try: + # Check if source file/directory exists + if not os.path.exists(source_path): + logger.error(f"Source path does not exist: {source_path}") + return False + + # Ensure destination directory exists + dest_dir = os.path.dirname(dest_path) + if not os.path.exists(dest_dir): + logger.debug(f"Creating destination directory: {dest_dir}") + os.makedirs(dest_dir, exist_ok=True) + + # Execute the cp command result = os.system(f"cp -r '{source_path}' '{dest_path}'") if result == 0: - logger.info("Transfer completed successfully.") + logger.info(f"Local copy of '{file_path}' completed successfully") return True else: - logger.error(f"Transfer failed with exit code {result}.") + logger.error(f"Local copy of '{file_path}' failed with exit code {result}") return False except Exception as e: - logger.error(f"Transfer failed: {e}") + logger.error(f"Unexpected error during local copy: {str(e)}", exc_info=True) return False finally: # Stop the timer and calculate the duration elapsed_time = time.time() - start_time - logger.info(f"Transfer process took {elapsed_time:.2f} seconds.") + logger.info(f"Local copy process took {elapsed_time:.2f} seconds") class CopyMethod(Enum): """ Enum representing different transfer methods. - Use enum names as strings to identify transfer methods, ensuring a standard set of values. + + These values are used to select the appropriate transfer controller + through the factory function get_transfer_controller(). """ - GLOBUS = "globus" - SIMPLE = "simple" - CFS_TO_HPSS = "cfs_to_hpss" - HPSS_TO_CFS = "hpss_to_cfs" + GLOBUS = "globus" # Transfer between Globus endpoints + SIMPLE = "simple" # Local filesystem copy + CFS_TO_HPSS = "cfs_to_hpss" # NERSC CFS to HPSS tape archive + HPSS_TO_CFS = "hpss_to_cfs" # HPSS tape archive to NERSC CFS def get_transfer_controller( @@ -345,20 +405,28 @@ def get_transfer_controller( prometheus_metrics: Optional[PrometheusMetrics] = None ) -> TransferController: """ - Get the appropriate transfer controller based on the transfer type. + Factory function to get the appropriate transfer controller based on the transfer type. Args: - transfer_type (str): The type of transfer to perform. - config (BeamlineConfig): The configuration object. + transfer_type (CopyMethod): The type of transfer to perform + config (BeamlineConfig): The configuration object containing endpoint information Returns: - TransferController: The transfer controller object. + TransferController: The appropriate transfer controller instance + + Raises: + ValueError: If an invalid transfer type is provided """ + logger.debug(f"Creating transfer controller of type: {transfer_type.name}") + if transfer_type == CopyMethod.GLOBUS: + logger.debug("Returning GlobusTransferController") return GlobusTransferController(config, prometheus_metrics) elif transfer_type == CopyMethod.SIMPLE: + logger.debug("Returning SimpleTransferController") return SimpleTransferController(config) elif transfer_type == CopyMethod.CFS_TO_HPSS: + logger.debug("Importing and returning CFSToHPSSTransferController") from orchestration.hpss import CFSToHPSSTransferController from orchestration.sfapi import create_sfapi_client return CFSToHPSSTransferController( @@ -366,6 +434,7 @@ def get_transfer_controller( config=config ) elif transfer_type == CopyMethod.HPSS_TO_CFS: + logger.debug("Importing and returning HPSSToCFSTransferController") from orchestration.hpss import HPSSToCFSTransferController from orchestration.sfapi import create_sfapi_client return HPSSToCFSTransferController( @@ -373,7 +442,9 @@ def get_transfer_controller( config=config ) else: - raise ValueError(f"Invalid transfer type: {transfer_type}") + error_msg = f"Invalid transfer type: {transfer_type}" + logger.error(error_msg) + raise ValueError(error_msg) if __name__ == "__main__": From 0fb0f3e4d204b2952f090a1a33c91bb3b36e60d6 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 4 Mar 2025 16:49:40 -0800 Subject: [PATCH 034/128] Updated docstrings, logging, typing, error handling --- orchestration/prune_controller.py | 281 +++++++++++++++++++++++------- 1 file changed, 218 insertions(+), 63 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 9b1d72ea..180866da 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -3,7 +3,7 @@ from enum import Enum import logging import os -from typing import Generic, TypeVar, Union +from typing import Generic, Optional, TypeVar from prefect import flow from prefect.blocks.system import JSON @@ -21,82 +21,160 @@ class PruneController(Generic[Endpoint], ABC): """ - Abstract class for pruning controllers. - Provides interface methods for pruning data. + Abstract base class for pruning controllers. + + This class defines the common interface that all prune controllers must implement, + regardless of the specific pruning mechanism they use. + + Args: + config (BeamlineConfig): Configuration object containing endpoints and credentials """ def __init__( self, config: BeamlineConfig, ) -> None: + """ + Initialize the prune controller with configuration. + + Args: + config (BeamlineConfig): Configuration object containing endpoints and credentials + """ self.config = config + logger.debug(f"Initialized {self.__class__.__name__} with config for beamline {config.beamline_id}") @abstractmethod def prune( self, file_path: str = None, source_endpoint: Endpoint = None, - check_endpoint: Endpoint = None, + check_endpoint: Optional[Endpoint] = None, days_from_now: datetime.timedelta = 0 ) -> bool: - """Prune data from the source endpoint. + """ + Prune (delete) data from the source endpoint. + + This method either executes the pruning immediately or schedules it for future execution, + depending on the days_from_now parameter. Args: - file_path (str): The path to the file to prune. - source (Endpoint): The source endpoint. - destination (Endpoint): The destination endpoint. - days_from_now (datetime.timedelta): The number of days from now to prune. Defaults to 0. + file_path (str): The path to the file or directory to prune + source_endpoint (Endpoint): The endpoint containing the data to be pruned + check_endpoint (Optional[Endpoint]): If provided, verify data exists here before pruning + days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately Returns: - bool: True if successful, False otherwise. + bool: True if pruning was successful or scheduled successfully, False otherwise """ pass class FileSystemPruneController(PruneController[FileSystemEndpoint]): + """ + Controller for pruning files from local file systems. + + This controller handles pruning operations on local or mounted file systems + using standard file system operations. + + Args: + config (BeamlineConfig): Configuration object containing file system paths + """ def __init__( self, - config + config: BeamlineConfig ) -> None: + """ + Initialize the file system prune controller. + + Args: + config (BeamlineConfig): Configuration object containing file system paths + """ super().__init__(config) + logger.debug(f"Initialized FileSystemPruneController for beamline {config.beamline_id}") def prune( self, file_path: str = None, source_endpoint: FileSystemEndpoint = None, - check_endpoint: FileSystemEndpoint = None, + check_endpoint: Optional[FileSystemEndpoint] = None, days_from_now: datetime.timedelta = 0 ) -> bool: - flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - schedule_prefect_flow( - "prune_filesystem_endpoint/prune_filesystem_endpoint", - flow_name, - { - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - - datetime.timedelta(days=days_from_now), - ) - return True + """ + Prune (delete) data from a file system endpoint. + + If days_from_now is 0, executes pruning immediately. + Otherwise, schedules pruning for future execution using Prefect. + Args: + file_path (str): The path to the file or directory to prune + source_endpoint (FileSystemEndpoint): The file system endpoint containing the data + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning + days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + + Returns: + bool: True if pruning was successful or scheduled successfully, False otherwise + """ + if not file_path: + logger.error("No file_path provided for pruning operation") + return False + + if not source_endpoint: + logger.error("No source_endpoint provided for pruning operation") + return False + + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Setting up pruning of '{file_path}' from '{source_endpoint.name}'") + + # If days_from_now is 0, prune immediately + if days_from_now.total_seconds() == 0: + logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") + return self._prune_filesystem_endpoint( + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + config=self.config + ) + else: + # Otherwise, schedule pruning for future execution + logger.info(f"Scheduling pruning of '{file_path}' from '{source_endpoint.name}' " + f"in {days_from_now.total_seconds()/86400:.1f} days") + + try: + schedule_prefect_flow( + deployment_name="prune_filesystem_endpoint/prune_filesystem_endpoint", + flow_run_name=flow_name, + parameters={ + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + duration_from_now=days_from_now, + ) + logger.info(f"Successfully scheduled pruning task for {days_from_now.total_seconds()/86400:.1f} days from now") + return True + except Exception as e: + logger.error(f"Failed to schedule pruning task: {str(e)}", exc_info=True) + return False + + @staticmethod @flow(name="prune_filesystem_endpoint") def _prune_filesystem_endpoint( relative_path: str, source_endpoint: FileSystemEndpoint, - check_endpoint: Union[FileSystemEndpoint, None] = None, + check_endpoint: Optional[FileSystemEndpoint] = None, config: BeamlineConfig = None - ): + ) -> None: """ - Prune files from a File System. + Prefect flow that performs the actual filesystem pruning operation. Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (FileSystemEndpoint): The Globus source endpoint to prune from. - check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + relative_path (str): The path of the file or directory to prune + source_endpoint (FileSystemEndpoint): The source endpoint to prune from + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning + config (Optional[BeamlineConfig]): Configuration object, if needed + + Returns: + bool: True if pruning was successful, False otherwise """ logger.info(f"Running flow: prune_from_{source_endpoint.name}") logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") @@ -125,54 +203,114 @@ def _prune_filesystem_endpoint( class GlobusPruneController(PruneController[GlobusEndpoint]): + """ + Controller for pruning files from Globus endpoints. + + This controller handles pruning operations on Globus endpoints using + the Globus Transfer API. + + Args: + config (BeamlineConfig): Configuration object containing Globus endpoints and credentials + """ def __init__( self, - config + config: BeamlineConfig ) -> None: + """ + Initialize the file system prune controller. + + Args: + config (BeamlineConfig): Configuration object containing file system paths + """ super().__init__(config) + logger.debug(f"Initialized FileSystemPruneController for beamline {config.beamline_id}") def prune( self, file_path: str = None, source_endpoint: GlobusEndpoint = None, - check_endpoint: GlobusEndpoint = None, + check_endpoint: Optional[GlobusEndpoint] = None, days_from_now: datetime.timedelta = 0 ) -> bool: + """ + Prune (delete) data from a file system endpoint. + + If days_from_now is 0, executes pruning immediately. + Otherwise, schedules pruning for future execution using Prefect. + + Args: + file_path (str): The path to the file or directory to prune + source_endpoint (FileSystemEndpoint): The file system endpoint containing the data + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning + days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + + Returns: + bool: True if pruning was successful or scheduled successfully, False otherwise + """ + if not file_path: + logger.error("No file_path provided for pruning operation") + return False + + if not source_endpoint: + logger.error("No source_endpoint provided for pruning operation") + return False # globus_settings = JSON.load("globus-settings").value # max_wait_seconds = globus_settings["max_wait_seconds"] flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") - schedule_prefect_flow( - "prune_globus_endpoint/prune_globus_endpoint", - flow_name, - { - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - - datetime.timedelta(days=days_from_now), - ) - return True - + logger.info(f"Setting up pruning of '{file_path}' from '{source_endpoint.name}'") + + # If days_from_now is 0, prune immediately + if days_from_now.total_seconds() == 0: + logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") + return self._prune_filesystem_endpoint( + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + config=self.config + ) + else: + # Otherwise, schedule pruning for future execution + logger.info(f"Scheduling pruning of '{file_path}' from '{source_endpoint.name}' " + f"in {days_from_now.total_seconds()/86400:.1f} days") + + try: + schedule_prefect_flow( + deployment_name="prune_filesystem_endpoint/prune_filesystem_endpoint", + flow_run_name=flow_name, + parameters={ + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + duration_from_now=days_from_now, + ) + logger.info(f"Successfully scheduled pruning task for {days_from_now.total_seconds()/86400:.1f} days from now") + return True + except Exception as e: + logger.error(f"Failed to schedule pruning task: {str(e)}", exc_info=True) + return False + + @staticmethod @flow(name="prune_globus_endpoint") def _prune_globus_endpoint( relative_path: str, source_endpoint: GlobusEndpoint, - check_endpoint: Union[GlobusEndpoint, None] = None, + check_endpoint: Optional[GlobusEndpoint] = None, config: BeamlineConfig = None ) -> None: """ - Prune files from a Globus endpoint. + Prefect flow that performs the actual Globus endpoint pruning operation. Args: - relative_path (str): The path of the file or directory to prune. - source_endpoint (GlobusEndpoint): The Globus source endpoint to prune from. - check_endpoint (GlobusEndpoint, optional): The Globus target endpoint to check. Defaults to None. + relative_path (str): The path of the file or directory to prune + source_endpoint (GlobusEndpoint): The Globus endpoint to prune from + check_endpoint (Optional[GlobusEndpoint]): If provided, verify data exists here before pruning + config (BeamlineConfig): Configuration object with transfer client """ + logger.info(f"Running Globus pruning flow for '{relative_path}' from '{source_endpoint.name}'") + globus_settings = JSON.load("globus-settings").value max_wait_seconds = globus_settings["max_wait_seconds"] flow_name = f"prune_from_{source_endpoint.name}" @@ -192,7 +330,14 @@ def _prune_globus_endpoint( class PruneMethod(Enum): """ Enum representing different prune methods. - Use enum names as strings to identify trpruneansfer methods, ensuring a standard set of values. + + These values are used to select the appropriate prune controller + through the factory function get_prune_controller(). + + Attributes: + GLOBUS: Use Globus Transfer API for pruning operations + SIMPLE: Use local file system operations for pruning + HPSS: Use HPSS tape archive specific commands for pruning """ GLOBUS = "globus" SIMPLE = "simple" @@ -204,20 +349,28 @@ def get_prune_controller( config: BeamlineConfig ) -> PruneController: """ - Get the appropriate prune controller based on the prune type. + Factory function to get the appropriate prune controller based on the prune type. Args: - prune_type (str): The type of transfer to perform. - config (BeamlineConfig): The configuration object. + prune_type (PruneMethod): The type of pruning to perform + config (BeamlineConfig): The configuration object containing endpoint information Returns: - PruneController: The transfer controller object. + PruneController: The appropriate prune controller instance + + Raises: + ValueError: If an invalid prune type is provided """ + logger.debug(f"Creating prune controller of type: {prune_type.name}") + if prune_type == PruneMethod.GLOBUS: + logger.debug("Returning GlobusPruneController") return GlobusPruneController(config) elif prune_type == PruneMethod.SIMPLE: + logger.debug("Returning FileSystemPruneController") return FileSystemPruneController(config) elif prune_type == PruneMethod.HPSS: + logger.debug("Importing and returning HPSSPruneController") from orchestration.hpss import HPSSPruneController from orchestration.sfapi import create_sfapi_client return HPSSPruneController( @@ -225,4 +378,6 @@ def get_prune_controller( config=config ) else: - raise ValueError(f"Invalid transfer type: {prune_type}") + error_msg = f"Invalid prune type: {prune_type}" + logger.error(error_msg) + raise ValueError(error_msg) From 8674291897431440a4f1eed999a186b0b5a4d2d1 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 6 Mar 2025 16:06:26 -0800 Subject: [PATCH 035/128] Improved error logging and exception handling for tape flows --- orchestration/hpss.py | 275 ++++++++++++++++++++++++++++++------------ 1 file changed, 199 insertions(+), 76 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index b4e21504..b3fe2b37 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -1,6 +1,18 @@ """ -This module contains HPSS related functions and classes. +HPSS Module - Handling transfers to and from NERSC's High Performance Storage System (HPSS). + +This module provides functionality for transferring data between NERSC's Community File System (CFS) +and the High Performance Storage System (HPSS) tape archive. It includes: + +1. Prefect flows for initiating transfers in both directions +2. Transfer controllers for CFS to HPSS and HPSS to CFS operations +3. HPSS-specific pruning controller for managing data lifecycle +4. Slurm job scripts for executing HPSS operations via SFAPI + +The module follows tape-safe practices as recommended in NERSC documentation: +https://docs.nersc.gov/filesystems/HPSS-best-practices/ """ + import datetime import logging from pathlib import Path @@ -34,43 +46,70 @@ def cfs_to_hpss_flow( config: BeamlineConfig = None ) -> bool: """ - The CFS to HPSS flow. - - Parameters - ---------- - file_path : Union[str, List[str]] - A single file path or a list of file paths to transfer. - source : FileSystemEndpoint - The source endpoint. - destination : HPSSEndpoints - The destination endpoint. - config : BeamlineConfig - The beamline configuration. - - Returns - ------- - bool - True if all transfers succeeded, False otherwise. - """ + Prefect flow for transferring data from CFS to HPSS tape archive. + + This flow handles the transfer of files or directories from NERSC's Community + File System (CFS) to the High Performance Storage System (HPSS) tape archive. + For directories, files are bundled into tar archives based on time periods. + Args: + file_path (Union[str, List[str]]): A single file path or a list of file paths to transfer + source (FileSystemEndpoint): The CFS source endpoint + destination (HPSSEndpoint): The HPSS destination endpoint + config (BeamlineConfig): The beamline configuration containing endpoints and credentials + + Returns: + bool: True if all transfers succeeded, False otherwise + """ logger.info("Running cfs_to_hpss_flow") - logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") + if not file_path: + logger.error("No file path provided for CFS to HPSS transfer") + return False + + if not source or not destination: + logger.error("Source or destination endpoint not provided for CFS to HPSS transfer") + return False + + if not config: + logger.error("No configuration provided for CFS to HPSS transfer") + return False + + # Log detailed information about the transfer + if isinstance(file_path, list): + logger.info(f"Transferring {len(file_path)} files/directories from {source.name} to {destination.name}") + for path in file_path: + logger.debug(f" - {path}") + else: + logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") + + # Configure the transfer controller for CFS to HPSS logger.info("Configuring transfer controller for CFS_TO_HPSS.") - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.CFS_TO_HPSS, - config=config - ) + try: + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.CFS_TO_HPSS, + config=config + ) + except Exception as e: + logger.error(f"Failed to initialize CFS to HPSS transfer controller: {str(e)}", exc_info=True) + return False logger.info("CFSToHPSSTransferController selected. Initiating transfer for all file paths.") - result = transfer_controller.copy( - file_path=file_path, - source=source, - destination=destination - ) - - return result + try: + result = transfer_controller.copy( + file_path=file_path, + source=source, + destination=destination + ) + if result: + logger.info("CFS to HPSS transfer completed successfully") + else: + logger.error("CFS to HPSS transfer failed") + return result + except Exception as e: + logger.error(f"Error during CFS to HPSS transfer: {str(e)}", exc_info=True) + return False @flow(name="hpss_to_cfs_flow") @@ -82,37 +121,76 @@ def hpss_to_cfs_flow( config: BeamlineConfig = None ) -> bool: """ - The HPSS to CFS flow. - - Parameters - ---------- - file_path : str - The path of the file to transfer. - source_endpoint : HPSSEndpoint - The source endpoint. - destination_endpoint : FileSystemEndpoint - The destination endpoint. - """ + Prefect flow for retrieving data from HPSS tape archive to CFS. + + This flow handles the retrieval of files or tar archives from NERSC's High + Performance Storage System (HPSS) to the Community File System (CFS). + For tar archives, you can optionally specify specific files to extract. + + Args: + file_path (str): The path of the file or tar archive on HPSS + source (HPSSEndpoint): The HPSS source endpoint + destination (FileSystemEndpoint): The CFS destination endpoint + files_to_extract (Optional[List[str]]): Specific files to extract from the tar archive + config (BeamlineConfig): The beamline configuration containing endpoints and credentials + Returns: + bool: True if the transfer succeeded, False otherwise + """ logger.info("Running hpss_to_cfs_flow") + + if not file_path: + logger.error("No file path provided for HPSS to CFS transfer") + return False + + if not source or not destination: + logger.error("Source or destination endpoint not provided for HPSS to CFS transfer") + return False + + if not config: + logger.error("No configuration provided for HPSS to CFS transfer") + return False + logger.info(f"Transferring {file_path} from {source.name} to {destination.name}") + # Log detailed information about the transfer + if files_to_extract: + logger.info(f"Extracting {len(files_to_extract)} specific files from tar archive:") + for file in files_to_extract: + logger.debug(f" - {file}") + + # Configure transfer controller for HPSS_TO_CFS logger.info("Configuring transfer controller for HPSS_TO_CFS.") - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.HPSS_TO_CFS, - config=config - ) + try: + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.HPSS_TO_CFS, + config=config + ) + except Exception as e: + logger.error(f"Failed to initialize HPSS to CFS transfer controller: {str(e)}", exc_info=True) + return False logger.info("HPSSToCFSTransferController selected. Initiating transfer for all file paths.") - result = transfer_controller.copy( - file_path=file_path, - source=source, - destination=destination, - files_to_extract=files_to_extract, - ) + # Initiate transfer + logger.info("HPSSToCFSTransferController selected. Initiating transfer.") + try: + result = transfer_controller.copy( + file_path=file_path, + source=source, + destination=destination, + files_to_extract=files_to_extract, + ) + + if result: + logger.info("HPSS to CFS transfer completed successfully") + else: + logger.error("HPSS to CFS transfer failed") - return result + return result + except Exception as e: + logger.error(f"Error during HPSS to CFS transfer: {str(e)}", exc_info=True) + return False # ---------------------------------- @@ -121,18 +199,31 @@ def hpss_to_cfs_flow( class HPSSPruneController(PruneController[HPSSEndpoint]): """ - Use SFAPI, Slurm, and hsi to prune data from HPSS at NERSC. - This controller requires the source to be an HPSSEndpoint and the - optional destination to be a FileSystemEndpoint. It uses "hsi rm" to prune - files from HPSS. + Controller for pruning files from HPSS tape archive. + + This controller uses SFAPI, Slurm, and hsi to prune data from HPSS at NERSC. + It requires the source to be an HPSSEndpoint and the optional destination to + be a FileSystemEndpoint. It uses "hsi rm" to prune files from HPSS. + + Args: + client (Client): The SFAPI client for submitting jobs to NERSC + config (BeamlineConfig): Configuration object containing endpoints and credentials """ def __init__( self, client: Client, config: BeamlineConfig, ) -> None: + """ + Initialize the HPSS prune controller. + + Args: + client (Client): The SFAPI client for submitting jobs to NERSC + config (BeamlineConfig): Configuration object containing endpoints and credentials + """ super().__init__(config) self.client = client + logger.debug(f"Initialized HPSSPruneController with client for beamline {config.beamline_id}") def prune( self, @@ -141,30 +232,62 @@ def prune( check_endpoint: Optional[FileSystemEndpoint] = None, days_from_now: datetime.timedelta = 0 ) -> bool: + """ + Prune (delete) data from HPSS tape archive. + + If days_from_now is 0, executes pruning immediately. + Otherwise, schedules pruning for future execution using Prefect. + + Args: + file_path (str): The path to the file or directory to prune on HPSS + source_endpoint (HPSSEndpoint): The HPSS endpoint containing the data + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning + days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + + Returns: + bool: True if pruning was successful or scheduled successfully, False otherwise + """ + if not file_path: + logger.error("No file_path provided for HPSS pruning operation") + return False + + if not source_endpoint: + logger.error("No source_endpoint provided for HPSS pruning operation") + return False + flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {file_path} from source endpoint: {source_endpoint.name}") + logger.info(f"Setting up pruning of '{file_path}' from HPSS endpoint '{source_endpoint.name}'") - if days_from_now == 0: + # If days_from_now is 0, prune immediately + if days_from_now.total_seconds() == 0: self._prune_hpss_endpoint( self, relative_path=file_path, source_endpoint=source_endpoint, check_endpoint=check_endpoint, ) + # Otherwise, schedule pruning for future execution else: - schedule_prefect_flow( - deployment_name="prune_hpss_endpoint/prune_hpss_endpoint", - flow_run_name=flow_name, - parameters={ - "relative_path": file_path, - "source_endpoint": source_endpoint, - "check_endpoint": check_endpoint, - "config": self.config - }, - duration_from_now=days_from_now - ) - return True + logger.info(f"Scheduling pruning of '{file_path}' from '{source_endpoint.name}' " + f"in {days_from_now.total_seconds()/86400:.1f} days") + + try: + schedule_prefect_flow( + deployment_name="prune_hpss_endpoint/prune_hpss_endpoint", + flow_run_name=flow_name, + parameters={ + "relative_path": file_path, + "source_endpoint": source_endpoint, + "check_endpoint": check_endpoint, + "config": self.config + }, + duration_from_now=days_from_now + ) + logger.info(f"Successfully scheduled HPSS pruning task in {days_from_now.total_seconds()/86400:.1f} days") + return True + except Exception as e: + logger.error(f"Failed to schedule HPSS pruning task: {str(e)}", exc_info=True) + return False @flow(name="prune_hpss_endpoint") def _prune_hpss_endpoint( @@ -174,12 +297,12 @@ def _prune_hpss_endpoint( check_endpoint: Optional[Union[FileSystemEndpoint, None]] = None, ) -> None: """ - Prune files from HPSS. + Prefect flow that performs the actual HPSS pruning operation. Args: - relative_path (str): The HPSS path of the file or directory to prune. - source_endpoint (HPSSEndpoint): The Globus source endpoint to prune from. - check_endpoint (FileSystemEndpoint, optional): The Globus target endpoint to check. Defaults to None. + relative_path (str): The HPSS path of the file or directory to prune + source_endpoint (HPSSEndpoint): The HPSS endpoint to prune from + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning """ logger.info("Pruning files from HPSS") logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") From f0b765187a9e3a5c8fe6d6176dd6c9dcd90c5cd4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 6 Mar 2025 16:33:59 -0800 Subject: [PATCH 036/128] Verified SciCat login to the latest Docker version of scicatlive. It first tries the currently implemented way of logging in (for backwards compatability), but if that fails, it falls back to a new workaround (related: https://github.com/SciCatProject/pyscicat/issues/61). Scicat ingestor credentials can be passed in, or read from environment variables. --- orchestration/flows/bl832/move_refactor.py | 4 +- orchestration/flows/bl832/scicat_ingestor.py | 4 +- .../flows/scicat/ingestor_controller.py | 99 ++++++++++++++++--- 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index b53d32eb..89233c16 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -70,7 +70,9 @@ def process_new_832_file( if nersc_transfer_success: logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") try: - ingestor = TomographyIngestorController(config, config.scicat_client) + ingestor = TomographyIngestorController(config) + # login_to_scicat assumes that the environment variables are set in the environment + ingestor.login_to_scicat() ingestor.ingest_new_raw_dataset(file_path) except Exception as e: logger.error(f"SciCat ingest failed with {e}") diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 7b6f0df7..e8a4e6f7 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -231,7 +231,9 @@ def _calculate_access_controls( beamline, proposal ) -> Dict: - """Calculate access controls for a dataset.""" + """ + Calculate access controls for a dataset. + """ # make an access group list that includes the name of the proposal and the name of the beamline access_groups = [] diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index ac820278..468bb50b 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -1,11 +1,16 @@ from abc import ABC, abstractmethod +import logging from logging import getLogger +import os +import requests from typing import Optional -from orchestration.config import BeamlineConfig from pyscicat.client import ScicatClient, from_credentials +from orchestration.config import BeamlineConfig + +logging.basicConfig(level=logging.INFO) logger = getLogger(__name__) @@ -18,23 +23,72 @@ class BeamlineIngestorController(ABC): def __init__( self, config: BeamlineConfig, - scicat_client: ScicatClient + scicat_client: Optional[ScicatClient] = None ) -> None: self.config = config self.scicat_client = scicat_client - def _login_to_scicat( + def login_to_scicat( self, - scicat_base_url: str, - scicat_user: str, - scicat_password: str + scicat_base_url: Optional[str] = None, + scicat_user: Optional[str] = None, + scicat_password: Optional[str] = None ) -> ScicatClient: - scicat_client = from_credentials( - base_url=scicat_base_url, - username=scicat_user, - password=scicat_password - ) - return scicat_client + """ + Log in to SciCat using the provided credentials. + + :param scicat_base_url: Base URL of the SciCat instance. Defaults to the environment variable 'SCICAT_API_URL'. + :param scicat_user: Username for the SciCat instance. Defaults to the environment variable 'SCICAT_INGEST_USER'. + :param scicat_password: Password for the SciCat instance. Defaults to the environment variable 'SCICAT_INGEST_PASSWORD' + :return: An instance of ScicatClient with an authenticated session. + :raises ValueError: If any required credentials are missing. + """ + # Use environment variables as defaults if parameters are not provided. + scicat_base_url = scicat_base_url or os.getenv("SCICAT_API_URL") + scicat_user = scicat_user or os.getenv("SCICAT_INGEST_USER") + scicat_password = scicat_password or os.getenv("SCICAT_INGEST_PASSWORD") + + # Ensure that all required credentials are provided. + if not (scicat_base_url and scicat_user and scicat_password): + raise ValueError( + "Missing required SciCat credentials. Provide scicat_base_url, scicat_user, " + "and scicat_password as parameters or set them in the environment variables: " + "SCICAT_API_URL, SCICAT_INGEST_USER, SCICAT_INGEST_PASSWORD." + ) + + # Try to log in using the pyscicat client first. + # This method seems deprecated, but leaving it here for backwards compatability + # https://github.com/SciCatProject/pyscicat/issues/61 + try: + self.scicat_client = from_credentials( + base_url=scicat_base_url, + username=scicat_user, + password=scicat_password + ) + logger.info("Logged in to SciCat.") + return self.scicat_client + except Exception as e: + logger.error(f"Failed to log in to SciCat: {e}, trying alternative method.") + + # This method works for scicatlive 3.2.5 + try: + response = requests.post( + url=scicat_base_url, + json={"username": scicat_user, "password": scicat_password}, + stream=False, + verify=True, + ) + self.scicat_client = ScicatClient(scicat_base_url, response.json()["access_token"]) + logger.info("Logged in to SciCat.") + # logger.info(f"SciCat token: {response.json()['access_token']}") + return self.scicat_client + + except requests.exceptions.RequestException as e: + logger.error(f"Failed to log in to SciCat: {e}") + raise e + except Exception as e: + logger.error(f"Failed to log in to SciCat: {e}") + raise e @abstractmethod def ingest_new_raw_dataset( @@ -155,3 +209,24 @@ def _find_dataset( raise ValueError("The dataset returned does not have a valid 'pid' field.") return dataset_id + + +# Concrete implementation for testing and instantiation. +class ConcreteBeamlineIngestorController(BeamlineIngestorController): + def ingest_new_raw_dataset(self, file_path: str = "") -> str: + # Dummy implementation for testing. + return "raw_dataset_id_dummy" + + def ingest_new_derived_dataset(self, file_path: str = "", raw_dataset_id: str = "") -> str: + # Dummy implementation for testing. + return "derived_dataset_id_dummy" + + +if __name__ == "__main__": + logger.info("Testing SciCat ingestor controller") + test_ingestor = ConcreteBeamlineIngestorController(BeamlineConfig) + test_ingestor.login_to_scicat( + scicat_base_url="http://localhost:3000/api/v3/auth/login", + scicat_user="ingestor", + scicat_password="aman" + ) From 274418fcea104d8732e4336339476dace0427772 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 6 Mar 2025 17:00:03 -0800 Subject: [PATCH 037/128] Fixing pytest errors and failures --- orchestration/_tests/test_prune_controller.py | 3 +- .../_tests/test_transfer_controller.py | 68 +++++++++---------- orchestration/transfer_controller.py | 6 ++ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index 9e6ef35b..0d051989 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -22,9 +22,10 @@ class DummyPruneController(PruneController): - prune_files(retention): deletes files older than the given retention period. """ def __init__(self, base_dir: Path): - # Create a dummy configuration object (the real config isnā€™t used in these tests) + # Create a dummy configuration object with a default beamline_id. dummy_config = type("DummyConfig", (), {})() dummy_config.tc = None + dummy_config.beamline_id = "dummy" # Add a default beamline_id for testing super().__init__(dummy_config) self.base_dir = base_dir diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index f7ffd7e9..fb9c8570 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -321,52 +321,52 @@ def test_simple_transfer_controller_copy_success( mock_config832, mock_file_system_endpoint, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("os.system", return_value=0) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - - assert result is True, "Expected True when os.system returns 0." - mock_os_system.assert_called_once() - command_called = mock_os_system.call_args[0][0] - assert "cp -r" in command_called, "Expected cp command in os.system call." + with patch("orchestration.transfer_controller.os.path.exists", return_value=True): # patch in module namespace + with patch("orchestration.transfer_controller.os.system", return_value=0) as mock_os_system: + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="some_dir/test_file.txt", + source=mock_file_system_endpoint, + destination=mock_file_system_endpoint, + ) + assert result is True, "Expected True when os.system returns 0." + mock_os_system.assert_called_once() + command_called = mock_os_system.call_args[0][0] + assert "cp -r" in command_called, "Expected cp command in os.system call." def test_simple_transfer_controller_copy_failure( mock_config832, mock_file_system_endpoint, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("os.system", return_value=1) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - - assert result is False, "Expected False when os.system returns non-zero." - mock_os_system.assert_called_once() - command_called = mock_os_system.call_args[0][0] - assert "cp -r" in command_called, "Expected cp command in os.system call." + with patch("orchestration.transfer_controller.os.path.exists", return_value=True): # ensure source file exists + with patch("orchestration.transfer_controller.os.system", return_value=1) as mock_os_system: + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="some_dir/test_file.txt", + source=mock_file_system_endpoint, + destination=mock_file_system_endpoint, + ) + assert result is False, "Expected False when os.system returns non-zero." + mock_os_system.assert_called_once() + command_called = mock_os_system.call_args[0][0] + assert "cp -r" in command_called, "Expected cp command in os.system call." def test_simple_transfer_controller_copy_exception( mock_config832, mock_file_system_endpoint, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("os.system", side_effect=Exception("Mocked cp error")) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - - assert result is False, "Expected False when an exception is raised during copy." - mock_os_system.assert_called_once() + with patch("orchestration.transfer_controller.os.path.exists", return_value=True): + with patch("orchestration.transfer_controller.os.system", side_effect=Exception("Mocked cp error")) as mock_os_system: + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="some_dir/test_file.txt", + source=mock_file_system_endpoint, + destination=mock_file_system_endpoint, + ) + assert result is False, "Expected False when an exception is raised during copy." + mock_os_system.assert_called_once() # -------------------------------------------------------------------------- diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index a019c7ac..3f442c3c 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -417,6 +417,12 @@ def get_transfer_controller( Raises: ValueError: If an invalid transfer type is provided """ + # Add explicit type checking to handle non-enum inputs + if not isinstance(transfer_type, CopyMethod): + error_msg = f"Invalid transfer type: {transfer_type}" + logger.error(error_msg) + raise ValueError(error_msg) + logger.debug(f"Creating transfer controller of type: {transfer_type.name}") if transfer_type == CopyMethod.GLOBUS: From df53e7ceab3a29c0cc8ba21746141a1e22c87edd Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 7 Mar 2025 13:56:05 -0800 Subject: [PATCH 038/128] Testing scicat ingestion locally on a small test h5 dataset from 832. Able to ingest using the default admin account on a local scicatlive instance. --- orchestration/flows/bl832/scicat_ingestor.py | 23 +++++++++++++++++-- .../flows/scicat/ingestor_controller.py | 15 ++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index e8a4e6f7..ae19c201 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -3,7 +3,7 @@ from logging import getLogger import os from pathlib import Path -from typing import Any, Dict, List # Optional, Union +from typing import Any, Dict, List, Optional import h5py from pyscicat.client import ScicatClient @@ -119,7 +119,7 @@ class TomographyIngestorController(BeamlineIngestorController): def __init__( self, config: Config832, - scicat_client: ScicatClient + scicat_client: Optional[ScicatClient] = None ) -> None: super().__init__(config, scicat_client) @@ -227,6 +227,7 @@ def ingest_new_derived_dataset( pass def _calculate_access_controls( + self, username, beamline, proposal @@ -417,3 +418,21 @@ def _upload_raw_dataset( logger.debug(f"dataset: {dataset}") dataset_id = self.scicat_client.upload_new_dataset(dataset) return dataset_id + + +if __name__ == "__main__": + config = Config832() + file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" + ingestor = TomographyIngestorController(config) + # login_to_scicat assumes that the environment variables are set in the environment + # in this test, just using the scicatlive backend defaults to the admin user + ingestor.login_to_scicat( + scicat_base_url="http://localhost:3000/api/v3/", + scicat_user="admin", + scicat_password="2jf70TPNZsS" + ) + # INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set + os.environ["INGEST_STORAGE_ROOT_PATH"] = "/global/cfs/cdirs/als/data_mover/8.3.2" + os.environ["INGEST_SOURCE_ROOT_PATH"] = "/data832-raw" + + ingestor.ingest_new_raw_dataset(file_path) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 468bb50b..66ed7cac 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -1,17 +1,17 @@ from abc import ABC, abstractmethod import logging -from logging import getLogger import os import requests from typing import Optional +from urllib.parse import urljoin from pyscicat.client import ScicatClient, from_credentials from orchestration.config import BeamlineConfig -logging.basicConfig(level=logging.INFO) -logger = getLogger(__name__) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) class BeamlineIngestorController(ABC): @@ -48,6 +48,8 @@ def login_to_scicat( scicat_user = scicat_user or os.getenv("SCICAT_INGEST_USER") scicat_password = scicat_password or os.getenv("SCICAT_INGEST_PASSWORD") + logger.info(f"Logging in to SciCat at {scicat_base_url} as {scicat_user}.") + # Ensure that all required credentials are provided. if not (scicat_base_url and scicat_user and scicat_password): raise ValueError( @@ -72,12 +74,15 @@ def login_to_scicat( # This method works for scicatlive 3.2.5 try: + url = urljoin(scicat_base_url, "auth/login") + logger.info(url) response = requests.post( - url=scicat_base_url, + url=url, json={"username": scicat_user, "password": scicat_password}, stream=False, verify=True, ) + logger.info(f"Login response: {response.json()}") self.scicat_client = ScicatClient(scicat_base_url, response.json()["access_token"]) logger.info("Logged in to SciCat.") # logger.info(f"SciCat token: {response.json()['access_token']}") @@ -226,7 +231,7 @@ def ingest_new_derived_dataset(self, file_path: str = "", raw_dataset_id: str = logger.info("Testing SciCat ingestor controller") test_ingestor = ConcreteBeamlineIngestorController(BeamlineConfig) test_ingestor.login_to_scicat( - scicat_base_url="http://localhost:3000/api/v3/auth/login", + scicat_base_url="http://localhost:3000/api/v3/", scicat_user="ingestor", scicat_password="aman" ) From 1f60795f6951cee5a177456dfa42ac23328ea49d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 7 Mar 2025 15:09:41 -0800 Subject: [PATCH 039/128] Testing scicat ingestion locally on a small test h5 dataset from 832. Able to ingest using the default admin account on a local scicatlive instance. --- orchestration/flows/bl832/scicat_ingestor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index ae19c201..6dcc192f 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -425,7 +425,7 @@ def _upload_raw_dataset( file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" ingestor = TomographyIngestorController(config) # login_to_scicat assumes that the environment variables are set in the environment - # in this test, just using the scicatlive backend defaults to the admin user + # in this test, just using the scicatlive backend defaults (admin user) ingestor.login_to_scicat( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="admin", From 3504f5a400484e4f417d79a26c897294bfd7feba Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 10 Mar 2025 13:58:28 -0700 Subject: [PATCH 040/128] updated and tested add_new_dataset_location() in orchestration/flows/scicat/ingestor_controller.py, which now creates a new OrigDataBlock with the updated filesystem host and path. The new path appears under the Datafiles tab in the SciCat UI. --- orchestration/flows/bl832/move_refactor.py | 18 ++++ orchestration/flows/bl832/scicat_ingestor.py | 45 +++++++- .../flows/scicat/ingestor_controller.py | 102 ++++++++++++------ 3 files changed, 130 insertions(+), 35 deletions(-) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index 89233c16..da23b6fa 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -144,3 +144,21 @@ def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): ) logger.info(f"File successfully transferred from data832 to NERSC {new_file}. Success: {nersc_success}") pass + + +if __name__ == "__main__": + config = Config832() + file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" + ingestor = TomographyIngestorController(config) + # login_to_scicat assumes that the environment variables are set in the environment + # in this test, just using the scicatlive backend defaults (admin user) + ingestor.login_to_scicat( + scicat_base_url="http://localhost:3000/api/v3/", + scicat_user="admin", + scicat_password="2jf70TPNZsS" + ) + # INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set + os.environ["INGEST_STORAGE_ROOT_PATH"] = "/global/cfs/cdirs/als/data_mover/8.3.2" + os.environ["INGEST_SOURCE_ROOT_PATH"] = "/data832-raw" + + ingestor.ingest_new_raw_dataset(file_path) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 6dcc192f..331ae5c9 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -262,7 +262,9 @@ def _create_data_files( file_path: Path, storage_path: str ) -> List[DataFile]: - "Collects all fits files" + """ + Collects all fits files + """ datafiles = [] datafile = DataFile( path=storage_path, @@ -343,7 +345,9 @@ def _upload_data_block( storage_root_path: str, source_root_path: str ) -> Datablock: - "Creates a datablock of files" + """ + Creates a datablock of files + """ # calculate the path where the file will as known to SciCat storage_path = str(file_path).replace(source_root_path, storage_root_path) datafiles = self._create_data_files(file_path, storage_path) @@ -421,18 +425,51 @@ def _upload_raw_dataset( if __name__ == "__main__": + config = Config832() file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" + proposal_id = "test832" ingestor = TomographyIngestorController(config) + # login_to_scicat assumes that the environment variables are set in the environment - # in this test, just using the scicatlive backend defaults (admin user) + # in this test, just using the scicatlive (3.2.5) backend defaults (admin user) + + logger.info("Setting up metadata SciCat ingestion") + ingestor.login_to_scicat( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="admin", scicat_password="2jf70TPNZsS" ) + # INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set os.environ["INGEST_STORAGE_ROOT_PATH"] = "/global/cfs/cdirs/als/data_mover/8.3.2" os.environ["INGEST_SOURCE_ROOT_PATH"] = "/data832-raw" - ingestor.ingest_new_raw_dataset(file_path) + logger.info(f"Ingesting {file_path}") + id = ingestor.ingest_new_raw_dataset(file_path) + logger.info(f"Ingested with SciCat ID: {id}") + + # logger.info(f"Testing SciCat ID lookup after ingestion based on {file_path}") + # try: + # # Test lookup based on filename after ingestion + # id = ingestor._find_dataset(file_name=file_path) + # logger.info(f"Found dataset id {id}") + # except Exception as e: + # logger.error(f"Failed to find dataset {e}") + + # Pretend we moved to tape: + # /home/a/alsdev/data_mover/[beamline]/raw/[proposal_name]/[proposal_name]_[year]-[cycle].tar + ingestor.add_new_dataset_location( + dataset_id=id, + proposal_id=proposal_id, + file_name="20241216_153047_ddd.h5", + source_folder=f"/home/a/alsdev/data_mover/{config.beamline_id}/raw/{proposal_id}/{proposal_id}_2024-12.tar", + source_folder_host="HPSS", + ) + + # ingestor needs to add the derived dataset ingestion method + + # ingestor needs to add new "origdatablock" method for raw data on different filesystems + # ingestor needs to add new "datablock" method for raw data on HPSS system + # same for derived data diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 66ed7cac..ba66e317 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -6,7 +6,10 @@ from urllib.parse import urljoin from pyscicat.client import ScicatClient, from_credentials - +from pyscicat.model import ( + CreateDatasetOrigDatablockDto, + DataFile, +) from orchestration.config import BeamlineConfig @@ -140,18 +143,52 @@ def add_new_dataset_location( optionally including a protocol e.g. [protocol://]fileserver1.example.com", """ - # If dataset_id is not provided, we need to find it using proposal_id and file_name. - # Otherwise, we use the provided dataset_id directly. - if dataset_id is None and proposal_id and file_name: + # If dataset_id is not provided, we need to find it using file_name. + if dataset_id is None and file_name: dataset_id = self._find_dataset(proposal_id=proposal_id, file_name=file_name) + # Get the dataset to retrieve its metadata dataset = self.scicat_client.datasets_get_one(dataset_id) + if not dataset: + raise ValueError(f"Dataset with ID {dataset_id} not found") + + logger.info(f"Creating new datablock for dataset {dataset_id} at location {source_folder}") + + try: + # Create a datafile for the new location + basename = dataset.get("datasetName", "dataset") + file_path = f"{source_folder}/{basename}" + + # Get size from existing dataset if available + size = dataset.get("size", 0) + + # Create a single datafile + datafile = DataFile( + path=file_path, + size=size, + time=dataset.get("creationTime") + ) + + # Create a minimal datablock for the new location + datablock = CreateDatasetOrigDatablockDto( + size=size, + dataFileList=[datafile] + ) + + # Add location information to the path if host is provided + if source_folder_host: + datafile.path = f"{source_folder_host}:{file_path}" + + # Upload the datablock + self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) + logger.info(f"Created new datablock for dataset {dataset_id} at location {source_folder}") + + # Note: We're skipping the dataset update since it's causing validation issues + + except Exception as e: + logger.error(f"Failed to create new datablock for dataset {dataset_id}: {e}") + # Continue without raising to maintain the workflow - # sourceFolder sourceFolderHost are each a string - dataset["sourceFolder"] = source_folder - dataset["sourceFolderHost"] = source_folder_host - self.scicat_client.datasets_update(dataset, dataset_id) - logger.info(f"Added location {source_folder} to dataset {dataset_id}") return dataset_id def remove_dataset_location( @@ -170,45 +207,48 @@ def _find_dataset( file_name: Optional[str] = None ) -> str: """ - Find a dataset in SciCat and return the ID based on proposal ID and file name. - This method is used when a dataset ID is not provided. - If more than one dataset is found, an error is raised, and the user is advised to check the logs. - If no dataset is found, an error is raised. - If exactly one dataset is found, its ID is returned. - This method is intended to be used internally within the class. + Find a dataset in SciCat and return its ID based on proposal ID and file name. + The dataset name in SciCat is expected to be saved as the base filename without the extension, + e.g. '20241216_153047_ddd' for a file named '20241216_153047_ddd.h5'. Parameters: - self, proposal_id (Optional[str]): The proposal identifier used in ingestion. - file_name (Optional[str]): The dataset name (derived from file name). + file_name (Optional[str]): The full path to the file; its base name (without extension) will be used. + + Returns: + str: The SciCat ID of the dataset. Raises: - ValueError: If insufficient search parameters are provided, - no dataset is found, or multiple datasets match. + ValueError: If no dataset or multiple datasets are found, or if the found dataset does not have a valid 'pid'. """ - # Require both search terms if no dataset_id is given. - if not (proposal_id and file_name): - raise ValueError("Either a dataset ID must be provided or both proposal_id and file_name must be given.") + if file_name: + # Extract the datasetName from the file_name by stripping the directory and extension. + extracted_name = os.path.splitext(os.path.basename(file_name))[0] + else: + extracted_name = None query_fields = { "proposalId": proposal_id, - "datasetName": file_name + "datasetName": extracted_name } results = self.scicat_client.datasets_find(query_fields=query_fields) - count = results.get("count", 0) + + # Assuming the client returns a list of datasets. + count = len(results) if count == 0: - raise ValueError(f"No dataset found for proposal '{proposal_id}' with name '{file_name}'.") + raise ValueError(f"No dataset found for proposal '{proposal_id}' with dataset name '{extracted_name}'.") elif count > 1: # Log all found dataset IDs for human review. - dataset_ids = [d.get("pid", "N/A") for d in results["data"]] + dataset_ids = [d.get("pid", "N/A") for d in results] logger.error( - f"Multiple datasets found for proposal '{proposal_id}' with name '{file_name}': {dataset_ids}. Please verify." - ) - raise ValueError( - f"Multiple datasets found for proposal '{proposal_id}' with name '{file_name}'. See log for details." + f"Multiple datasets found for proposal '{proposal_id}' with dataset name '{extracted_name}': {dataset_ids}." ) - dataset = results["data"][0] + # raise ValueError( + # f"Multiple datasets found for proposal '{proposal_id}' with dataset name '{extracted_name}'." + # ) + + dataset = results[0] dataset_id = dataset.get("pid") if not dataset_id: raise ValueError("The dataset returned does not have a valid 'pid' field.") From 300f92eb4abd2e732a4f80a01dd8d09b2b6feb0a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 10 Mar 2025 17:21:18 -0700 Subject: [PATCH 041/128] Addedsupport for linking tomography reconstructions as derived datasets in SciCat. Tested it and it works, Tiffs and Zarrs are findable under the Related Datasets tab in the UI. Need to fix the thumbnails --- orchestration/flows/bl832/scicat_ingestor.py | 294 ++++++++++++++++++- 1 file changed, 291 insertions(+), 3 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 331ae5c9..45695e0e 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -19,6 +19,7 @@ from orchestration.flows.bl832.config import Config832 from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController +from pyscicat.model import DerivedDataset from orchestration.flows.scicat.utils import ( build_search_terms, build_thumbnail, @@ -221,10 +222,288 @@ def ingest_new_derived_dataset( """ Ingest a new derived dataset from the Tomo832 beamline. - :param file_path: Path to the file to ingest. - :return: SciCat ID of the dataset. + This method handles ingestion of derived datasets generated during tomography reconstruction: + 1. A directory of TIFF slices + 2. A Zarr directory (derived from the TIFFs) + + :param folder_path: Path to the folder containing the derived data. + :param raw_dataset_id: ID of the raw dataset this derived data is based on. + :return: SciCat ID of the derived dataset. + :raises ValueError: If required environment variables are missing. + :raises Exception: If any issues are encountered during ingestion. """ - pass + issues: List[Issue] = [] + logger.setLevel("INFO") + + logger.info(f"Ingesting derived dataset from folder: {folder_path}") + # Retrieve required environment variables for storage paths + INGEST_STORAGE_ROOT_PATH = os.getenv("INGEST_STORAGE_ROOT_PATH") + INGEST_SOURCE_ROOT_PATH = os.getenv("INGEST_SOURCE_ROOT_PATH") + if not INGEST_STORAGE_ROOT_PATH or not INGEST_SOURCE_ROOT_PATH: + raise ValueError( + "INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set" + ) + + logger.info("Getting raw dataset from SciCat to link to") + # Get the raw dataset to link to + try: + raw_dataset = self.scicat_client.datasets_get_one(raw_dataset_id) + logger.info(f"Found raw dataset to link: {raw_dataset_id}") + except Exception as e: + raise ValueError(f"Failed to find raw dataset with ID {raw_dataset_id}: {e}") + + folder_path_obj = Path(folder_path) + if not folder_path_obj.exists(): + raise ValueError(f"Folder path does not exist: {folder_path}") + + logger.info(raw_dataset) + # Calculate access controls - use the same as the raw dataset + access_controls = { + "owner_group": raw_dataset["ownerGroup"], + "access_groups": raw_dataset["accessGroups"] + } + logger.info(f"Using access controls from raw dataset: {access_controls}") + + ownable = Ownable( + ownerGroup=access_controls["owner_group"], + accessGroups=access_controls["access_groups"], + ) + + # Get main HDF5 file if exists, otherwise use first file + main_file = None + for file in folder_path_obj.glob("*.h5"): + main_file = file + break + + if not main_file: + # If no HDF5 file, use the first file in the directory + for file in folder_path_obj.iterdir(): + if file.is_file(): + main_file = file + break + if not main_file: + raise ValueError(f"No files found in directory: {folder_path}") + + # Extract scientific metadata + scientific_metadata = { + "derived_from": raw_dataset_id, + "processing_date": get_file_mod_time(main_file), + } + + # Try to extract metadata from HDF5 file if available + if main_file and main_file.suffix.lower() == ".h5": + try: + with h5py.File(main_file, "r") as file: + scientific_metadata.update( + self._extract_fields(file, self.SCIENTIFIC_METADATA_KEYS, issues) + ) + except Exception as e: + logger.warning(f"Could not extract metadata from HDF5 file: {e}") + + # Encode scientific metadata using NPArrayEncoder + encoded_scientific_metadata = json.loads( + json.dumps(scientific_metadata, cls=NPArrayEncoder) + ) + + # Create and upload the derived dataset + + # Use folder name as dataset name if nothing better + dataset_name = folder_path_obj.name + + # Determine if this is a TIFF directory or a Zarr directory + is_zarr = dataset_name.endswith('.zarr') + data_format = "Zarr" if is_zarr else "TIFF" + + # Build description/keywords from the folder name + description = build_search_terms(dataset_name) + keywords = description.split() + + # Add additional descriptive information + if is_zarr: + description = f"Multi-resolution Zarr dataset derived from reconstructed tomography slices: {description}" + keywords.extend(["zarr", "multi-resolution", "volume"]) + else: + description = f"Reconstructed tomography slices: {description}" + keywords.extend(["tiff", "slices", "reconstruction"]) + + # Create the derived dataset + dataset = DerivedDataset( + owner=raw_dataset.get("owner"), + contactEmail=raw_dataset.get("contactEmail"), + creationLocation=raw_dataset.get("creationLocation"), + datasetName=dataset_name, + type=DatasetType.derived, + proposalId=raw_dataset.get("proposalId"), + dataFormat=data_format, + principalInvestigator=raw_dataset.get("principalInvestigator"), + sourceFolder=str(folder_path_obj), + size=sum(f.stat().st_size for f in folder_path_obj.glob("**/*") if f.is_file()), + scientificMetadata=encoded_scientific_metadata, + sampleId=description, + isPublished=False, + description=description, + keywords=keywords, + creationTime=get_file_mod_time(folder_path_obj), + investigator=raw_dataset.get("owner"), + inputDatasets=[raw_dataset_id], + usedSoftware=["TomoPy", "Zarr"] if is_zarr else ["TomoPy"], + jobParameters={"source_folder": str(folder_path_obj)}, + **ownable.dict(), + ) + # Upload the derived dataset + dataset_id = self.scicat_client.upload_new_dataset(dataset) + logger.info(f"Created derived dataset with ID: {dataset_id}") + + # Upload datablock for all files in the directory + total_size = 0 + datafiles = [] + + for file_path in folder_path_obj.glob("**/*"): + if file_path.is_file(): + storage_path = str(file_path).replace(INGEST_SOURCE_ROOT_PATH, INGEST_STORAGE_ROOT_PATH) + datafile = DataFile( + path=storage_path, + size=get_file_size(file_path), + time=get_file_mod_time(file_path), + type="DerivedDatasets", + ) + datafiles.append(datafile) + total_size += datafile.size + + # Upload the datablock + datablock = CreateDatasetOrigDatablockDto( + size=total_size, + dataFileList=datafiles, + datasetId=dataset_id, + **ownable.dict(), + ) + self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) + logger.info(f"Uploaded datablock with {len(datafiles)} files") + + # Try to generate and upload a thumbnail if possible + try: + if is_zarr: + # For Zarr, we could extract a central slice or create a representative image + zarr_thumbnail_path = self._generate_zarr_thumbnail(folder_path_obj) + if zarr_thumbnail_path: + with open(zarr_thumbnail_path, 'rb') as thumb_file: + encoded_thumbnail = encode_image_2_thumbnail(thumb_file) + self._upload_attachment( + encoded_thumbnail, + dataset_id, + ownable, + ) + logger.info("Uploaded thumbnail for Zarr dataset") + elif main_file and main_file.suffix.lower() == ".h5": + with h5py.File(main_file, "r") as file: + # Try to find a suitable dataset for thumbnail + for key in ["/exchange/data", "/data", "/reconstruction"]: + if key in file: + thumbnail_file = build_thumbnail(file[key][0]) + encoded_thumbnail = encode_image_2_thumbnail(thumbnail_file) + self._upload_attachment( + encoded_thumbnail, + dataset_id, + ownable, + ) + logger.info("Uploaded thumbnail for derived dataset") + break + else: + # For TIFF files, use a middle slice as thumbnail + tiff_files = sorted(list(folder_path_obj.glob("*.tiff"))) + sorted(list(folder_path_obj.glob("*.tif"))) + if tiff_files: + # Use a slice from the middle of the volume for the thumbnail + middle_slice = tiff_files[len(tiff_files) // 2] + from PIL import Image + import io + image = Image.open(middle_slice) + if image.mode == "F": + image = image.convert("L") + thumbnail_buffer = io.BytesIO() + image.save(thumbnail_buffer, format="PNG") + thumbnail_buffer.seek(0) + encoded_thumbnail = encode_image_2_thumbnail(thumbnail_buffer) + self._upload_attachment(encoded_thumbnail, dataset_id, ownable) + + logger.info("Uploaded thumbnail from TIFF slice") + except Exception as e: + logger.warning(f"Failed to generate thumbnail: {e}") + + if issues: + for issue in issues: + logger.error(issue) + raise Exception(f"SciCat derived dataset ingest failed with {len(issues)} issues") + + return dataset_id + + def _generate_zarr_thumbnail(self, zarr_path: Path): + """ + Generate a thumbnail image from a Zarr dataset. + This implementation extracts a middle slice and converts it to a PNG. + """ + try: + # Ensure the zarr module is available + import zarr + except ImportError: + logger.warning("Zarr package is not installed. Install it with `pip install zarr`.") + return None + + try: + import numpy as np + from PIL import Image + import tempfile + + # Open the Zarr dataset + z = zarr.open(str(zarr_path), mode='r') + + # Find the main data array - typically at the highest resolution + if hasattr(z, 'keys'): + if '0' in z: + data = z['0'] + else: + # Find the first array-like object + for key in z.keys(): + if isinstance(z[key], zarr.core.Array): + data = z[key] + break + else: + return None + elif isinstance(z, zarr.core.Array): + data = z + else: + return None + + # Extract a slice from the middle of the volume + if data.ndim == 3: + middle_idx = data.shape[1] // 2 + slice_data = data[0, middle_idx, :] + elif data.ndim == 4: + middle_idx = data.shape[2] // 2 + slice_data = data[0, 0, middle_idx, :] + else: + if data.shape[0] > 0: + slice_data = data[0] + else: + return None + + # Normalize the data for visualization + slice_data = slice_data.astype(np.float32) + slice_data = (slice_data - np.min(slice_data)) / (np.max(slice_data) - np.min(slice_data) + 1e-8) + slice_data = (slice_data * 255).astype(np.uint8) + + # Create an image and convert if necessary + img = Image.fromarray(slice_data) + if img.mode == "F": + img = img.convert("L") + + # Save to a temporary file + temp_file = tempfile.NamedTemporaryFile(suffix='.png', delete=False) + img.save(temp_file.name) + temp_file.close() + return temp_file.name + except Exception as e: + logger.warning(f"Failed to generate Zarr thumbnail: {e}") + return None def _calculate_access_controls( self, @@ -473,3 +752,12 @@ def _upload_raw_dataset( # ingestor needs to add new "origdatablock" method for raw data on different filesystems # ingestor needs to add new "datablock" method for raw data on HPSS system # same for derived data + + ingestor.ingest_new_derived_dataset( + folder_path="/Users/david/Documents/data/tomo/scratch/rec20230606_152011_jong-seto_fungal-mycelia_flat-AQ_fungi2_fast.zarr", + raw_dataset_id=id + ) + ingestor.ingest_new_derived_dataset( + folder_path="/Users/david/Documents/data/tomo/scratch/rec20230224_132553_sea_shell", + raw_dataset_id=id + ) From 004546128d218786f75b4f76414e057b2bdaa932 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 11 Mar 2025 09:56:04 -0700 Subject: [PATCH 042/128] Fixed thumbnails uploaded for derived tiff/zarr datasets in SciCat --- orchestration/flows/bl832/scicat_ingestor.py | 132 +++++++++---------- 1 file changed, 61 insertions(+), 71 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 45695e0e..13bab584 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -383,17 +383,12 @@ def ingest_new_derived_dataset( # Try to generate and upload a thumbnail if possible try: if is_zarr: - # For Zarr, we could extract a central slice or create a representative image - zarr_thumbnail_path = self._generate_zarr_thumbnail(folder_path_obj) - if zarr_thumbnail_path: - with open(zarr_thumbnail_path, 'rb') as thumb_file: - encoded_thumbnail = encode_image_2_thumbnail(thumb_file) - self._upload_attachment( - encoded_thumbnail, - dataset_id, - ownable, - ) - logger.info("Uploaded thumbnail for Zarr dataset") + # For Zarr, generate the thumbnail in memory + thumb_buffer = self._generate_zarr_thumbnail(folder_path_obj) + if thumb_buffer: + encoded_thumbnail = encode_image_2_thumbnail(thumb_buffer) + self._upload_attachment(encoded_thumbnail, dataset_id, ownable) + logger.info("Uploaded thumbnail for Zarr dataset") elif main_file and main_file.suffix.lower() == ".h5": with h5py.File(main_file, "r") as file: # Try to find a suitable dataset for thumbnail @@ -416,11 +411,25 @@ def ingest_new_derived_dataset( middle_slice = tiff_files[len(tiff_files) // 2] from PIL import Image import io + import numpy as np image = Image.open(middle_slice) - if image.mode == "F": - image = image.convert("L") + # Convert image to a numpy array + arr = np.array(image, dtype=np.float32) + + # Compute min and max; if they are equal, use a default scaling to avoid division by zero. + arr_min = np.min(arr) + arr_max = np.max(arr) + if arr_max == arr_min: + # In case of no contrast, simply use a zeros array or leave the image unchanged. + arr_scaled = np.zeros(arr.shape, dtype=np.uint8) + else: + # Normalize the array to 0-255 + arr_scaled = ((arr - arr_min) / (arr_max - arr_min) * 255).astype(np.uint8) + + # Create a new image from the scaled array + scaled_image = Image.fromarray(arr_scaled) thumbnail_buffer = io.BytesIO() - image.save(thumbnail_buffer, format="PNG") + scaled_image.save(thumbnail_buffer, format="PNG") thumbnail_buffer.seek(0) encoded_thumbnail = encode_image_2_thumbnail(thumbnail_buffer) self._upload_attachment(encoded_thumbnail, dataset_id, ownable) @@ -438,71 +447,51 @@ def ingest_new_derived_dataset( def _generate_zarr_thumbnail(self, zarr_path: Path): """ - Generate a thumbnail image from a Zarr dataset. - This implementation extracts a middle slice and converts it to a PNG. - """ - try: - # Ensure the zarr module is available - import zarr - except ImportError: - logger.warning("Zarr package is not installed. Install it with `pip install zarr`.") - return None + Generate a thumbnail image from an NGFF Zarr dataset using ngff_zarr. + This implementation extracts a mid-slice and returns a BytesIO buffer. + :param zarr_path: Path to the Zarr directory. + :return: A BytesIO object containing the PNG image data, or None on failure. + """ try: - import numpy as np + import ngff_zarr as nz from PIL import Image - import tempfile - - # Open the Zarr dataset - z = zarr.open(str(zarr_path), mode='r') - - # Find the main data array - typically at the highest resolution - if hasattr(z, 'keys'): - if '0' in z: - data = z['0'] - else: - # Find the first array-like object - for key in z.keys(): - if isinstance(z[key], zarr.core.Array): - data = z[key] - break - else: - return None - elif isinstance(z, zarr.core.Array): - data = z - else: - return None - - # Extract a slice from the middle of the volume - if data.ndim == 3: - middle_idx = data.shape[1] // 2 - slice_data = data[0, middle_idx, :] - elif data.ndim == 4: - middle_idx = data.shape[2] // 2 - slice_data = data[0, 0, middle_idx, :] + import numpy as np + import io + + # Load the multiscale image from the Zarr store + multiscales = nz.from_ngff_zarr(str(zarr_path)) + # Here we assume a specific scale index (e.g. 3) and take the mid-slice along the first dimension + # Adjust this index as needed for your dataset. + image = multiscales.images[3].data + middle_index = image.shape[0] // 2 + mid_slice = image[middle_index, :, :] + # Ensure we have a NumPy array + mid_slice = mid_slice.compute() if hasattr(mid_slice, "compute") else np.array(mid_slice) + + # Normalize the image to 8-bit + mid_slice = mid_slice.astype(np.float32) + dmin, dmax = np.min(mid_slice), np.max(mid_slice) + if dmax != dmin: + norm_array = ((mid_slice - dmin) / (dmax - dmin) * 255).astype(np.uint8) else: - if data.shape[0] > 0: - slice_data = data[0] - else: - return None - - # Normalize the data for visualization - slice_data = slice_data.astype(np.float32) - slice_data = (slice_data - np.min(slice_data)) / (np.max(slice_data) - np.min(slice_data) + 1e-8) - slice_data = (slice_data * 255).astype(np.uint8) + norm_array = np.zeros_like(mid_slice, dtype=np.uint8) - # Create an image and convert if necessary - img = Image.fromarray(slice_data) + # Create a PIL image from the normalized array + img = Image.fromarray(norm_array) if img.mode == "F": img = img.convert("L") - # Save to a temporary file - temp_file = tempfile.NamedTemporaryFile(suffix='.png', delete=False) - img.save(temp_file.name) - temp_file.close() - return temp_file.name + # Save the image to an in-memory bytes buffer + buffer = io.BytesIO() + img.save(buffer, format="PNG") + buffer.seek(0) + return buffer + except ImportError: + logger.warning("ngff_zarr package is not installed. Install it with `pip install ngff_zarr`.") + return None except Exception as e: - logger.warning(f"Failed to generate Zarr thumbnail: {e}") + logger.warning(f"Failed to generate Zarr thumbnail using ngff_zarr: {e}") return None def _calculate_access_controls( @@ -754,7 +743,8 @@ def _upload_raw_dataset( # same for derived data ingestor.ingest_new_derived_dataset( - folder_path="/Users/david/Documents/data/tomo/scratch/rec20230606_152011_jong-seto_fungal-mycelia_flat-AQ_fungi2_fast.zarr", + folder_path="/Users/david/Documents/data/tomo/scratch/" + "rec20230606_152011_jong-seto_fungal-mycelia_flat-AQ_fungi2_fast.zarr", raw_dataset_id=id ) ingestor.ingest_new_derived_dataset( From db6b4fcf8478c9932b868e3a14f3ee4391707c7a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 12 Mar 2025 16:49:27 -0700 Subject: [PATCH 043/128] Adding logic to the remove_dataset_location() function in ingestor_controller.py. For some reason, none of the default accounts that are configured with scicatlive seem to have deletion permissions, so any attempt results in a 403 Forbidden error. I will need to look into this further. --- orchestration/flows/bl832/scicat_ingestor.py | 11 ++++ .../flows/scicat/ingestor_controller.py | 57 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 13bab584..6201173a 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -751,3 +751,14 @@ def _upload_raw_dataset( folder_path="/Users/david/Documents/data/tomo/scratch/rec20230224_132553_sea_shell", raw_dataset_id=id ) + + admin_ingestor = TomographyIngestorController(config) + admin_ingestor.login_to_scicat( + scicat_base_url="http://localhost:3000/api/v3/", + scicat_user="archiveManager", + scicat_password="aman" + ) + admin_ingestor.remove_dataset_location( + dataset_id=id, + source_folder_host="HPSS", + ) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index ba66e317..1015d7db 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -194,12 +194,63 @@ def add_new_dataset_location( def remove_dataset_location( self, dataset_id: str = "", - source: str = "", + source_folder_host: str = "", ) -> bool: """ - + Remove a location from an existing dataset in SciCat. """ - pass + logger.info(f"Removing location with host {source_folder_host} from dataset {dataset_id}") + + try: + # Get the datablocks directly + datablocks = self.scicat_client.datasets_origdatablocks_get_one(dataset_id) + if not datablocks: + logger.warning(f"No datablocks found for dataset {dataset_id}") + return False + + # Find datablock matching the specified source_folder_host + matching_datablock = None + for datablock in datablocks: + for datafile in datablock.get("dataFileList", []): + file_path = datafile.get("path", "") + if source_folder_host in file_path or ( + "sourceFolderHost" in datablock and + datablock["sourceFolderHost"] == source_folder_host + ): + matching_datablock = datablock + break + if matching_datablock: + break + + if not matching_datablock: + logger.warning( + f"No datablock found for dataset {dataset_id} with source folder host {source_folder_host}" + ) + return False + + # Delete the datablock using its ID + datablock_id = matching_datablock.get("id") + if not datablock_id: + logger.error(f"Datablock found but has no ID for dataset {dataset_id}") + return False + + # Delete the datablock using the appropriate endpoint + response = self.scicat_client.datasets_delete(datablock_id) + if response: + logger.info(f"Successfully removed datablock {datablock_id} from dataset {dataset_id}") + return True + else: + logger.error(f"Failed to delete datablock {datablock_id} from dataset {dataset_id}") + return False + + except requests.exceptions.HTTPError as e: + if e.response.status_code == 403: + logger.error(f"Forbidden: You do not have permission to delete the datablock {datablock_id}") + else: + logger.error(f"HTTP error occurred: {e}") + except Exception as e: + logger.error(f"Failed to remove datablock from dataset {dataset_id}: {e}") + return False def _find_dataset( self, From f65b800f206576afe759a5766dad39b2cbeee93f Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 18 Mar 2025 14:45:57 -0700 Subject: [PATCH 044/128] Adding a test script (not pytest) for end-to-end validation of the controller classes (orchestration/tests/test_controllers_end_to_end.py). I kept it as minimal/generic as possible as a way to validate to the core logic and API calls to the main connected services (Globus, Prefect, SciCat). Still need to implement HPSS tests into this script. --- .../_tests/test_controllers_end_to_end.py | 604 ++++++++++++++++++ orchestration/flows/bl832/scicat_ingestor.py | 2 +- .../flows/scicat/ingestor_controller.py | 2 +- orchestration/prune_controller.py | 43 +- scripts/login_to_globus_and_prefect.sh | 5 +- 5 files changed, 635 insertions(+), 21 deletions(-) create mode 100644 orchestration/_tests/test_controllers_end_to_end.py diff --git a/orchestration/_tests/test_controllers_end_to_end.py b/orchestration/_tests/test_controllers_end_to_end.py new file mode 100644 index 00000000..a05a6890 --- /dev/null +++ b/orchestration/_tests/test_controllers_end_to_end.py @@ -0,0 +1,604 @@ +""" +End-to-end tests for transfer, prune, and ingest controllers. +These tests are designed to be as generic as possible and should work with any beamline configuration. + +""" + +from datetime import timedelta, datetime +import logging +import os +import shutil +from typing import Optional + +from pyscicat.client import ScicatClient + +from orchestration.config import BeamlineConfig +from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController +from orchestration.globus.transfer import GlobusEndpoint +from orchestration.prune_controller import get_prune_controller, PruneMethod +from orchestration.transfer_controller import get_transfer_controller, CopyMethod +from orchestration.transfer_endpoints import FileSystemEndpoint +from globus_sdk import TransferClient + +from orchestration.globus import flows, transfer + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +# ---------------------------------------------------------------------------------------------------------------------- +# Setup Environment Configuration and Test Classes +# ---------------------------------------------------------------------------------------------------------------------- + + +def check_required_envvars() -> bool: + """ + Check for required environment variables before running the end-to-end tests. + """ + missing_vars = [] + + # Check Globus environment variables + globus_vars = ['GLOBUS_CLIENT_ID', 'GLOBUS_CLIENT_SECRET'] + for var in globus_vars: + if not os.getenv(var): + missing_vars.append(var) + + # Check Prefect environment variables + prefect_vars = ['PREFECT_API_URL', 'PREFECT_API_KEY'] + for var in prefect_vars: + if not os.getenv(var): + missing_vars.append(var) + + # Check SciCat environment variables + scicat_vars = ['SCICAT_API_URL', 'SCICAT_INGEST_USER', 'SCICAT_INGEST_PASSWORD'] + for var in scicat_vars: + if not os.getenv(var): + missing_vars.append(var) + + if missing_vars: + logger.error(f"Missing required environment variables: {', '.join(missing_vars)}") + logger.error("Please set these variables before running the end-to-end tests.") + return False + + logger.info("All required environment variables are set.") + return True + + +class TestConfig(BeamlineConfig): + """ + Test configuration class for a beamline + """ + def __init__(self) -> None: + super().__init__(beamline_id="0.0.0") + + def _beam_specific_config(self) -> None: + self.endpoints = transfer.build_endpoints(self.config) + self.apps = transfer.build_apps(self.config) + self.tc: TransferClient = transfer.init_transfer_client(self.apps["als_transfer"]) + self.flow_client = flows.get_flows_client() + self.nersc_alsdev = self.endpoints["nersc_alsdev"] # root: /global/homes/a/alsdev/test_directory/ + self.hpss_alsdev = self.config["hpss_alsdev"] + self.scicat = self.config["scicat"] + + +class TestIngestorController(BeamlineIngestorController): + """ + Test ingestor controller class for SciCat that does very basic ingest operations. + + Works with scicatlive v3.2.5 + https://github.com/SciCatProject/scicatlive + """ + def __init__( + self, + config: TestConfig, + scicat_client: Optional[ScicatClient] = None + ) -> None: + super().__init__(config, scicat_client) + + def ingest_new_raw_dataset( + self, + file_path: str = "", + ) -> str: + """ + Create a minimal raw dataset in SciCat for the given file. + + Args: + file_path: Path to the file to ingest. + + Returns: + str: The SciCat ID of the created dataset. + """ + if not self.scicat_client: + logger.error("SciCat client not initialized. Call login_to_scicat first.") + raise ValueError("SciCat client not initialized. Call login_to_scicat first.") + + # Create minimal metadata for the dataset + from pyscicat.model import CreateDatasetOrigDatablockDto, DataFile, RawDataset + + filename = os.path.basename(file_path) + basename = os.path.splitext(filename)[0] + + logger.info(f"Creating raw dataset for {filename}") + + try: + # Create a RawDataset object directly with parameters + # Making sure to include principalInvestigator as a string + dataset = RawDataset( + owner="ingestor", + contactEmail="test@example.com", + creationLocation=f"/test/location/{basename}", + sourceFolder="/test/source/folder", + datasetName=basename, + type="raw", + proposalId="test-proposal", + description=f"Test dataset for {filename}", + ownerGroup="admin", + accessGroups=["admin"], + creationTime=datetime.now().isoformat(), + principalInvestigator="Test Investigator" # Add this required field + ) + + # Upload dataset to SciCat + dataset_id = self.scicat_client.upload_new_dataset(dataset) + + logger.info(f"Created raw dataset with ID: {dataset_id}") + + # Add a dummy file to the datablock + dummy_file = DataFile( + path=file_path, + size=1024, # Dummy size + time=datetime.now().isoformat() + ) + + datablock = CreateDatasetOrigDatablockDto( + size=1024, # Dummy size + dataFileList=[dummy_file] + ) + + # Attach the datablock to the dataset + self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) + logger.info(f"Added datablock to dataset {dataset_id}") + + return dataset_id + + except Exception as e: + logger.error(f"Error creating raw dataset: {e}") + raise e + + def ingest_new_derived_dataset( + self, + file_path: str = "", + raw_dataset_id: str = "", + ) -> str: + """ + Create a minimal derived dataset in SciCat for the given file, + linked to the provided raw dataset. + + Args: + file_path: Path to the file to ingest. + raw_dataset_id: ID of the parent raw dataset. + + Returns: + str: The SciCat ID of the created dataset. + """ + if not self.scicat_client: + logger.error("SciCat client not initialized. Call login_to_scicat first.") + raise ValueError("SciCat client not initialized. Call login_to_scicat first.") + + # Create minimal metadata for the dataset + from pyscicat.model import CreateDatasetOrigDatablockDto, DataFile, DerivedDataset + + filename = os.path.basename(file_path) + basename = os.path.splitext(filename)[0] + + logger.info(f"Creating derived dataset for {filename} from {raw_dataset_id}") + + try: + # Create a DerivedDataset object + derived_dataset = DerivedDataset( + owner="ingestor", + contactEmail="test@example.com", + creationLocation=f"/test/location/{basename}_derived", + sourceFolder="/test/source/folder", + datasetName=f"{basename}_derived", + type="derived", + proposalId="test-proposal", + description=f"Derived dataset from {raw_dataset_id}", + ownerGroup="admin", + accessGroups=["admin"], + creationTime=datetime.now().isoformat(), + investigator="test-investigator", + inputDatasets=[raw_dataset_id], + principalInvestigator="Test Investigator", + usedSoftware=["TestSoftware"] + ) + + # Upload the dataset to SciCat + dataset_id = self.scicat_client.upload_new_dataset(derived_dataset) + + logger.info(f"Created derived dataset with ID: {dataset_id}") + + # Add a dummy file to the datablock + dummy_file = DataFile( + path=file_path, + size=1024, # Dummy size + time=datetime.now().isoformat() + ) + + datablock = CreateDatasetOrigDatablockDto( + size=1024, # Dummy size + dataFileList=[dummy_file] + ) + + # Attach the datablock to the dataset + self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) + logger.info(f"Added datablock to dataset {dataset_id}") + + return dataset_id + + except Exception as e: + logger.error(f"Error creating derived dataset: {e}") + raise e + + +# ---------------------------------------------------------------------------------------------------------------------- +# End-to-end Tests +# ---------------------------------------------------------------------------------------------------------------------- + + +def test_transfer_controllers( + file_path: str, + test_globus: bool, + test_filesystem: bool, + test_hpss: bool, + config: BeamlineConfig, +) -> None: + """ + Test the transfer controller by transferring a file to each endpoint. + + Args: + file_path (str): The path to the file to transfer. + test_globus (bool): Whether to test the Globus transfer controller. + test_filesystem (bool): Whether to test the FileSystem transfer controller. + test_hpss (bool): Whether to test the HPSS transfer controller. + config (BeamlineConfig): The beamline configuration. + + Returns: + None + """ + logger.info("Testing transfer controllers...") + logger.info(f"File path: {file_path}") + logger.info(f"Test Globus: {test_globus}") + logger.info(f"Test Filesystem: {test_filesystem}") + logger.info(f"Test HPSS: {test_hpss}") + + if test_globus: + # Create a transfer controller for Globus transfers + globus_transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.GLOBUS, + config=config + ) + + # Configure the source and destination endpoints + # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/source/ as the source + source_endpoint = GlobusEndpoint( + uuid=config.nersc_alsdev.uuid, + uri=config.nersc_alsdev.uri, + root_path=config.nersc_alsdev.root_path + "source/", + name="source_endpoint" + ) + + # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/destination/ as the destination + destination_endpoint = GlobusEndpoint( + uuid=config.nersc_alsdev.uuid, + uri=config.nersc_alsdev.uri, + root_path=config.nersc_alsdev.root_path + "destination/", + name="destination_endpoint" + ) + + globus_transfer_controller.copy( + file_path=file_path, + source=source_endpoint, + destination=destination_endpoint, + ) + + if test_filesystem: + # Create a transfer controller for filesystem transfers + filesystem_transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.SIMPLE, + config=config + ) + + # Configure the source and destination endpoints + + # Create temporary directories for testing + base_test_dir = os.path.join(os.getcwd(), "orchestration_test_dir") + source_dir = os.path.join(base_test_dir, "source") + dest_dir = os.path.join(base_test_dir, "destination") + + # Create directories + os.makedirs(source_dir, exist_ok=True) + os.makedirs(dest_dir, exist_ok=True) + + # Create a test file in the source directory + test_file_path = os.path.join(source_dir, file_path) + with open(test_file_path, "w") as f: + f.write("This is a test file for SimpleTransferController") + + logger.info(f"Created test file at {test_file_path}") + + # Use the defined FileSystemEndpoint + source_endpoint = FileSystemEndpoint( + name="source_endpoint", + root_path=source_dir, + uri="source.test" + ) + destination_endpoint = FileSystemEndpoint( + name="destination_endpoint", + root_path=dest_dir, + uri="destination.test" + ) + + result = filesystem_transfer_controller.copy( + file_path=file_path, + source=source_endpoint, + destination=destination_endpoint, + ) + + # Verify the transfer + dest_file_path = os.path.join(dest_dir, file_path) + if os.path.exists(dest_file_path): + logger.info(f"File successfully transferred to {dest_file_path}") + else: + logger.error(f"Transfer failed: file not found at {dest_file_path}") + + assert result is True, "Transfer operation returned False" + assert os.path.exists(dest_file_path), "File wasn't copied to destination" + + if test_hpss: + hpss_transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.CFS_TO_HPSS, + config=config + ) + + hpss_transfer_controller.copy() + # TODO: Finish this test + + +def test_prune_controllers( + file_path: str, + test_globus: bool, + test_filesystem: bool, + test_hpss: bool, + config: BeamlineConfig, +) -> None: + """ + Test the prune controllers by pruning files from each endpoint. + + Note: not pruning the source endpoint test.txt file, so it can be used in future tests. + + Args: + file_path (str): Path to the file to prune. + test_globus (bool): Whether to test the Globus pruner. + test_filesystem (bool): Whether to test the filesystem pruner. + test_hpss (bool): Whether to test the HPSS pruner. + config (BeamlineConfig): Configuration object for the beam + + Returns: + None + """ + logger.info("Testing prune controllers...") + logger.info(f"File path: {file_path}") + logger.info(f"Test Globus: {test_globus}") + logger.info(f"Test Filesystem: {test_filesystem}") + logger.info(f"Test HPSS: {test_hpss}") + + if test_globus: + globus_prune_controller = get_prune_controller( + prune_type=PruneMethod.GLOBUS, + config=config + ) + + # Configure the source and destination endpoints + # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/source/ as the source + # source_endpoint = GlobusEndpoint( + # uuid=config.nersc_alsdev.uuid, + # uri=config.nersc_alsdev.uri, + # root_path=config.nersc_alsdev.root_path + "source/", + # name="source_endpoint" + # ) + + # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/destination/ as the destination + destination_endpoint = GlobusEndpoint( + uuid=config.nersc_alsdev.uuid, + uri=config.nersc_alsdev.uri, + root_path=config.nersc_alsdev.root_path + "destination/", + name="destination_endpoint" + ) + # Assume files were created and transferred in the previous test + + # Prune the source endpoint + # globus_prune_controller.prune( + # file_path=file_path, + # source_endpoint=source_endpoint, + # check_endpoint=None, + # days_from_now=timedelta(days=0) + # ) + + # Prune the destination endpoint + globus_prune_controller.prune( + file_path=file_path, + source_endpoint=destination_endpoint, + check_endpoint=None, + days_from_now=timedelta(days=0) + ) + + if test_filesystem: + filesystem_prune_controller = get_prune_controller( + prune_type=PruneMethod.SIMPLE, + config=config + ) + + # Configure the source and destination endpoints to match the TransferController test + # Create temporary directories for testing + base_test_dir = os.path.join(os.getcwd(), "orchestration_test_dir") + source_dir = os.path.join(base_test_dir, "source") + dest_dir = os.path.join(base_test_dir, "destination") + + # Use the defined FileSystemEndpoint + source_endpoint = FileSystemEndpoint( + name="source_endpoint", + root_path=source_dir, + uri="source.test" + ) + + destination_endpoint = FileSystemEndpoint( + name="destination_endpoint", + root_path=dest_dir, + uri="destination.test" + ) + + # Assume files were created and transferred in the previous test + + # Prune the source endpoint + filesystem_prune_controller.prune( + file_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=None, + days_from_now=timedelta(days=0) + ) + + # Prune the destination endpoint + filesystem_prune_controller.prune( + file_path=file_path, + source_endpoint=destination_endpoint, + check_endpoint=None, + days_from_now=timedelta(days=0) + ) + + # After pruning in the filesystem pruner test + source_file_path = os.path.join(source_dir, file_path) + assert not os.path.exists(source_file_path), "File wasn't removed from source" + if not os.path.exists(source_file_path): + logger.info(f"File successfully deleted from source: {source_file_path}") + dest_file_path = os.path.join(dest_dir, file_path) + assert not os.path.exists(dest_file_path), "File wasn't removed from destination" + if not os.path.exists(dest_file_path): + logger.info(f"File successfully deleted from destination: {dest_file_path}") + + if test_hpss: + hpss_prune_controller = get_prune_controller( + prune_type=PruneMethod.HPSS, + config=config + ) + + hpss_prune_controller.prune() + # TODO: Finish this test + + +def test_scicat_ingest( + file_path: str = "test.txt" + +) -> None: + """ + Test the SciCat ingestor controller by ingesting a file. + """ + config = TestConfig() + test_ingestor = TestIngestorController(config) + + # Login to SciCat, assuming credentials are saved in environment variables + # If not, and you are testing with scicatlive, use these defaults: + # SCICAT_API_URL="http://localhost:3000/api/v3/" + # SCICAT_INGEST_USER="admin" + # SCICAT_INGEST_PASSWORD="2jf70TPNZsS" + + test_ingestor.login_to_scicat( + scicat_base_url=os.getenv("SCICAT_API_URL"), + scicat_user=os.getenv("SCICAT_INGEST_USER"), + scicat_password=os.getenv("SCICAT_INGEST_PASSWORD") + ) + + raw_id = test_ingestor.ingest_new_raw_dataset( + file_path=file_path + ) + + test_ingestor.ingest_new_derived_dataset( + file_path=file_path, + raw_dataset_id=raw_id + ) + + test_ingestor.add_new_dataset_location( + file_name=file_path, + dataset_id=raw_id, + source_folder="test_folder", + source_folder_host="test_host" + ) + + # This will probably fail. Need to figure out default scicatlive user permissions. + test_ingestor.remove_dataset_location( + dataset_id=raw_id, + source_folder_host="test_host" + ) + + +def test_it_all( + test_globus: bool = True, + test_filesystem: bool = False, + test_hpss: bool = False, + test_scicat: bool = True +) -> None: + """ + Run end-to-end tests for transfer and prune controllers." + """ + try: + check_required_envvars() + except Exception as e: + logger.error(f"Error checking environment variables: {e}") + return + finally: + logger.info("Continuing with tests...") + + config = TestConfig() + + try: + test_transfer_controllers( + file_path="test.txt", + test_globus=test_globus, + test_filesystem=test_filesystem, + test_hpss=test_hpss, + config=config + ) + logger.info("Transfer controller tests passed.") + except Exception as e: + logger.error(f"Error running transfer controller tests: {e}") + return + + try: + test_prune_controllers( + file_path="test.txt", + test_globus=test_globus, + test_filesystem=test_filesystem, + test_hpss=test_hpss, + config=config + ) + logger.info("Prune controller tests passed.") + except Exception as e: + logger.error(f"Error running prune controller tests: {e}") + return + + if test_scicat: + try: + test_scicat_ingest( + file_path="test.txt" + ) + except Exception as e: + logger.error(f"Error running SciCat ingestor tests: {e}") + return + + logger.info("All tests passed. Cleaning up...") + base_test_dir = os.path.join(os.getcwd(), "orchestration_test_dir") + if os.path.exists(base_test_dir): + shutil.rmtree(base_test_dir) + + +if __name__ == "__main__": + test_it_all() diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 6201173a..cc7c6f1a 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -12,6 +12,7 @@ CreateDatasetOrigDatablockDto, Datablock, DataFile, + DerivedDataset, RawDataset, DatasetType, Ownable, @@ -19,7 +20,6 @@ from orchestration.flows.bl832.config import Config832 from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController -from pyscicat.model import DerivedDataset from orchestration.flows.scicat.utils import ( build_search_terms, build_thumbnail, diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 1015d7db..4104de9d 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -73,7 +73,7 @@ def login_to_scicat( logger.info("Logged in to SciCat.") return self.scicat_client except Exception as e: - logger.error(f"Failed to log in to SciCat: {e}, trying alternative method.") + logger.warning(f"Failed to log in to SciCat: {e}, trying alternative method.") # This method works for scicatlive 3.2.5 try: diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 180866da..6ac2f71c 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -179,27 +179,34 @@ def _prune_filesystem_endpoint( logger.info(f"Running flow: prune_from_{source_endpoint.name}") logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") - # Check if the file exists at the source endpoint - if not source_endpoint.exists(relative_path): + # Check if the file exists at the source endpoint using os.path + source_full_path = source_endpoint.full_path(relative_path) + if not os.path.exists(source_full_path): logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") - return - - # Check if the file exists at the check endpoint - if check_endpoint is not None and check_endpoint.exists(relative_path): - logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") - logger.info("Safe to prune.") + return False - # Check if it is a file or directory - if source_endpoint.is_dir(relative_path): - logger.info(f"Pruning directory {relative_path}") - source_endpoint.rmdir(relative_path) + # If check_endpoint is provided, verify file exists there before pruning + if check_endpoint is not None: + check_full_path = check_endpoint.full_path(relative_path) + if os.path.exists(check_full_path): + logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") + logger.info("Safe to prune.") else: - logger.info(f"Pruning file {relative_path}") - os.remove(source_endpoint.full_path(relative_path)) + logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") + logger.warning("Not safe to prune.") + return False + + # Now perform the pruning operation + if os.path.isdir(source_full_path): + logger.info(f"Pruning directory {relative_path}") + import shutil + shutil.rmtree(source_full_path) else: - logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") - logger.warning("Not safe to prune.") - return + logger.info(f"Pruning file {relative_path}") + os.remove(source_full_path) + + logger.info(f"Successfully pruned {relative_path} from {source_endpoint.name}") + return True class GlobusPruneController(PruneController[GlobusEndpoint]): @@ -263,7 +270,7 @@ def prune( # If days_from_now is 0, prune immediately if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") - return self._prune_filesystem_endpoint( + return self._prune_globus_endpoint( relative_path=file_path, source_endpoint=source_endpoint, check_endpoint=check_endpoint, diff --git a/scripts/login_to_globus_and_prefect.sh b/scripts/login_to_globus_and_prefect.sh index dbc57f9e..75ae958f 100755 --- a/scripts/login_to_globus_and_prefect.sh +++ b/scripts/login_to_globus_and_prefect.sh @@ -17,4 +17,7 @@ export GLOBUS_CLI_CLIENT_SECRET="$GLOBUS_CLIENT_SECRET" export GLOBUS_COMPUTE_CLIENT_ID="$GLOBUS_CLIENT_ID" export GLOBUS_COMPUTE_CLIENT_SECRET="$GLOBUS_CLIENT_SECRET" export PREFECT_API_KEY="$PREFECT_API_KEY" -export PREFECT_API_URL="$PREFECT_API_URL" \ No newline at end of file +export PREFECT_API_URL="$PREFECT_API_URL" +export SCICAT_API_URL="$SCICAT_API_URL" +export SCICAT_INGEST_USER="$SCICAT_INGEST_USER" +export SCICAT_INGEST_PASSWORD="$SCICAT_INGEST_PASSWORD" \ No newline at end of file From 05ed15a6bac7332717a6409ebe72bd976804ccb1 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 18 Mar 2025 14:48:42 -0700 Subject: [PATCH 045/128] Moved test_controllers_end_to_end.py to the scripts/ folder does it does not conflict with pytest. --- .../_tests => scripts}/test_controllers_end_to_end.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename {orchestration/_tests => scripts}/test_controllers_end_to_end.py (100%) diff --git a/orchestration/_tests/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py similarity index 100% rename from orchestration/_tests/test_controllers_end_to_end.py rename to scripts/test_controllers_end_to_end.py index a05a6890..2c00f745 100644 --- a/orchestration/_tests/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -14,13 +14,13 @@ from orchestration.config import BeamlineConfig from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController +from orchestration.globus import flows, transfer from orchestration.globus.transfer import GlobusEndpoint from orchestration.prune_controller import get_prune_controller, PruneMethod from orchestration.transfer_controller import get_transfer_controller, CopyMethod from orchestration.transfer_endpoints import FileSystemEndpoint from globus_sdk import TransferClient -from orchestration.globus import flows, transfer logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) From ded7becfa914645b9c25b4d72049a366101da66c Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 7 Apr 2025 13:59:08 -0700 Subject: [PATCH 046/128] Addressing Dylan and Raja's comments --- docs/mkdocs/docs/hpss.md | 24 +++++++++++++++++++ docs/mkdocs/docs/tomography_workflow.md | 8 ++++++- .../flows/scicat/ingestor_controller.py | 10 ++------ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index d56607ce..9b4d1d6b 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -2,12 +2,36 @@ HPSS is the tape-based data storage system we use for long term storage of experimental data at the ALS. Tape storage, while it may seem antiquated, is still a very economical and secure medium for infrequently accessed data as tape does not need to be powered except for reading and writing. This requires certain considerations when working with this system. + + ## Overview **Purpose:** Archive and retrieve large experimental datasets using HPSS. **Approach:** Use HPSS tools (hsi and htar) within a structured transfer framework orchestrated via SFAPI and SLURM jobs. **Key Considerations:** File sizes should typically be between 100 GB and 2 TB. Larger projects are segmented into multiple archives. +### "User" in this context + +It is important to clarify who users are when we talk about transferring to tape. In terms of the flows we support, that includes beamline scientists, visiting users, and computing staff. In this context, it's important to differentiate between who is collecting the data and who is doing the work of moving to and from tape. + +**NERSC Users** + - Can move data to and from HPSS via `htar` and `hsi` commands on Perlmutter in a terminal, in Jupyter, or in a script via SFAPI as outlined below. + - There are limitations and caveats to interacting with the tape system that users should be aware of. + +**ALS Users** + - Generate data! + - Sometimes they are also NERSC users, and can move data to HPSS if they want. + - Either way, we support the long term storage of data that they collect by archiving it on HPSS. + +**Splash Flows Globus Users** + - Can use the Prefect Flows and Slurm scripts provided to help perform transfers to HPSS in an automated way. + - Have transparent and reproducible knowledge on where data is stored on tape. + - Perform transfers that bundle data in a way that is optimized for tape storage and retrieval. + - Apply it across different beamlines. + +**Service Users** + - We use use a service account at NERSC for automating our transfers. This "service" user can perform the same sets of tasks as other NERSC users, but has wider access to data systems. ALS Users benefit from, but do not directly interact with this account. + In `orchestration/transfer_controller.py` we have included two transfer classes for moving data from CFS to HPSS and vice versa (HPSS to CFS). We are following the [HPSS best practices](https://docs.nersc.gov/filesystems/HPSS-best-practices/) outlined in the NERSC documentation. diff --git a/docs/mkdocs/docs/tomography_workflow.md b/docs/mkdocs/docs/tomography_workflow.md index d50ef8ac..108e0b2d 100644 --- a/docs/mkdocs/docs/tomography_workflow.md +++ b/docs/mkdocs/docs/tomography_workflow.md @@ -16,6 +16,7 @@ flowchart LR n20["data832"] n21["NERSC CFS"] n22@{ label: "SciCat
[Metadata Database]" } + n46["spot832"] end subgraph s3["NERSC Reconstruction [Prefect Flow]"] n28["NERSC CFS"] @@ -36,7 +37,6 @@ flowchart LR end n17 -- Raw Data [Globus Transfer] --> n18 n23["spot832"] -- File Watcher --> n24["Dispatcher
[Prefect Worker]"] - n23 -- "Raw Data [Globus Transfer]" --> n20 n25["Detector"] -- Raw Data --> n23 n24 --> s2 & s1 & s3 & s4 n20 -- Raw Data [Globus Transfer] --> n21 @@ -60,11 +60,13 @@ flowchart LR n43 -- Recon Data --> n44 n43 -- Metadata [SciCat Ingestion] --> n45 n45 -- Hyperlink --> n44 + n46 -- "Raw Data [Globus Transfer]" --> n20 n17@{ shape: internal-storage} n18@{ shape: disk} n20@{ shape: internal-storage} n21@{ shape: disk} n22@{ shape: db} + n46@{ shape: internal-storage} n28@{ shape: disk} n29@{ shape: disk} n42@{ shape: internal-storage} @@ -90,6 +92,9 @@ flowchart LR n20:::Peach n21:::Sky n22:::Sky + n46:::collection + n46:::storage + n46:::Peach n28:::Sky n29:::storage n29:::Sky @@ -126,4 +131,5 @@ flowchart LR style s3 stroke:#757575 style s4 stroke:#757575 style s5 stroke:#757575 + ``` \ No newline at end of file diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 4104de9d..9ec28484 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -114,7 +114,7 @@ def ingest_new_raw_dataset( def ingest_new_derived_dataset( self, file_path: str = "", - raw_dataset_id: str = "", + raw_dataset_id: Optional[str] = "", ) -> str: """Ingest data from the beamline. @@ -125,9 +125,7 @@ def ingest_new_derived_dataset( def add_new_dataset_location( self, - dataset_id: Optional[str] = None, - proposal_id: Optional[str] = None, - file_name: Optional[str] = None, + dataset_id: str = None, source_folder: str = None, source_folder_host: str = None ) -> str: @@ -143,10 +141,6 @@ def add_new_dataset_location( optionally including a protocol e.g. [protocol://]fileserver1.example.com", """ - # If dataset_id is not provided, we need to find it using file_name. - if dataset_id is None and file_name: - dataset_id = self._find_dataset(proposal_id=proposal_id, file_name=file_name) - # Get the dataset to retrieve its metadata dataset = self.scicat_client.datasets_get_one(dataset_id) if not dataset: From 99d64cf64154187b87a30bc54f1bd3db8a37f900 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 23 Apr 2025 14:51:23 -0700 Subject: [PATCH 047/128] Updating end-to-end tests --- scripts/test_controllers_end_to_end.py | 106 ++++++++++++++++++------- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index 2c00f745..995e07a2 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -16,9 +16,10 @@ from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController from orchestration.globus import flows, transfer from orchestration.globus.transfer import GlobusEndpoint +from orchestration.hpss import cfs_to_hpss_flow, hpss_to_cfs_flow from orchestration.prune_controller import get_prune_controller, PruneMethod from orchestration.transfer_controller import get_transfer_controller, CopyMethod -from orchestration.transfer_endpoints import FileSystemEndpoint +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint from globus_sdk import TransferClient @@ -54,6 +55,8 @@ def check_required_envvars() -> bool: if not os.getenv(var): missing_vars.append(var) + # TODO: Add SFAPI Keys check + if missing_vars: logger.error(f"Missing required environment variables: {', '.join(missing_vars)}") logger.error("Please set these variables before running the end-to-end tests.") @@ -239,6 +242,8 @@ def ingest_new_derived_dataset( logger.error(f"Error creating derived dataset: {e}") raise e + # TODO: Add methods to add and remove dataset locations + # ---------------------------------------------------------------------------------------------------------------------- # End-to-end Tests @@ -310,7 +315,7 @@ def test_transfer_controllers( # Configure the source and destination endpoints - # Create temporary directories for testing + # Create temporary directories for testing in current working directory base_test_dir = os.path.join(os.getcwd(), "orchestration_test_dir") source_dir = os.path.join(base_test_dir, "source") dest_dir = os.path.join(base_test_dir, "destination") @@ -355,13 +360,54 @@ def test_transfer_controllers( assert os.path.exists(dest_file_path), "File wasn't copied to destination" if test_hpss: - hpss_transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.CFS_TO_HPSS, + from orchestration.flows.bl832.config import Config832 + + config = Config832() + project_name = "BLS-00520_dyparkinson" + source = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", + uri="nersc.gov" + ) + destination = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + success = cfs_to_hpss_flow( + file_path=project_name, + source=source, + destination=destination, config=config ) + logger.info(f"Transfer success: {success}") + config = Config832() + relative_file_path = f"{config.beamline_id}/raw/BLS-00520_dyparkinson/BLS-00520_dyparkinson_2022-2.tar" + source = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], # root_path: /home/a/alsdev/data_mover + uri=config.hpss_alsdev["uri"] + ) + destination = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/retrieved_from_tape", + uri="nersc.gov" + ) - hpss_transfer_controller.copy() - # TODO: Finish this test + files_to_extract = [ + "20221028_101514_arun_JSC-1.h5", + "20220923_160531_ethan_robin_climbing-vine_x00y05.h5", + "20221222_082548_strangpresse_20pCFABS_800rpm_Non-vacuum.h5", + "20220923_160531_ethan_robin_climbing-vine_x00y04.h5" + ] + + hpss_to_cfs_flow( + file_path=f"{relative_file_path}", + source=source, + destination=destination, + files_to_extract=files_to_extract, + config=config + ) def test_prune_controllers( @@ -398,6 +444,7 @@ def test_prune_controllers( config=config ) + # PRUNE FROM SOURCE ENDPOINT # Configure the source and destination endpoints # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/source/ as the source # source_endpoint = GlobusEndpoint( @@ -407,15 +454,6 @@ def test_prune_controllers( # name="source_endpoint" # ) - # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/destination/ as the destination - destination_endpoint = GlobusEndpoint( - uuid=config.nersc_alsdev.uuid, - uri=config.nersc_alsdev.uri, - root_path=config.nersc_alsdev.root_path + "destination/", - name="destination_endpoint" - ) - # Assume files were created and transferred in the previous test - # Prune the source endpoint # globus_prune_controller.prune( # file_path=file_path, @@ -424,6 +462,16 @@ def test_prune_controllers( # days_from_now=timedelta(days=0) # ) + # PRUNE FROM DESTINATION ENDPOINT + # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/destination/ as the destination + destination_endpoint = GlobusEndpoint( + uuid=config.nersc_alsdev.uuid, + uri=config.nersc_alsdev.uri, + root_path=config.nersc_alsdev.root_path + "destination/", + name="destination_endpoint" + ) + + # Assume files were created and transferred in the previous test # Prune the destination endpoint globus_prune_controller.prune( file_path=file_path, @@ -527,7 +575,6 @@ def test_scicat_ingest( ) test_ingestor.add_new_dataset_location( - file_name=file_path, dataset_id=raw_id, source_folder="test_folder", source_folder_host="test_host" @@ -544,7 +591,7 @@ def test_it_all( test_globus: bool = True, test_filesystem: bool = False, test_hpss: bool = False, - test_scicat: bool = True + test_scicat: bool = False ) -> None: """ Run end-to-end tests for transfer and prune controllers." @@ -572,6 +619,15 @@ def test_it_all( logger.error(f"Error running transfer controller tests: {e}") return + if test_scicat: + try: + test_scicat_ingest( + file_path="test.txt" + ) + except Exception as e: + logger.error(f"Error running SciCat ingestor tests: {e}") + return + try: test_prune_controllers( file_path="test.txt", @@ -585,15 +641,6 @@ def test_it_all( logger.error(f"Error running prune controller tests: {e}") return - if test_scicat: - try: - test_scicat_ingest( - file_path="test.txt" - ) - except Exception as e: - logger.error(f"Error running SciCat ingestor tests: {e}") - return - logger.info("All tests passed. Cleaning up...") base_test_dir = os.path.join(os.getcwd(), "orchestration_test_dir") if os.path.exists(base_test_dir): @@ -601,4 +648,9 @@ def test_it_all( if __name__ == "__main__": - test_it_all() + test_it_all( + test_globus=False, + test_filesystem=False, + test_hpss=True, + test_scicat=False + ) From edfe26fc4c3098879da7bd9e2381db751bbd39fd Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 23 Apr 2025 14:52:16 -0700 Subject: [PATCH 048/128] For the HPSS->CFS controller, convert slashes to underscores for the log file name to make it easier to find --- orchestration/hpss.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index b3fe2b37..c2f79878 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -15,10 +15,11 @@ import datetime import logging +import os from pathlib import Path import re import time -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from prefect import flow from sfapi_client import Client @@ -796,6 +797,18 @@ def copy( logger.error("Missing required parameters: file_path, source, or destination.") return False + # Sanitize the file_path for the log file names. + # Build a small ordā†’char map + translate_dict: Dict[int, str] = { + ord("/"): "_", # always replace forward slash + ord(" "): "_", # replace spaces + ord(os.sep): "_", # replace primary separator + ord("\\"): "_" # In case of Windows + } + + # One-pass replacement + sanitized_path = file_path.translate(translate_dict) + # Compute the full HPSS path from the source endpoint. hpss_path = source.full_path(file_path) dest_root = destination.root_path @@ -814,13 +827,13 @@ def copy( # else MODE is "tar". # - Otherwise, MODE is "single" and hsi get is used. job_script = fr"""#!/bin/bash -#SBATCH -q xfer # Specify the SLURM queue to u +#SBATCH -q xfer # Specify the SLURM queue #SBATCH -A als # Specify the account. #SBATCH -C cron # Use the 'cron' constraint. #SBATCH --time=12:00:00 # Maximum runtime of 12 hours. #SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/{file_path}_from_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{file_path}_from_hpss_%j.err # Standard error log file. +#SBATCH --output={logs_path}/{sanitized_path}_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{sanitized_path}_from_hpss_%j.err # Standard error log file. #SBATCH --licenses=SCRATCH # Request the SCRATCH license. #SBATCH --mem=20GB # Request #GB of memory. Default 2GB. set -euo pipefail # Enable strict error checking. @@ -881,16 +894,16 @@ def copy( if [ "$MODE" = "single" ]; then echo "[LOG] Single file detected. Using hsi get." - # mkdir -p "$DEST_ROOT" - # hsi get "$SOURCE_PATH" "$DEST_ROOT/" + mkdir -p "$DEST_ROOT" + hsi get "$SOURCE_PATH" "$DEST_ROOT/" elif [ "$MODE" = "tar" ]; then echo "[LOG] Tar archive detected. Extracting entire archive using htar." ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" echo "[LOG] Extracting to: $DEST_PATH" - # mkdir -p "$DEST_PATH" - # htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" + mkdir -p "$DEST_PATH" + htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" elif [ "$MODE" = "partial" ]; then echo "[LOG] Partial extraction detected. Extracting selected files using htar." ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") From 60250b89ee91706c6e421c89d94f16dc9c8f0c28 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 23 Apr 2025 16:38:42 -0700 Subject: [PATCH 049/128] Added filter for files >65GB to be moved with HTAR. Verified that this works. --- orchestration/hpss.py | 74 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index c2f79878..2959dbd8 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -452,8 +452,10 @@ def copy( if not proposal_name or proposal_name == ".": # if file_path is in the root directory proposal_name = file_path - logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + logger.info(f"Proposal name derived from file path: {proposal_name}") + logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + logger.info(f"Logs will be saved to: {logs_path}") # Build the SLURM job script with detailed inline comments for clarity. job_script = rf"""#!/bin/bash # ------------------------------------------------------------------ @@ -489,6 +491,7 @@ def copy( # ------------------------------------------------------------------ # Define source and destination variables. # ------------------------------------------------------------------ + echo "[LOG] Defining source and destination paths." # SOURCE_PATH: Full path of the file or directory on CFS. @@ -564,6 +567,7 @@ def copy( # ------------------------------------------------------------------ # Transfer Logic: Check if SOURCE_PATH is a file or directory. # ------------------------------------------------------------------ + echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" if [ -f "$SOURCE_PATH" ]; then # Case: Single file detected. @@ -575,8 +579,17 @@ def copy( elif [ -d "$SOURCE_PATH" ]; then # Case: Directory detected. echo "[LOG] Directory detected. Initiating bundling process." + + # ------------------------------------------------------------------ + # Define thresholds + # - THRESHOLD: maximum total size per HTAR archive (2 TB). + # - MEMBER_LIMIT: maximum size per member file in an HTAR (set to 65 GB). + # ------------------------------------------------------------------ + THRESHOLD=2199023255552 # 2 TB in bytes. - echo "[LOG] Threshold set to 2 TB (in bytes: $THRESHOLD)" + MEMBER_LIMIT=$((65*1024**3)) # 65 GB in bytes. 68 GB is the htar limit. Move files >65 GB than this using hsi cput. + echo "[LOG] Threshold set to 2 TB (bytes): $THRESHOLD" + echo "[LOG] Threshold for individual file transfer (bytes): $MEMBER_LIMIT" # ------------------------------------------------------------------ # Generate a list of relative file paths in the project directory. @@ -596,6 +609,7 @@ def copy( # # 3. The output is then redirected into the temporary file specified by FILE_LIST. # ------------------------------------------------------------------ + echo "[LOG] Grouping files by modification date." FILE_LIST=$(mktemp) @@ -603,6 +617,62 @@ def copy( echo "[LOG] List of files stored in temporary file: $FILE_LIST" + # ------------------------------------------------------------------ + # Filter out oversized files (>65GB) for immediate transfer + # - For each file: + # ā€¢ If fsize > MEMBER_LIMIT: transfer via hsi cput. + # ā€¢ Else: add path to new list for bundling. + # ------------------------------------------------------------------ + + echo "[LOG] Beginning oversized-file filtering (> $MEMBER_LIMIT bytes)" + FILTERED_LIST=$(mktemp) + echo "[LOG] Writing remaining file paths to $FILTERED_LIST" + + while IFS= read -r f; do + # Absolute local path and size + full_local="$SOURCE_PATH/$f" + fsize=$(stat -c %s "$full_local") + + if (( fsize > MEMBER_LIMIT )); then + # Relative subdirectory and filename + rel_dir=$(dirname "$f") + fname=$(basename "$f") + + # Compute HPSS directory under project (create if needed) + if [ "$rel_dir" = "." ]; then + dest_dir="$DEST_PATH" + else + dest_dir="$DEST_PATH/$rel_dir" + fi + + if ! hsi -q "ls $dest_dir" >/dev/null 2>&1; then + echo "[LOG] Creating HPSS directory $dest_dir" + hsi mkdir "$dest_dir" + fi + + # Full remote file path (directory + filename) + remote_file="$dest_dir/$fname" + + # Transfer via conditional put + echo "[LOG] Transferring oversized file '$f' ($fsize bytes) to HPSS path $remote_file" + echo "[DEBUG] hsi cput \"$full_local\" : \"$remote_file\"" + hsi cput "$full_local" : "$remote_file" + echo "[LOG] Completed hsi cput for '$f'." + else + # Keep for bundling later + echo "$f" >> "$FILTERED_LIST" + fi + done < "$FILE_LIST" + + # Swap in the filtered list and report + mv "$FILTERED_LIST" "$FILE_LIST" + remaining=$(wc -l < "$FILE_LIST") + echo "[LOG] Oversized-file transfer done. Remaining for bundling: $remaining files." + + # ------------------------------------------------------------------ + # Cycle-based grouping & tar-bundling logic (unchanged). + # ------------------------------------------------------------------ + # Declare associative arrays to hold grouped file paths and sizes. declare -A group_files declare -A group_sizes From 07663de3bab78520c76a7811aa0975462ab0ba20 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 23 Apr 2025 16:45:01 -0700 Subject: [PATCH 050/128] Updating .env.example and README with current environment requirements. --- .env.example | 22 +++++++++++++++------- README.md | 16 ++++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index e3728e89..abb006d8 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,15 @@ -GLOBUS_CLIENT_ID= -GLOBUS_CLIENT_SECRET= -PREFECT_API_URL= -PREFECT_API_KEY= -PUSHGATEWAY_URL= -JOB_NAME= -INSTANCE_LABEL= \ No newline at end of file +GLOBUS_CLIENT_ID= # For Globus Transfer +GLOBUS_CLIENT_SECRET= # For Globus Transfer +GLOBUS_COMPUTE_CLIENT_ID= # For ALCF Jobs +GLOBUS_COMPUTE_CLIENT_SECRET= # For ALCF Jobs +GLOBUS_COMPUTE_ENDPOINT= # For ALCF Jobs +PREFECT_API_URL= # For Prefect Flows +PREFECT_API_KEY= # For Prefect Flows +SCICAT_API_URL= # For SciCat Ingest +SCICAT_INGEST_USER= # For SciCat Ingest +SCICAT_INGEST_PASSWORD= # For SciCat Ingest +PATH_NERSC_CLIENT_ID= # For NERSC SFAPI +PATH_NERSC_PRI_KEY= # For NERSC SFAPI +PUSHGATEWAY_URL= # For Grafana Pushgateway +JOB_NAME= # For Grafana Pushgateway +INSTANCE_LABEL= # For Grafana Pushgateway diff --git a/README.md b/README.md index d1745784..ba1e4453 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,18 @@ $ pip3 install -e . Use `.env.example` as a template. ``` -GLOBUS_CLIENT_ID= -GLOBUS_CLIENT_SECRET= -PREFECT_API_URL= -PREFECT_API_KEY= +GLOBUS_CLIENT_ID= # For Globus Transfer +GLOBUS_CLIENT_SECRET= # For Globus Transfer +GLOBUS_COMPUTE_CLIENT_ID= # For ALCF Jobs +GLOBUS_COMPUTE_CLIENT_SECRET= # For ALCF Jobs +GLOBUS_COMPUTE_ENDPOINT= # For ALCF Jobs +PREFECT_API_URL= # For Prefect Flows +PREFECT_API_KEY= # For Prefect Flows +SCICAT_API_URL= # For SciCat Ingest +SCICAT_INGEST_USER= # For SciCat Ingest +SCICAT_INGEST_PASSWORD= # For SciCat Ingest +PATH_NERSC_CLIENT_ID= # For NERSC SFAPI, generate on https://iris.nersc.gov/ +PATH_NERSC_PRI_KEY= # For NERSC SFAPI ``` ## Current workflow overview and status: From 72c7449cf91efed7d5198d24ea373ce55d9dba46 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 23 Apr 2025 16:47:52 -0700 Subject: [PATCH 051/128] Added comment that this will be moved to the scicat_beamline repo at some point --- orchestration/flows/scicat/ingestor_controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 9ec28484..3feb34bb 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -16,6 +16,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +# NOTE: This BeamlineIngestorController will be relocated to https://github.com/als-computing/scicat_beamline soon. + class BeamlineIngestorController(ABC): """ From 6ec45bbf797aea0ae87412f664e6ab90e2ebb1de Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 25 Apr 2025 14:00:30 -0700 Subject: [PATCH 052/128] Updating documentation for HPSS --- docs/mkdocs/docs/hpss.md | 117 +++++++++++++++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 12 deletions(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index 9b4d1d6b..b05f4d7b 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -1,7 +1,8 @@ -# Working with The High Performance Storage System (HPSS) at NERSC -HPSS is the tape-based data storage system we use for long term storage of experimental data at the ALS. Tape storage, while it may seem antiquated, is still a very economical and secure medium for infrequently accessed data as tape does not need to be powered except for reading and writing. This requires certain considerations when working with this system. +# Developing for the High Performance Storage System (HPSS) at NERSC + +HPSS is the tape-based data storage system we use for long term storage of experimental data at the ALS. Tape storage, while it may seem antiquated, is still a very economical and secure medium for infrequently accessed data as tape does not need to be powered except for reading and writing. This requires certain considerations when working with this system. ## Overview @@ -39,7 +40,99 @@ HPSS is intended for long-term storage of data that is not frequently accessed, While there are Globus endpoints for HPSS, the NERSC documentation recommends against it as there are certain conditions (i.e. network disconnection) that are not as robust as their recommended HPSS tools `hsi` and `htar`, which they say is the fastest approach. Together, these tools allow us to work with the HPSS filesystem and carefully bundle our projects into `tar` archives that are built directly on HPSS. Another couple of drawbacks to using Globus here is 1) if you have small files, you need to tar them regardless before transferring, and 2) HPSS does not support collab accounts (i.e. alsdev). -## Working with `hsi` +### Storing and Retrieving from Tape +#### Important note about retrieval + +In production, we are using the `alsdev` collab account. To run these commands, you must have a valid SFAPI client/key pair from Iris at NERSC in your environment. + +**Files are stored on HPC in the following location:** +- `/home/a/alsdev/data_mover` + +**Data retrieved from tape will be found here on NERSC CFS:** +- `/global/cfs/cdirs/als/data_mover/8.3.2/retrieved_from_tape` + +**Logs about data transfers to/from tape are organized here on CFS:** +- `/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}` +- Find details about whether files were stored via hsi or htar + +----------------------- + + +In `orchestraiton/hpss.py` there are two Prefect flows, which utilize two special TransferController classes for interacting with HPSS: + +#### `cfs_to_hpss_flow` + Prefect flow for transferring data from CFS to HPSS tape archive. + + This flow handles the transfer of files or directories from NERSC's Community + File System (CFS) to the High Performance Storage System (HPSS) tape archive. + For directories, files are bundled into tar archives based on time periods. + + Args: + file_path (Union[str, List[str]]): A single file path or a list of file paths to transfer + source (FileSystemEndpoint): The CFS source endpoint + destination (HPSSEndpoint): The HPSS destination endpoint + config (BeamlineConfig): The beamline configuration containing endpoints and credentials + + Returns: + bool: True if all transfers succeeded, False otherwise + +#### `hpss_to_cfs_flow` + Prefect flow for retrieving data from HPSS tape archive to CFS. + + This flow handles the retrieval of files or tar archives from NERSC's High + Performance Storage System (HPSS) to the Community File System (CFS). + For tar archives, you can optionally specify specific files to extract. + + Args: + file_path (str): The path of the file or tar archive on HPSS + source (HPSSEndpoint): The HPSS source endpoint + destination (FileSystemEndpoint): The CFS destination endpoint + files_to_extract (Optional[List[str]]): Specific files to extract from the tar archive + config (BeamlineConfig): The beamline configuration containing endpoints and credentials + + Returns: + bool: True if the transfer succeeded, False otherwise + +#### `CFSToHPSSTransferController` + Use SFAPI, Slurm, hsi, and htar to move data from CFS to HPSS at NERSC. + + This controller requires the source to be a FileSystemEndpoint on CFS and the + destination to be an HPSSEndpoint. For a single file, the transfer is done using hsi (via hsi cput). + For a directory, the transfer is performed with htar. In this updated version, if the source is a + directory then the files are bundled into tar archives based on their modification dates as follows: + - Files with modification dates between Jan 1 and Jul 15 (inclusive) are grouped together + (Cycle 1 for that year). + - Files with modification dates between Jul 16 and Dec 31 are grouped together (Cycle 2). + + Within each group, if the total size exceeds 2 TB the files are partitioned into multiple tar bundles. + The resulting naming convention on HPSS is: + + /home/a/alsdev/data_mover/[beamline]/raw/[proposal_name]/ + [proposal_name]_[year]-[cycle].tar + [proposal_name]_[year]-[cycle]_part0.tar + [proposal_name]_[year]-[cycle]_part1.tar + ... + + At the end of the SLURM script, the directory tree for both the source (CFS) and destination (HPSS) + is echoed for logging purposes. + +#### `HPSSToCFSTransferController` + Use SFAPI, Slurm, hsi and htar to move data between HPSS and CFS at NERSC. + + This controller retrieves data from an HPSS source endpoint and places it on a CFS destination endpoint. + It supports the following modes: + - "single": Single file retrieval via hsi get. + - "tar": Full tar archive extraction via htar -xvf. + - "partial": Partial extraction from a tar archive: if a list of files is provided (via files_to_extract), + only the specified files will be extracted. + + A single SLURM job script is generated that branches based on the mode. + + + +## Developer Notes about HPSS + +### Working with `hsi` We use `hsi` for handling individual files on HPSS. [Here is the official NERSC documentation for `hsi`.](https://docs.nersc.gov/filesystems/hsi/) @@ -72,7 +165,7 @@ Find files that are more than 20 days old and redirects the output to the file t hsi -q "find . -ctime 20" > temp.txt 2>&1 ``` -## Working with `htar` +### Working with `htar` We can use `htar` to efficiently work with groups of files on HPSS. The basic syntax of `htar` is similar to the standard `tar` utility: @@ -125,11 +218,11 @@ If your `htar` files are >100GB, and you only want to extract one or two small m htar -Hnostage -xvf archive.tar project_directory/cool_scan4 ``` -## Transferring Data from CFS to HPSS +### Transferring Data from CFS to HPSS NERSC provides a special `xfer` QOS ("Quality of Service") for interacting with HPSS, which we can use with our SFAPI Slurm job scripts. -### Single Files +#### Single Files We can transfer single files over to HPSS using `hsi put` in a Slurm script: @@ -152,7 +245,7 @@ Notes: - NERSC users are at most allowed 15 concurrent `xfer` sessions, which can be used strategically for parallel transfers and reads. -### Multiple Files +#### Multiple Files NERSC recommends that when serving many files smaller than 100 GB we use `htar` to bundle them together before archiving. Since individual scans within a project may not be this large, we try to archive all of the scans in a project into a single `tar` file. If projects end up being larger than 2 TB, we can create multiple `tar` files. @@ -171,7 +264,7 @@ One great part about `htar` is that it builds the archive directly on `HPSS`, so htar -cvf als_user_project.tar /global/cfs/cdirs/als/data_mover/8.3.2/raw/als_user_project_folder ``` -## Transferring Data from HPSS to CFS +### Transferring Data from HPSS to CFS At some point you may want to access data from HPSS. An important thing to consider is whether you need to access single or multiple files. @@ -187,11 +280,11 @@ Or maybe a single file htar -xvf als_user_project_folder.tar cool_scan1.h5 ``` -## Prefect Flows for HPSS Transfers +### Prefect Flows for HPSS Transfers Most of the time we expect transfers to occur from CFS to HPSS on a scheduled basis, after users have completed scanning during their alotted beamtime. -### Transfer to HPSS Implementation +#### Transfer to HPSS Implementation **`orchestration/transfer_controller.py`:** - **`CFSToHPSSTransferController()`**: This controller uses a Slurm Job Script and SFAPI to launch the tape transfer job. The Slurm script handles the specific logic for handling single and multiple files, on a project by project basis. It reads the files sizes, and creates bundles that are <= 2TB. The groups within each tar archive are saved in a log on NERSC CFS for posterity. @@ -275,7 +368,7 @@ flowchart TD S -- "No" --> U ``` -### Transfer to CFS Implementation +#### Transfer to CFS Implementation **`orchestration/transfer_controller.py`:** - **`CFSToHPSSTransferController()`**: This controller uses a Slurm Job Script and SFAPI to copy data from tape to NERSC CFS. The Slurm script handles the specific logic for handling single and multiple files, on a project by project basis. Based on the file path, the Slurm job determines whether a single file or a tar archive has been requested (or even specific files within a tar archive), and run the correct routine to copy the data to CFS. @@ -284,7 +377,7 @@ flowchart TD - **`cfs_to_hpss_flow()`** This Prefect Flow sets up the HPSSToCFSTransferController() and calls the copy command. By registering this Flow, the HPSS transfers to CFS can be easily scheduled. While copying from CFS to HPSS is likely not going to be automated, it is still helpful to have this as a Prefect Flow to simplify data access in low-code manner. -## Update SciCat with HPSS file paths +### Update SciCat with HPSS file paths `BeamlineIngestorController()` in `orchestration/flows/scicat/ingestor_controller.py` contains a method `add_new_dataset_location()` that can be used to update the source folder and host metadata in SciCat with new HPSS location: From de1702b0641ea64218b14fe9f98a62d7c0224b0a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 25 Apr 2025 14:04:31 -0700 Subject: [PATCH 053/128] Updating documentation for HPSS --- docs/mkdocs/docs/hpss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index b05f4d7b..54d89adb 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -53,7 +53,7 @@ In production, we are using the `alsdev` collab account. To run these commands, **Logs about data transfers to/from tape are organized here on CFS:** - `/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}` -- Find details about whether files were stored via hsi or htar +- Find details about whether files were stored via hsi or htar, and what the path is on HPSS, etc. ----------------------- From e98c17b283e8251023371acdac5999e05f8c5dd4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 29 Apr 2025 16:09:30 -0700 Subject: [PATCH 054/128] Adding ability to hpss controllers for ls command to see what's on tape --- orchestration/hpss.py | 170 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 2959dbd8..338c4242 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -194,10 +194,92 @@ def hpss_to_cfs_flow( return False +# ---------------------------------- +# HPSS ls Function +# ---------------------------------- + +def list_hpss_slurm( + client: Client, + endpoint: HPSSEndpoint, + remote_path: str, + recursive: bool = True +) -> str: + """ + Schedule and run a Slurm job on Perlmutter to list contents on HPSS, + then read back the result from the Slurm output file. + + If `remote_path` ends with '.tar', uses `htar -tvf` to list tar members; + otherwise uses `hsi ls [-R]` to list directory contents. + + Args: + client (Client): SFAPI client with compute permissions. + endpoint (HPSSEndpoint): HPSS endpoint (knows root_path & URI). + remote_path (str): Path relative to endpoint.root_path on HPSS. + recursive (bool): Recursively list directories (ignored for .tar). + + Returns: + . + + Raises: + RuntimeError: If job submission or output retrieval fails. + """ + logger = logging.getLogger(__name__) + # Build logs directory on CFS + beamline_id = remote_path.split("/")[0] + logs_dir = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}/ls" + + # Sanitize remote_path for filenames + safe_name = re.sub(r'[^A-Za-z0-9_]', '_', remote_path) + job_name = f"ls_hpss_{safe_name}" + out_pattern = f"{logs_dir}/{safe_name}_%j.out" + err_pattern = f"{logs_dir}/{safe_name}_%j.err" + + full_hpss = endpoint.full_path(remote_path) + + # for tar: list contents & then show .idx; otherwise do an hsi ls + if remote_path.lower().endswith(".tar"): + cmd = ( + f'echo "[LOG] TAR contents:" && htar -tvf "{full_hpss}"' + ) + else: + ls_flag = "-R" if recursive else "" + cmd = f'hsi ls {ls_flag} "{full_hpss}"' + + job_script = rf"""#!/bin/bash +#SBATCH -q xfer +#SBATCH -A als +#SBATCH -C cron +#SBATCH --time=00:10:00 +#SBATCH --job-name={job_name} +#SBATCH --output={out_pattern} +#SBATCH --error={err_pattern} + +set -euo pipefail + +echo "[LOG] Listing HPSS path: {full_hpss}" +{cmd} +""" + + # submit & wait + perlmutter = client.compute(Machine.perlmutter) + job = perlmutter.submit_job(job_script) + try: + job.update() + except Exception: + logger.debug("Initial job.update() failed, proceeding to wait") + job.complete() + + # print where you can find the actual log + out_file = out_pattern.replace("%j", str(job.jobid)) + print(f"HPSS listing complete. See Slurm output at: {out_file}") + + return out_file + # ---------------------------------- # HPSS Prune Controller # ---------------------------------- + class HPSSPruneController(PruneController[HPSSEndpoint]): """ Controller for pruning files from HPSS tape archive. @@ -416,6 +498,30 @@ def __init__( super().__init__(config) self.client = client + def list_hpss( + self, + endpoint: HPSSEndpoint, + remote_path: str, + recursive: bool = True + ) -> List[str]: + """ + Schedule and run a Slurm job to list contents on HPSS. + + Args: + endpoint (HPSSEndpoint): HPSS endpoint (knows root_path & URI). + remote_path (str): Path under endpoint.root_path to list. + recursive (bool): If True, pass -R to `hsi ls` (ignored for tar). + + Returns: + List[str]: Lines of output from the listing command. + """ + return list_hpss_slurm( + client=self.client, + endpoint=endpoint, + remote_path=remote_path, + recursive=recursive + ) + def copy( self, file_path: str = None, @@ -840,6 +946,30 @@ def __init__( super().__init__(config) self.client = client + def list_hpss( + self, + endpoint: HPSSEndpoint, + remote_path: str, + recursive: bool = True + ) -> List[str]: + """ + Schedule and run a Slurm job to list contents on HPSS. + + Args: + endpoint (HPSSEndpoint): HPSS endpoint (knows root_path & URI). + remote_path (str): Path under endpoint.root_path to list. + recursive (bool): If True, pass -R to `hsi ls` (ignored for tar). + + Returns: + List[str]: Lines of output from the listing command. + """ + return list_hpss_slurm( + client=self.client, + endpoint=endpoint, + remote_path=remote_path, + recursive=recursive + ) + def copy( self, file_path: str = None, @@ -1049,6 +1179,7 @@ def copy( TEST_HPSS_PRUNE = False TEST_CFS_TO_HPSS = False TEST_HPSS_TO_CFS = False + TEST_HPSS_LS = True # ------------------------------------------------------ # Test pruning from HPSS @@ -1129,3 +1260,42 @@ def copy( files_to_extract=files_to_extract, config=config ) + + # ------------------------------------------------------ + # Test listing HPSS files + # ------------------------------------------------------ + if TEST_HPSS_LS: + from orchestration.flows.bl832.config import Config832 + + # Build client, config, endpoint + config = Config832() + endpoint = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + + # Instantiate controller + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.CFS_TO_HPSS, + config=config + ) + + # Directory listing + project_path = f"{config.beamline_id}/raw/BLS-00564_dyparkinson" + logger.info("Controller-based directory listing on HPSS:") + output_file = transfer_controller.list_hpss( + endpoint=endpoint, + remote_path=project_path, + recursive=True + ) + + # TAR archive listing + archive_name = project_path.split("/")[-1] + tar_path = f"{project_path}/{archive_name}_2023-1.tar" + logger.info("Controller-based tar archive listing on HPSS:") + output_file = transfer_controller.list_hpss( + endpoint=endpoint, + remote_path=tar_path, + recursive=False + ) From 44455ac24b456a17c6439d5ff45cd704299fde9b Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 5 May 2025 13:14:20 -0700 Subject: [PATCH 055/128] Fixed commenting --- scripts/test_controllers_end_to_end.py | 51 +++++++++++++++++--------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index 995e07a2..e5b57da0 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -55,7 +55,11 @@ def check_required_envvars() -> bool: if not os.getenv(var): missing_vars.append(var) - # TODO: Add SFAPI Keys check + # Check NERSC SFAPI environment variables + nersc_vars = ['PATH_NERSC_CLIENT_ID', 'PATH_NERSC_PRI_KEY'] + for var in nersc_vars: + if not os.getenv(var): + missing_vars.append(var) if missing_vars: logger.error(f"Missing required environment variables: {', '.join(missing_vars)}") @@ -363,7 +367,7 @@ def test_transfer_controllers( from orchestration.flows.bl832.config import Config832 config = Config832() - project_name = "BLS-00520_dyparkinson" + project_name = "BLS-00564_dyparkinson" source = FileSystemEndpoint( name="CFS", root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", @@ -447,20 +451,20 @@ def test_prune_controllers( # PRUNE FROM SOURCE ENDPOINT # Configure the source and destination endpoints # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/source/ as the source - # source_endpoint = GlobusEndpoint( - # uuid=config.nersc_alsdev.uuid, - # uri=config.nersc_alsdev.uri, - # root_path=config.nersc_alsdev.root_path + "source/", - # name="source_endpoint" - # ) + source_endpoint = GlobusEndpoint( + uuid=config.nersc_alsdev.uuid, + uri=config.nersc_alsdev.uri, + root_path=config.nersc_alsdev.root_path + "source/", + name="source_endpoint" + ) # Prune the source endpoint - # globus_prune_controller.prune( - # file_path=file_path, - # source_endpoint=source_endpoint, - # check_endpoint=None, - # days_from_now=timedelta(days=0) - # ) + globus_prune_controller.prune( + file_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=None, + days_from_now=timedelta(days=0) + ) # PRUNE FROM DESTINATION ENDPOINT # Use the NERSC alsdev endpoint with root_path: /global/homes/a/alsdev/test_directory/destination/ as the destination @@ -648,9 +652,22 @@ def test_it_all( if __name__ == "__main__": - test_it_all( + + # Uncomment the following line to run all tests + # Set test_globus, test_filesystem, test_hpss, and test_scicat to True or False as needed + + # test_it_all( + # test_globus=False, + # test_filesystem=False, + # test_hpss=False, + # test_scicat=False + # ) + + # Test individual transfer controllers directly + test_transfer_controllers( + file_path="test.txt", test_globus=False, test_filesystem=False, - test_hpss=True, - test_scicat=False + test_hpss=False, + config=TestConfig() ) From 49d7d2a674d1420103140dbd3be0a9a9973fa4a4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 5 May 2025 13:52:52 -0700 Subject: [PATCH 056/128] fixed pytest after rebasing --- .../_tests/test_transfer_controller.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index fb9c8570..4bc27395 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -13,14 +13,11 @@ from .test_globus import MockTransferClient -<<<<<<< HEAD -======= @pytest.fixture(autouse=True) def fast_sleep(monkeypatch): """Patch time.sleep to return immediately to speed up tests.""" monkeypatch.setattr(time, "sleep", lambda x: None) ->>>>>>> 67e515a (Added logic for HPSSToCFSTransferController() copy() method. Now it will launch an SFAPI Slurm job script that handles each case: a single file is requested (non-tar), an entire tar file, or specified files within a tar (partial). Added pytest cases in test_transfer_controller.py for the new HPSS controllers.) @pytest.fixture(autouse=True, scope="session") def prefect_test_fixture(): @@ -209,7 +206,8 @@ def test_globus_transfer_controller_copy_failure( mocker.patch('prefect.blocks.system.Secret.load', return_value=MockSecretClass()) - with patch("orchestration.transfer_controller.start_transfer", return_value=(False, "mock-task-id")) as mock_start_transfer: + with patch("orchestration.transfer_controller.start_transfer", + return_value=(False, "mock-task-id")) as mock_start_transfer: controller = GlobusTransferController(mock_config832) result = controller.copy( file_path="some_dir/test_file.txt", @@ -244,6 +242,7 @@ def test_globus_transfer_controller_copy_exception( assert result is False, "Expected False when TransferAPIError is raised." mock_start_transfer.assert_called_once() + def test_globus_transfer_controller_with_metrics( mock_config832, mock_globus_endpoint, transfer_controller_module ): @@ -253,30 +252,30 @@ def test_globus_transfer_controller_with_metrics( GlobusTransferController = transfer_controller_module["GlobusTransferController"] from orchestration.prometheus_utils import PrometheusMetrics mock_prometheus = MagicMock(spec=PrometheusMetrics) - + with patch("orchestration.transfer_controller.start_transfer", return_value=(True, "mock-task-id")) as mock_start_transfer: # Create the controller with mock prometheus metrics controller = GlobusTransferController(mock_config832, prometheus_metrics=mock_prometheus) - + # Set up mock for get_transfer_file_info mock_transfer_info = {"bytes_transferred": 1024 * 1024} # 1MB controller.get_transfer_file_info = MagicMock(return_value=mock_transfer_info) - + # Execute the copy operation result = controller.copy( file_path="some_dir/test_file.txt", source=mock_globus_endpoint, destination=mock_globus_endpoint, ) - + # Verify transfer was successful assert result is True mock_start_transfer.assert_called_once() - + # Verify metrics were collected and pushed controller.get_transfer_file_info.assert_called_once_with("mock-task-id") mock_prometheus.push_metrics_to_prometheus.assert_called_once() - + # Verify the metrics data metrics_data = mock_prometheus.push_metrics_to_prometheus.call_args[0][0] assert metrics_data["bytes_transferred"] == 1024 * 1024 @@ -293,6 +292,7 @@ def test_globus_transfer_controller_with_metrics( # Tests for SimpleTransferController # -------------------------------------------------------------------------- + def test_simple_transfer_controller_no_file_path( mock_config832, mock_file_system_endpoint, transfer_controller_module ): From 9568274cbf216a7d87b52dc6e6d9e4f2e7046e55 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 5 May 2025 14:21:31 -0700 Subject: [PATCH 057/128] bumping python from 3.11->3.12.5 to see if it fixes a TypeError where type object is not iterable. I could not reproduce the pytest error locally, so I am bumping to the same version --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 1164dede..e160e455 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.11 + - name: Set up Python 3.12.5 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12.5 cache: 'pip' - name: Install dependencies run: | From af03920624d7347e5e7e01c50c4627ebd669a3e8 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 15 May 2025 10:54:53 -0700 Subject: [PATCH 058/128] Fix error message grammar --- orchestration/sfapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/sfapi.py b/orchestration/sfapi.py index b1d2f46f..a0e24e22 100644 --- a/orchestration/sfapi.py +++ b/orchestration/sfapi.py @@ -23,7 +23,7 @@ def create_sfapi_client( if not client_id_path or not client_secret_path: logger.error("NERSC credentials paths are missing.") - raise ValueError("Missing NERSC credentials paths.") + raise ValueError("NERSC credentials paths are missing.") if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): logger.error("NERSC credential files are missing.") raise FileNotFoundError("NERSC credential files are missing.") From 35dc149690e39308ebaee665c63663eda152490c Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 15 May 2025 10:56:15 -0700 Subject: [PATCH 059/128] Fixing which flow the globus prune controller calls --- orchestration/prune_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 6ac2f71c..82b2e031 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -283,7 +283,7 @@ def prune( try: schedule_prefect_flow( - deployment_name="prune_filesystem_endpoint/prune_filesystem_endpoint", + deployment_name="prune_globus_endpoint/prune_globus_endpoint", flow_run_name=flow_name, parameters={ "relative_path": file_path, From 7e808cf43ce82cf31834ef4e403e90ae98bfb5a6 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 15 May 2025 11:16:27 -0700 Subject: [PATCH 060/128] making the days_from_now parameter a float, which is converted into datetime.timedelta within the functions --- orchestration/prune_controller.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 82b2e031..8180a5ae 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -48,7 +48,7 @@ def prune( file_path: str = None, source_endpoint: Endpoint = None, check_endpoint: Optional[Endpoint] = None, - days_from_now: datetime.timedelta = 0 + days_from_now: float = 0.0 ) -> bool: """ Prune (delete) data from the source endpoint. @@ -60,7 +60,7 @@ def prune( file_path (str): The path to the file or directory to prune source_endpoint (Endpoint): The endpoint containing the data to be pruned check_endpoint (Optional[Endpoint]): If provided, verify data exists here before pruning - days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + days_from_now (float): Delay in days before pruning; if 0.0, prune immediately. Returns: bool: True if pruning was successful or scheduled successfully, False otherwise @@ -96,7 +96,7 @@ def prune( file_path: str = None, source_endpoint: FileSystemEndpoint = None, check_endpoint: Optional[FileSystemEndpoint] = None, - days_from_now: datetime.timedelta = 0 + days_from_now: float = 0.0, ) -> bool: """ Prune (delete) data from a file system endpoint. @@ -108,7 +108,7 @@ def prune( file_path (str): The path to the file or directory to prune source_endpoint (FileSystemEndpoint): The file system endpoint containing the data check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning - days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + days_from_now (float): Delay in days before pruning; if 0.0, prune immediately. Returns: bool: True if pruning was successful or scheduled successfully, False otherwise @@ -124,6 +124,9 @@ def prune( flow_name = f"prune_from_{source_endpoint.name}" logger.info(f"Setting up pruning of '{file_path}' from '{source_endpoint.name}'") + # convert float days ā†’ timedelta + days_from_now: datetime.timedelta = datetime.timedelta(days=days_from_now) + # If days_from_now is 0, prune immediately if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") @@ -237,7 +240,7 @@ def prune( file_path: str = None, source_endpoint: GlobusEndpoint = None, check_endpoint: Optional[GlobusEndpoint] = None, - days_from_now: datetime.timedelta = 0 + days_from_now: float = 0.0 ) -> bool: """ Prune (delete) data from a file system endpoint. @@ -267,6 +270,9 @@ def prune( flow_name = f"prune_from_{source_endpoint.name}" logger.info(f"Setting up pruning of '{file_path}' from '{source_endpoint.name}'") + # convert float days ā†’ timedelta + days_from_now: datetime.timedelta = datetime.timedelta(days=days_from_now) + # If days_from_now is 0, prune immediately if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") From 220b3a48ca393bdffda246023e78432d979490e3 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 15 May 2025 11:34:43 -0700 Subject: [PATCH 061/128] adding try and except to build_thumbnail, in case of edge cases (like divide by 0) --- orchestration/flows/scicat/utils.py | 36 ++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 86ef283a..7dcf0ebb 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -60,17 +60,31 @@ def build_thumbnail( image_array: npt.ArrayLike ) -> io.BytesIO: """Create a thumbnail from an image array.""" - image_array = image_array - np.min(image_array) + 1.001 - image_array = np.log(image_array) - image_array = 205 * image_array / (np.max(image_array)) - auto_contrast_image = Image.fromarray(image_array.astype("uint8")) - auto_contrast_image = ImageOps.autocontrast(auto_contrast_image, cutoff=0.1) - # filename = str(uuid4()) + ".png" - file = io.BytesIO() - # file = thumbnail_dir / Path(filename) - auto_contrast_image.save(file, format="png") - file.seek(0) - return file + try: + image_array = image_array - np.min(image_array) + 1.001 + image_array = np.log(image_array) + image_array = 205 * image_array / (np.max(image_array)) + auto_contrast_image = Image.fromarray(image_array.astype("uint8")) + auto_contrast_image = ImageOps.autocontrast(auto_contrast_image, cutoff=0.1) + # filename = str(uuid4()) + ".png" + file = io.BytesIO() + # file = thumbnail_dir / Path(filename) + auto_contrast_image.save(file, format="png") + file.seek(0) + return file + except Exception as e: + logger.error(f"build_thumbnail failed; returning blank image. Error: {e}", exc_info=True) + # determine original size (height, width) + try: + h, w = image_array.shape[:2] + except Exception: + h, w = 1, 1 + # create blank RGB image of same dimensions + blank = Image.new("RGB", (w, h), color=(0, 0, 0)) + buf = io.BytesIO() + blank.save(buf, format="PNG") + buf.seek(0) + return buf def calculate_access_controls( From 2121eb94853688d3b2ee366f3abbc3bf71191116 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:10:50 -0700 Subject: [PATCH 062/128] Updated documentation regarding hsi mkdir -p --- docs/mkdocs/docs/hpss.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index 54d89adb..df180b40 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -294,6 +294,7 @@ Most of the time we expect transfers to occur from CFS to HPSS on a scheduled ba - Recursively check each part of the incoming file path if the folder exists - If the folder does not exist, use `hsi mkdir` - Repeat until the file path is built + - Note: In the [hsi documentation](https://hpss-collaboration.org/wp-content/uploads/2023/09/hpss_hsi_10.2_reference_manual.pdf), there is a command `mkdir -p` to create\ missing intermediate path name directories. If the -p flag is not specified, the parent directory of each newly-created directory must already exist. This is not currently implemented, but something to consider in the future. 3. Determine if the source is a file or a directory. - If a file, transfer it using 'hsi cput'. - If a directory, group files by beam cycle and archive them. From afdcda03050e3d1cec93c880055edf78371b0e83 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:33:06 -0700 Subject: [PATCH 063/128] Addressing Garrett's comment about the add_new_dataset_location method. Making it less brittle by requiring the caller to compose the full path when calling the function --- .../flows/scicat/ingestor_controller.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 3feb34bb..29640c23 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -128,32 +128,32 @@ def ingest_new_derived_dataset( def add_new_dataset_location( self, dataset_id: str = None, - source_folder: str = None, - source_folder_host: str = None + datafile_path: str = None, + source_folder_host: Optional[str] = None ) -> str: """ Add a new location to an existing dataset in SciCat. :param dataset_id: SciCat ID of the dataset. - :param source_folder: "Absolute file path on file server containing the files of this dataset, - e.g. /some/path/to/sourcefolder. In case of a single file dataset, e.g. HDF5 data, - it contains the path up to, but excluding the filename. Trailing slashes are removed.", - + :param datafile_path: Absolute file path to the data file (excluding protocol/host). + Caller is responsible for full path composition, including filename. :param source_folder_host: "DNS host name of file server hosting sourceFolder, optionally including a protocol e.g. [protocol://]fileserver1.example.com", - + :return: The dataset ID after successful datablock addition. + :raises ValueError: If the dataset ID is not found or if the dataset does not have a valid 'pid'. """ # Get the dataset to retrieve its metadata dataset = self.scicat_client.datasets_get_one(dataset_id) if not dataset: raise ValueError(f"Dataset with ID {dataset_id} not found") - logger.info(f"Creating new datablock for dataset {dataset_id} at location {source_folder}") + logger.info(f"Creating new datablock for dataset {dataset_id} at location {datafile_path}") try: # Create a datafile for the new location - basename = dataset.get("datasetName", "dataset") - file_path = f"{source_folder}/{basename}" + file_path = datafile_path + if source_folder_host: + file_path = f"{source_folder_host}:{datafile_path}" # Get size from existing dataset if available size = dataset.get("size", 0) @@ -177,7 +177,7 @@ def add_new_dataset_location( # Upload the datablock self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) - logger.info(f"Created new datablock for dataset {dataset_id} at location {source_folder}") + logger.info(f"Created new datablock for dataset {dataset_id} at location {datafile_path}") # Note: We're skipping the dataset update since it's causing validation issues From 6286208998ac64b2d69d7cb64ceea2b34765cc3d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:35:55 -0700 Subject: [PATCH 064/128] Removing redunant datafile.path = file_path lines, as it is now configured at the top of the method, before calling datafile = DataFile(...) --- orchestration/flows/scicat/ingestor_controller.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 29640c23..64fcfcf0 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -171,10 +171,6 @@ def add_new_dataset_location( dataFileList=[datafile] ) - # Add location information to the path if host is provided - if source_folder_host: - datafile.path = f"{source_folder_host}:{file_path}" - # Upload the datablock self.scicat_client.upload_dataset_origdatablock(dataset_id, datablock) logger.info(f"Created new datablock for dataset {dataset_id} at location {datafile_path}") From 036ac15a1706c54d23b697df9a5c75f4e5c9287f Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:43:12 -0700 Subject: [PATCH 065/128] updating requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 52d6b872..4453a70f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ prefect==2.20.17 prometheus_client==0.21.1 pyscicat pyyaml -sfapi_client +sfapi_client \ No newline at end of file From 2137159b633b2540578c1bf9161e2d3105bc8bc9 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:50:44 -0700 Subject: [PATCH 066/128] Adding a TODO comment to _find_dataset to support searching dataFileLists. This method isn't currently called anywhere, so we can build this feature out as needed. --- orchestration/flows/scicat/ingestor_controller.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 64fcfcf0..2eb17735 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -264,6 +264,14 @@ def _find_dataset( Raises: ValueError: If no dataset or multiple datasets are found, or if the found dataset does not have a valid 'pid'. """ + + # TODO: I'm not sure if SciCat's advanced query API supports this, but, if we're actually searching by file_name, + # wouldn't it make more sense to look in all the dataFileList entries for all datasets? + # This comes to mind because at 733, scientists organize their data mostly by creating dated folders, + # and there's no guarantee that the files in those folders have unique names relative to the other folders. + # If they were searching for a data file, they would need to use a path fragment, e.g. '20241216_153047/new_run.h5' + # If this function could search dataFileLists by path fragment, it would be some future-proofing for those users... + if file_name: # Extract the datasetName from the file_name by stripping the directory and extension. extracted_name = os.path.splitext(os.path.basename(file_name))[0] From 0c26fb3d03b39efd9770a2fefbd27163a26b8cfd Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 10:54:40 -0700 Subject: [PATCH 067/128] Adjusting sfapi pytest to match new expected value --- orchestration/_tests/test_sfapi_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/_tests/test_sfapi_flow.py b/orchestration/_tests/test_sfapi_flow.py index aa5a9d50..c9cc5ab4 100644 --- a/orchestration/_tests/test_sfapi_flow.py +++ b/orchestration/_tests/test_sfapi_flow.py @@ -81,7 +81,7 @@ def test_create_sfapi_client_missing_paths(): from orchestration.sfapi import create_sfapi_client # Passing None for both paths should trigger a ValueError. - with pytest.raises(ValueError, match="Missing NERSC credentials paths."): + with pytest.raises(ValueError, match="NERSC credentials paths are missing."): create_sfapi_client(None, None) From e11cc0fd90be4849ff0888611448998036eb4303 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 21 May 2025 11:00:28 -0700 Subject: [PATCH 068/128] Adjusting prune_controller.prune() calls to use a float for days_from_now, rather than datetime, for simpler calls --- orchestration/flows/bl832/move_refactor.py | 4 ++-- scripts/test_controllers_end_to_end.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index da23b6fa..7a57cf87 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -91,7 +91,7 @@ def process_new_832_file( file_path=relative_path, source_endpoint=config.spot832, check_endpoint=config.data832, - days_from_now=datetime.timedelta(days=schedule_spot832_delete_days) + days_from_now=schedule_spot832_delete_days ) logger.info( f"Scheduled delete from spot832 at {datetime.timedelta(days=schedule_spot832_delete_days)}" @@ -101,7 +101,7 @@ def process_new_832_file( file_path=relative_path, source_endpoint=config.data832, check_endpoint=config.nersc832, - days_from_now=datetime.timedelta(days=schedule_data832_delete_days) + days_from_now=schedule_data832_delete_days ) logger.info( f"Scheduled delete from data832 at {datetime.timedelta(days=schedule_data832_delete_days)}" diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index e5b57da0..5ef43f29 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -4,7 +4,7 @@ """ -from datetime import timedelta, datetime +from datetime import datetime import logging import os import shutil @@ -463,7 +463,7 @@ def test_prune_controllers( file_path=file_path, source_endpoint=source_endpoint, check_endpoint=None, - days_from_now=timedelta(days=0) + days_from_now=0.0 ) # PRUNE FROM DESTINATION ENDPOINT @@ -481,7 +481,7 @@ def test_prune_controllers( file_path=file_path, source_endpoint=destination_endpoint, check_endpoint=None, - days_from_now=timedelta(days=0) + days_from_now=0.0 ) if test_filesystem: @@ -516,7 +516,7 @@ def test_prune_controllers( file_path=file_path, source_endpoint=source_endpoint, check_endpoint=None, - days_from_now=timedelta(days=0) + days_from_now=0.0 ) # Prune the destination endpoint @@ -524,7 +524,7 @@ def test_prune_controllers( file_path=file_path, source_endpoint=destination_endpoint, check_endpoint=None, - days_from_now=timedelta(days=0) + days_from_now=0.0 ) # After pruning in the filesystem pruner test From 36c40f73520351d22745c44e9a069b3d2d31e3c0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 5 Jun 2025 13:19:04 -0700 Subject: [PATCH 069/128] Renamining dummy to mock --- orchestration/_tests/test_prune_controller.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index 0d051989..ad61cb77 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -10,11 +10,11 @@ ############################################################################### -# Dummy Implementation for Testing +# Mock Implementation for Testing ############################################################################### -class DummyPruneController(PruneController): +class MockPruneController(PruneController): """ - A concrete dummy implementation of PruneController for testing purposes. + A concrete mock implementation of PruneController for testing purposes. This class uses a local directory (self.base_dir) to simulate file pruning. It provides additional helper methods: @@ -22,11 +22,11 @@ class DummyPruneController(PruneController): - prune_files(retention): deletes files older than the given retention period. """ def __init__(self, base_dir: Path): - # Create a dummy configuration object with a default beamline_id. - dummy_config = type("DummyConfig", (), {})() - dummy_config.tc = None - dummy_config.beamline_id = "dummy" # Add a default beamline_id for testing - super().__init__(dummy_config) + # Create a mock configuration object with a default beamline_id. + mock_config = type("MockConfig", (), {})() + mock_config.tc = None + mock_config.beamline_id = "mock" # Add a default beamline_id for testing + super().__init__(mock_config) self.base_dir = base_dir def get_files_to_delete(self, retention: int): @@ -83,7 +83,7 @@ def prune( days_from_now: timedelta = timedelta(0) ) -> bool: """ - Dummy implementation of the abstract method. + Mock implementation of the abstract method. (Not used in these tests.) """ return True @@ -107,9 +107,9 @@ def test_dir(): @pytest.fixture def prune_controller(test_dir): """ - Fixture that returns an instance of DummyPruneController using the temporary directory. + Fixture that returns an instance of MockPruneController using the temporary directory. """ - return DummyPruneController(base_dir=test_dir) + return MockPruneController(base_dir=test_dir) ############################################################################### @@ -140,7 +140,7 @@ def create_test_files(directory: Path, dates): ############################################################################### def test_prune_controller_initialization(prune_controller): from orchestration.prune_controller import PruneController - # Verify that our dummy controller is an instance of the abstract base class + # Verify that our mock controller is an instance of the abstract base class assert isinstance(prune_controller, PruneController) # And that the base directory exists assert prune_controller.base_dir.exists() From 980e681d09e3c3209575394229dc0e6fb903b953 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 5 Jun 2025 14:35:40 -0700 Subject: [PATCH 070/128] linting --- orchestration/prometheus_utils.py | 73 ++++++++++++++++++------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/orchestration/prometheus_utils.py b/orchestration/prometheus_utils.py index 9b3ca566..d988cea3 100644 --- a/orchestration/prometheus_utils.py +++ b/orchestration/prometheus_utils.py @@ -2,41 +2,52 @@ import uuid from prometheus_client import Gauge, CollectorRegistry, push_to_gateway + class PrometheusMetrics(): def __init__(self): try: # Create a new registry self.registry = CollectorRegistry() - + # Define the required metrics # 1. Count of requests - Gauge - self.request_counter = Gauge('nersc_transfer_request_count', - 'Number of times the flow has been executed', - ['execution_id'], - registry=self.registry) - - self.bytes_counter = Gauge('nersc_transfer_total_bytes', - 'Number of bytes for all the executed flows', - ['execution_id'], - registry=self.registry) - + self.request_counter = Gauge( + 'nersc_transfer_request_count', + 'Number of times the flow has been executed', + ['execution_id'], + registry=self.registry + ) + + self.bytes_counter = Gauge( + 'nersc_transfer_total_bytes', + 'Number of bytes for all the executed flows', + ['execution_id'], + registry=self.registry + ) + # 2. Total bytes transferred - Gauge - self.transfer_bytes = Gauge('nersc_transfer_file_bytes', - 'Total size of all file transfers to NERSC', - ['machine'], - registry=self.registry) - + self.transfer_bytes = Gauge( + 'nersc_transfer_file_bytes', + 'Total size of all file transfers to NERSC', + ['machine'], + registry=self.registry + ) + # 3. Transfer speed - Gauge - self.transfer_speed = Gauge('nersc_transfer_speed_bytes_per_second', - 'Transfer speed for NERSC file transfers in bytes per second', - ['machine'], - registry=self.registry) - + self.transfer_speed = Gauge( + 'nersc_transfer_speed_bytes_per_second', + 'Transfer speed for NERSC file transfers in bytes per second', + ['machine'], + registry=self.registry + ) + # 4. Transfer time - Gauge - self.transfer_time = Gauge('nersc_transfer_time_seconds', - 'Time taken for NERSC file transfers in seconds', - ['machine'], - registry=self.registry) + self.transfer_time = Gauge( + 'nersc_transfer_time_seconds', + 'Time taken for NERSC file transfers in seconds', + ['machine'], + registry=self.registry + ) except Exception as e: print(f"Error initializing Prometheus metrics: {e}") @@ -45,22 +56,22 @@ def push_metrics_to_prometheus(self, metrics, logger): PUSHGATEWAY_URL = os.getenv('PUSHGATEWAY_URL', 'http://localhost:9091') JOB_NAME = os.getenv('JOB_NAME', 'nersc_transfer') INSTANCE_LABEL = os.getenv('INSTANCE_LABEL', 'data_transfer') - + try: # Generate a unique execution ID for this transfer execution_id = f"exec_{str(uuid.uuid4())}" - + # Set the metrics self.request_counter.labels(execution_id=execution_id).set(1) self.bytes_counter.labels(execution_id=execution_id).set(metrics['bytes_transferred']) self.transfer_bytes.labels(machine=metrics['machine']).set(metrics['bytes_transferred']) self.transfer_time.labels(machine=metrics['machine']).set(metrics['duration_seconds']) self.transfer_speed.labels(machine=metrics['machine']).set(metrics['transfer_speed']) - + # Log metrics for debugging logger.info(f"Pushing metrics: transfer_bytes = {metrics['bytes_transferred']} bytes") logger.info(f"Pushing metrics: transfer_speed = {metrics['transfer_speed']} bytes/second") - + # Push to Pushgateway with error handling try: push_to_gateway( @@ -72,6 +83,6 @@ def push_metrics_to_prometheus(self, metrics, logger): logger.info(f"Successfully pushed metrics to Pushgateway at {PUSHGATEWAY_URL}") except Exception as push_error: logger.error(f"Error pushing to Pushgateway at {PUSHGATEWAY_URL}: {push_error}") - + except Exception as e: - logger.error(f"Error preparing metrics for Prometheus: {e}") \ No newline at end of file + logger.error(f"Error preparing metrics for Prometheus: {e}") From 4c699a3b4dcd0f33cdbf451a1eeea5b5fc05e9c8 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:06:41 -0700 Subject: [PATCH 071/128] Fixed type (tranfer_client -> transfer_client) --- orchestration/prune_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index 8180a5ae..b4d01056 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -332,7 +332,7 @@ def _prune_globus_endpoint( prune_one_safe( file=relative_path, if_older_than_days=0, - tranfer_client=config.tc, + transfer_client=config.tc, source_endpoint=source_endpoint, check_endpoint=check_endpoint, logger=logger, From 3bb47af78dd08f79405b46b44d7c8b738e8f53d5 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:07:21 -0700 Subject: [PATCH 072/128] Fixed typo (tranfer_client -> transfer_client) --- docs/bl832_ALCF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bl832_ALCF.md b/docs/bl832_ALCF.md index b6f9f8c0..6eda4975 100644 --- a/docs/bl832_ALCF.md +++ b/docs/bl832_ALCF.md @@ -428,7 +428,7 @@ def prune_alcf832_raw(relative_path: str): prune_one_safe( file=relative_path, if_older_than_days=0, - tranfer_client=tc, + transfer_client=tc, source_endpoint=config.alcf832_raw, check_endpoint=config.nersc832_alsdev_raw, logger=p_logger, From b71ea426f40e4d94227bf110b60deca53df21441 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:07:41 -0700 Subject: [PATCH 073/128] Fixed typo (tranfer_client -> transfer_client) --- docs/mkdocs/docs/alcf832.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mkdocs/docs/alcf832.md b/docs/mkdocs/docs/alcf832.md index 3774832f..3d441e0d 100644 --- a/docs/mkdocs/docs/alcf832.md +++ b/docs/mkdocs/docs/alcf832.md @@ -389,7 +389,7 @@ def prune_alcf832_raw(relative_path: str): prune_one_safe( file=relative_path, if_older_than_days=0, - tranfer_client=tc, + transfer_client=tc, source_endpoint=config.alcf832_raw, check_endpoint=config.nersc832_alsdev_raw, logger=p_logger, From cc9d55ff0f54778f72227baa79e87edd9dbee717 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:08:48 -0700 Subject: [PATCH 074/128] Fixed typo (tranfer_client -> transfer_client) --- orchestration/flows/bl832/prune.py | 2 +- orchestration/globus/transfer.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/orchestration/flows/bl832/prune.py b/orchestration/flows/bl832/prune.py index 1de05085..91ddb54f 100644 --- a/orchestration/flows/bl832/prune.py +++ b/orchestration/flows/bl832/prune.py @@ -36,7 +36,7 @@ def prune_files( prune_one_safe( file=relative_path, if_older_than_days=0, - tranfer_client=config.tc, + transfer_client=config.tc, source_endpoint=source_endpoint, check_endpoint=check_endpoint, logger=p_logger, diff --git a/orchestration/globus/transfer.py b/orchestration/globus/transfer.py index 764ba6ad..e5ddfadd 100644 --- a/orchestration/globus/transfer.py +++ b/orchestration/globus/transfer.py @@ -1,3 +1,4 @@ +# orchestration/globus/transfer.py from dataclasses import dataclass from datetime import datetime, timezone, timedelta from dateutil import parser @@ -269,7 +270,7 @@ def task_wait( def prune_one_safe( file: str, if_older_than_days: int, - tranfer_client: TransferClient, + transfer_client: TransferClient, source_endpoint: GlobusEndpoint, check_endpoint: Union[GlobusEndpoint, None], max_wait_seconds: int = 120, @@ -281,7 +282,7 @@ def prune_one_safe( is also located at the check_endpoint. If not, raises """ # does the file exist at the source endpoint? - g_file_obj = get_globus_file_object(tranfer_client, source_endpoint, file) + g_file_obj = get_globus_file_object(transfer_client, source_endpoint, file) assert g_file_obj is not None, f"file not found {source_endpoint.uri}" logger.info(f"file: {file} found on {source_endpoint.uri}") @@ -289,7 +290,7 @@ def prune_one_safe( if check_endpoint is None: logger.info("No check endpoint provided, skipping check") else: - g_file_obj = get_globus_file_object(tranfer_client, check_endpoint, file) + g_file_obj = get_globus_file_object(transfer_client, check_endpoint, file) assert g_file_obj is not None, f"file not found {check_endpoint.uri}" logger.info(f"file: {file} found on {check_endpoint.uri}") @@ -306,14 +307,14 @@ def prune_one_safe( logger.info("Not checking dates, sent if_older_than_days==0") delete_id = prune_files( - tranfer_client, + transfer_client, source_endpoint, [file], max_wait_seconds=max_wait_seconds, logger=logger, ) - task_wait(tranfer_client, delete_id) + task_wait(transfer_client, delete_id) logger.info(f"file deleted from: {source_endpoint.uri}") From d030114548e0c79ed9910814ccfc005733579d5e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:10:54 -0700 Subject: [PATCH 075/128] Adding a comment at the top --- orchestration/transfer_endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/orchestration/transfer_endpoints.py b/orchestration/transfer_endpoints.py index 23e5ce46..55062774 100644 --- a/orchestration/transfer_endpoints.py +++ b/orchestration/transfer_endpoints.py @@ -1,3 +1,4 @@ +# orchestration/transfer_endpoints.py from abc import ABC From 4105d3bc3bee224d0488a1f06330ed50e928621b Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:28:28 -0700 Subject: [PATCH 076/128] Updating test_prune_controller.py based on Dylan's comments. Ensuring use of the term mock over other synonyms, creating fixtures as needed, and ensuring core functionality of the prune controller is comprehensively tested --- orchestration/_tests/test_prune_controller.py | 461 +++++++++++------- 1 file changed, 289 insertions(+), 172 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index ad61cb77..82d633fe 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -1,203 +1,320 @@ -import os -import shutil -from datetime import datetime, timedelta +# tests/orchestration/_tests/test_prune_controllers.py + from pathlib import Path +from typing import Any, Dict, Optional import pytest +from prefect.blocks.system import JSON +from prefect.testing.utilities import prefect_test_harness -# Import the abstract PruneController base -from orchestration.prune_controller import PruneController +from orchestration.prune_controller import ( + PruneController, + FileSystemPruneController, + GlobusPruneController, + get_prune_controller, + PruneMethod, +) +from orchestration.transfer_endpoints import FileSystemEndpoint +from orchestration.globus.transfer import GlobusEndpoint ############################################################################### -# Mock Implementation for Testing +# Shared Fixtures & Helpers ############################################################################### -class MockPruneController(PruneController): - """ - A concrete mock implementation of PruneController for testing purposes. - This class uses a local directory (self.base_dir) to simulate file pruning. - It provides additional helper methods: - - get_files_to_delete(retention): returns files older than the given retention period. - - prune_files(retention): deletes files older than the given retention period. +@pytest.fixture(autouse=True, scope="session") +def prefect_test_fixture(): + """Set up the Prefect test harness and register our JSON block.""" + with prefect_test_harness(): + JSON(value={"max_wait_seconds": 600}).save(name="globus-settings") + yield + + +class MockConfig: + """Minimal config stub for controllers (only beamline_id and tc).""" + def __init__(self, beamline_id: str = "test_beamline") -> None: + self.beamline_id = beamline_id + self.tc: Any = None # transfer client stub + + +@pytest.fixture +def mock_config() -> MockConfig: + """Provides a fresh MockConfig per test.""" + return MockConfig(beamline_id="unittest_beamline") + + +@pytest.fixture +def tmp_fs_path(tmp_path: Path) -> Path: + """Temporary directory for filesystem tests.""" + return tmp_path + + +@pytest.fixture +def fs_endpoint(tmp_fs_path: Path) -> FileSystemEndpoint: + """A FileSystemEndpoint rooted at our tmp directory.""" + return FileSystemEndpoint( + name="fs_endpoint", + root_path=str(tmp_fs_path), + uri=str(tmp_fs_path), + ) + + +@pytest.fixture +def globus_endpoint(tmp_fs_path: Path) -> GlobusEndpoint: + """A real GlobusEndpoint with a mock UUID.""" + return GlobusEndpoint( + uuid="mock-uuid", + uri=str(tmp_fs_path), + root_path=str(tmp_fs_path), + name="globus_endpoint", + ) + + +@pytest.fixture +def fs_controller(mock_config: MockConfig) -> FileSystemPruneController: + """FileSystemPruneController using mock_config.""" + return FileSystemPruneController(config=mock_config) + + +@pytest.fixture +def globus_controller(mock_config: MockConfig) -> GlobusPruneController: + """GlobusPruneController using mock_config.""" + return GlobusPruneController(config=mock_config) + + +def create_file_or_dir(root: Path, rel: str, mkdir: bool = False) -> Path: """ - def __init__(self, base_dir: Path): - # Create a mock configuration object with a default beamline_id. - mock_config = type("MockConfig", (), {})() - mock_config.tc = None - mock_config.beamline_id = "mock" # Add a default beamline_id for testing - super().__init__(mock_config) - self.base_dir = base_dir - - def get_files_to_delete(self, retention: int): - """ - Return a list of files in self.base_dir whose modification time is older than - 'retention' days. - - Args: - retention (int): Retention period in days. Must be > 0. - - Returns: - List[Path]: Files older than the retention period. - - Raises: - ValueError: If retention is not positive. - """ - if retention <= 0: - raise ValueError("Retention must be a positive number of days") - now_ts = datetime.now().timestamp() - retention_seconds = retention * 24 * 3600 - files_to_delete = [] - for f in self.base_dir.glob("*"): - if f.is_file(): - mod_time = f.stat().st_mtime - if now_ts - mod_time > retention_seconds: - files_to_delete.append(f) - return files_to_delete - - def prune_files(self, retention: int): - """ - Delete files older than 'retention' days and return the list of deleted files. - - Args: - retention (int): Retention period in days. Must be > 0. - - Returns: - List[Path]: List of files that were deleted. - - Raises: - ValueError: If retention is not positive. - """ - if retention <= 0: - raise ValueError("Retention must be a positive number of days") - files_to_delete = self.get_files_to_delete(retention) - for f in files_to_delete: - f.unlink() # Delete the file - return files_to_delete - - def prune( - self, - file_path: str = None, - source_endpoint=None, - check_endpoint=None, - days_from_now: timedelta = timedelta(0) - ) -> bool: - """ - Mock implementation of the abstract method. - (Not used in these tests.) - """ - return True + Create either a file or directory under root/rel. + If mkdir is True, makes a directory; otherwise creates an empty file. + """ + p = root / rel + if mkdir: + p.mkdir(parents=True, exist_ok=True) + else: + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + return p ############################################################################### -# Pytest Fixtures +# Mock Fixtures for External Calls ############################################################################### + @pytest.fixture -def test_dir(): +def mock_scheduler(monkeypatch): """ - Fixture that creates (and later cleans up) a temporary directory for tests. + Monkeypatches schedule_prefect_flow ā†’ a mock that records its args and returns True. + Returns the dict where call args are recorded. """ - test_path = Path("test_prune_data") - test_path.mkdir(exist_ok=True) - yield test_path - if test_path.exists(): - shutil.rmtree(test_path) + recorded: Dict[str, Any] = {} + + def _scheduler(deployment_name, flow_run_name, parameters, duration_from_now): + recorded.update( + deployment_name=deployment_name, + flow_run_name=flow_run_name, + parameters=parameters, + duration=duration_from_now, + ) + return True + + monkeypatch.setattr( + "orchestration.prune_controller.schedule_prefect_flow", + _scheduler, + ) + return recorded @pytest.fixture -def prune_controller(test_dir): +def mock_scheduler_raises(monkeypatch): """ - Fixture that returns an instance of MockPruneController using the temporary directory. + Monkeypatches schedule_prefect_flow ā†’ a mock that always raises. """ - return MockPruneController(base_dir=test_dir) + def _scheduler_raises(*args, **kwargs): + raise RuntimeError("scheduler failure") + monkeypatch.setattr( + "orchestration.prune_controller.schedule_prefect_flow", + _scheduler_raises, + ) -############################################################################### -# Helper Function for Creating Test Files -############################################################################### -def create_test_files(directory: Path, dates): - """ - Create test files in the specified directory with modification times given by `dates`. - - Args: - directory (Path): The directory in which to create files. - dates (List[datetime]): List of datetimes to set as the file's modification time. - Returns: - List[Path]: List of created file paths. +@pytest.fixture +def mock_prune_one_safe(monkeypatch): + """ + Monkeypatches prune_one_safe ā†’ a mock that records its kwargs and returns True. + Returns the dict where call args are recorded. """ - files = [] - for date in dates: - filepath = directory / f"test_file_{date.strftime('%Y%m%d')}.txt" - filepath.touch() # Create the empty file - os.utime(filepath, (date.timestamp(), date.timestamp())) - files.append(filepath) - return files + recorded: Dict[str, Any] = {} + + def _prune_one_safe( + file: str, + if_older_than_days: int, + transfer_client: Any, + source_endpoint: GlobusEndpoint, + check_endpoint: Optional[GlobusEndpoint], + logger: Any, + max_wait_seconds: int, + ) -> bool: + recorded.update( + file=file, + if_older_than_days=if_older_than_days, + transfer_client=transfer_client, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + max_wait_seconds=max_wait_seconds, + ) + return True + + monkeypatch.setattr( + "orchestration.prune_controller.prune_one_safe", + _prune_one_safe, + ) + return recorded ############################################################################### # Tests ############################################################################### -def test_prune_controller_initialization(prune_controller): - from orchestration.prune_controller import PruneController - # Verify that our mock controller is an instance of the abstract base class - assert isinstance(prune_controller, PruneController) - # And that the base directory exists - assert prune_controller.base_dir.exists() - - -def test_get_files_to_delete(prune_controller, test_dir): - # Create test files with various modification times. - now = datetime.now() - dates = [ - now - timedelta(days=31), # Old enough to be pruned - now - timedelta(days=20), # Recent - now - timedelta(days=40), # Old enough to be pruned - now - timedelta(days=10), # Recent - ] - test_files = create_test_files(test_dir, dates) - - # When using a 30-day retention, the two older files should be flagged. - files_to_delete = prune_controller.get_files_to_delete(30) - assert len(files_to_delete) == 2 - - # Check that the names of the older files are in the returned list. - file_names_to_delete = {f.name for f in files_to_delete} - assert test_files[0].name in file_names_to_delete - assert test_files[2].name in file_names_to_delete - - -def test_prune_files(prune_controller, test_dir): - # Create two files: one older than 30 days and one newer. - now = datetime.now() - dates = [ - now - timedelta(days=31), # Should be deleted - now - timedelta(days=20), # Should remain - ] - test_files = create_test_files(test_dir, dates) - - # Prune files older than 30 days. - deleted_files = prune_controller.prune_files(30) - # One file should have been deleted. - assert len(deleted_files) == 1 - # The older file should no longer exist. - assert not test_files[0].exists() - # The newer file should still exist. - assert test_files[1].exists() - - -def test_empty_directory(prune_controller): - # Ensure the test directory is empty. - for f in list(prune_controller.base_dir.glob("*")): - f.unlink() - deleted_files = prune_controller.prune_files(30) - # There should be no files to delete. - assert len(deleted_files) == 0 - - -def test_invalid_retention_period(prune_controller): - # Using retention periods <= 0 should raise a ValueError. - with pytest.raises(ValueError): - prune_controller.prune_files(-1) - with pytest.raises(ValueError): - prune_controller.prune_files(0) + +def test_prunecontroller_is_abstract(): + """PruneController must be abstract (cannot be instantiated directly).""" + with pytest.raises(TypeError): + PruneController(config=MockConfig()) + + +def test_get_prune_controller_factory_correct_types(mock_config): + """get_prune_controller returns the right subclass or raises on invalid.""" + assert isinstance(get_prune_controller(PruneMethod.SIMPLE, mock_config), FileSystemPruneController) + assert isinstance(get_prune_controller(PruneMethod.GLOBUS, mock_config), GlobusPruneController) + with pytest.raises((AttributeError, ValueError)): + get_prune_controller("invalid", mock_config) # type: ignore + + +def test_fs_prune_immediate_deletes_file_directly(fs_controller, fs_endpoint, tmp_fs_path): + """Immediate FileSystem prune should delete an existing file.""" + rel = "subdir/foo.txt" + p = create_file_or_dir(tmp_fs_path, rel) + assert p.exists() + + fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore + assert fn(relative_path=rel, source_endpoint=fs_endpoint, check_endpoint=None, config=fs_controller.config) + assert not p.exists() + + +def test_fs_prune_immediate_returns_false_if_missing(fs_controller, fs_endpoint, tmp_fs_path): + """Immediate FileSystem prune should return False for missing path.""" + rel = "no/such/file.txt" + assert not (tmp_fs_path / rel).exists() + + fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore + assert fn(relative_path=rel, source_endpoint=fs_endpoint, check_endpoint=None, config=fs_controller.config) is False + + +def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoint, tmp_fs_path, mock_scheduler): + """Calling prune with days_from_now>0 should schedule a Prefect flow.""" + rel = "to_schedule.txt" + create_file_or_dir(tmp_fs_path, rel) + + result = fs_controller.prune( + file_path=rel, + source_endpoint=fs_endpoint, + check_endpoint=None, + days_from_now=1.5, + ) + assert result is True + + assert mock_scheduler["flow_run_name"] == f"prune_from_{fs_endpoint.name}" + assert mock_scheduler["parameters"]["relative_path"] == rel + assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 1.5 * 86400 + + +def test_fs_prune_returns_false_if_schedule_raises(fs_controller, fs_endpoint, tmp_fs_path, mock_scheduler_raises): + """If scheduling fails, fs_controller.prune should return False.""" + rel = "error.txt" + create_file_or_dir(tmp_fs_path, rel) + + assert fs_controller.prune( + file_path=rel, + source_endpoint=fs_endpoint, + check_endpoint=None, + days_from_now=2.0, + ) is False + + +def test_globus_prune_immediate_calls_prune_one_safe_directly( + globus_controller, + globus_endpoint, + tmp_fs_path, + mock_prune_one_safe +): + """Immediate Globus prune should invoke prune_one_safe with correct arguments.""" + rel = "data.bin" + create_file_or_dir(tmp_fs_path, rel) + assert (tmp_fs_path / rel).exists() + + fn = globus_controller._prune_globus_endpoint.fn # type: ignore + _ = fn( + relative_path=rel, + source_endpoint=globus_endpoint, + check_endpoint=None, + config=globus_controller.config, + ) + + assert mock_prune_one_safe["file"] == rel + assert mock_prune_one_safe["if_older_than_days"] == 0 + assert mock_prune_one_safe["transfer_client"] is None + assert mock_prune_one_safe["source_endpoint"] is globus_endpoint + assert mock_prune_one_safe["max_wait_seconds"] == 600 + + +@pytest.mark.parametrize("invalid_fp", [None, ""]) +def test_globus_prune_rejects_missing_file_path_directly(globus_controller, globus_endpoint, invalid_fp): + """Globus prune should return False when file_path is None or empty.""" + assert globus_controller.prune( + file_path=invalid_fp, + source_endpoint=globus_endpoint, + check_endpoint=None, + days_from_now=0.0, + ) is False + + +def test_globus_prune_rejects_missing_endpoint_directly(globus_controller, tmp_fs_path): + """Globus prune should return False when source_endpoint is None.""" + (tmp_fs_path / "whatever").touch() + + assert globus_controller.prune( + file_path="whatever", + source_endpoint=None, # type: ignore + check_endpoint=None, + days_from_now=0.0, + ) is False + + +def test_globus_prune_schedules_when_days_from_now_positive(globus_controller, globus_endpoint, tmp_fs_path, mock_scheduler): + """Calling Globus prune with days_from_now>0 should schedule a Prefect flow.""" + rel = "sched.txt" + create_file_or_dir(tmp_fs_path, rel) + + result = globus_controller.prune( + file_path=rel, + source_endpoint=globus_endpoint, + check_endpoint=None, + days_from_now=3.0, + ) + assert result is True + + assert mock_scheduler["flow_run_name"] == f"prune_from_{globus_endpoint.name}" + assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 3.0 * 86400 + + +def test_globus_prune_returns_false_if_schedule_raises(globus_controller, globus_endpoint, tmp_fs_path, mock_scheduler_raises): + """If scheduling fails, globus_controller.prune should return False.""" + rel = "err.txt" + create_file_or_dir(tmp_fs_path, rel) + + assert globus_controller.prune( + file_path=rel, + source_endpoint=globus_endpoint, + check_endpoint=None, + days_from_now=4.0, + ) is False From 70ff27e0b39d80a1522725e46f14e42097aea5ba Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 11 Jun 2025 10:38:03 -0700 Subject: [PATCH 077/128] Removing redundant logger.debug messages when initializing transfer controllers --- orchestration/transfer_controller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 3f442c3c..107c9389 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -79,7 +79,6 @@ def __init__( ) -> None: super().__init__(config) self.prometheus_metrics = prometheus_metrics - logger.debug(f"Initialized GlobusTransferController for beamline {config.beamline_id}") """ Use Globus Transfer to move data between endpoints. @@ -313,7 +312,6 @@ def __init__( config: BeamlineConfig ) -> None: super().__init__(config) - logger.debug(f"Initialized SimpleTransferController for beamline {config.beamline_id}") def copy( self, From 917876928a45e3643b6617812ac7f514a54315b3 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 10:57:10 -0700 Subject: [PATCH 078/128] Updating 733 docs sequence diagram --- docs/mkdocs/docs/733.md | 217 +++++++++++++++------------------------- 1 file changed, 82 insertions(+), 135 deletions(-) diff --git a/docs/mkdocs/docs/733.md b/docs/mkdocs/docs/733.md index ea0c8918..74f2fc0f 100644 --- a/docs/mkdocs/docs/733.md +++ b/docs/mkdocs/docs/733.md @@ -1,141 +1,88 @@ +# Beamline 7.3.3 + + +## Flow Diagram ```mermaid sequenceDiagram - %% Participants - participant Detector as Detector - participant FileWatcher as File Watcher (data733) - participant Dispatcher as Dispatcher [Prefect Worker] + participant DET as Detector/
File Watcher + participant DISP as Prefect
Dispatcher + participant D733 as data733
Storage + participant GLOB as Globus
Transfer + participant CFS as NERSC
CFS + participant CAT as SciCat
Metadata + participant SFAPI as SFAPI + participant HPC as HPC
Compute + participant HPSS as HPSS
Tape - %% Flow 1: new_file_733 Prefect Flow - participant F1_Data as Flow1: data733 - participant F1_CFS as Flow1: NERSC CFS - participant F1_SciCat as Flow1: SciCat (Metadata DB) + %% Initial Trigger + DET->>DET: Monitor filesystem + DET->>DISP: Trigger on new file + DISP->>DISP: Coordinate flows - %% Flow 2: Scheduled HPSS Transfer Prefect Flow - participant F2_CFS as Flow2: NERSC CFS - participant F2_HPSS as Flow2: HPSS Tape Archive - participant F2_SciCat as Flow2: SciCat (Metadata DB) + %% Flow 1: new_file_733 + rect rgb(220, 230, 255) + note over DISP,CAT: FLOW 1: new_file_733 + DISP->>GLOB: Init transfer + activate GLOB + GLOB->>D733: Initiate copy + activate D733 + D733-->>GLOB: Copy initiated + deactivate D733 + %% note right of GLOB: Transfer in progress + GLOB-->>DISP: Transfer complete + deactivate GLOB + + DISP->>CAT: Register metadata + end - %% Flow 3: HPC Downstream Analysis Prefect Flow - participant F3_Data as Flow3: data733 - participant HPC_FS as HPC Filesystem - participant HPC_Compute as HPC Compute + %% Flow 2: HPSS Transfer + rect rgb(220, 255, 230) + note over DISP,CAT: FLOW 2: Scheduled HPSS Transfer + DISP->>SFAPI: Submit tape job + activate SFAPI + SFAPI->>HPSS: Initiate archive + activate HPSS + HPSS-->>SFAPI: Archive complete + deactivate HPSS + SFAPI-->>DISP: Job complete + deactivate SFAPI + + DISP->>CAT: Update metadata + end - %% Scheduled Pruning (triggered by all flows) - participant SPruning as Scheduled Pruning (Prefect Workers) - participant P_CFS as Prune Target: NERSC CFS - participant P_Data as Prune Target: data733 - - %% Initial Trigger Sequence - Detector->>FileWatcher: Send Raw Data - FileWatcher->>Dispatcher: File Watcher Trigger - Dispatcher->>F1_Data: Start new_file_733 Flow - Dispatcher->>F2_CFS: Start Scheduled HPSS Transfer Flow - Dispatcher->>F3_Data: Start HPC Downstream Analysis Flow - - %% Flow 1 interactions - F1_Data->>F1_CFS: Globus Transfer: Raw Data - F1_CFS->>F1_SciCat: SciCat Ingestion: Metadata - - %% Flow 2 interactions - F2_CFS->>F2_HPSS: SFAPI Slurm htar Transfer: Raw Data - F2_HPSS->>F2_SciCat: SciCat Ingestion: Metadata - - %% Flow 3 interactions - F3_Data->>HPC_FS: Globus Transfer: Raw Data - HPC_FS->>HPC_Compute: Transfer Raw Data - HPC_Compute->>HPC_FS: Return Scratch Data - HPC_FS->>F3_Data: Globus Transfer: Scratch Data - - %% Scheduled Pruning triggered by flows - F1_SciCat-->>SPruning: Trigger Pruning - F2_SciCat-->>SPruning: Trigger Pruning - F3_Data-->>SPruning: Trigger Pruning - - SPruning->>P_CFS: Prune NERSC CFS - SPruning->>P_Data: Prune data733 -``` - - -```mermaid ---- -config: - theme: neo - layout: elk - look: neo ---- -flowchart LR - subgraph s1["new_file_733
[Prefect Flow]"] - n20["data733"] - n21["NERSC CFS"] - n22@{ label: "SciCat
[Metadata Database]" } - end - subgraph s2["Scheduled HPSS Transfer
[Prefect Flow]"] - n38["NERSC CFS"] - n39["HPSS Tape Archive"] - n40["SciCat
[Metadata Database]"] - end - subgraph s3["HPC Downstream Analysis
[Prefect Flow]"] - n41["data733"] - n42["HPC
Filesystem"] - n43["HPC
Compute"] - end - n23["data733"] -- File Watcher --> n24["Dispatcher
[Prefect Worker]"] - n25["Detector"] -- Raw Data --> n23 - n24 --> s1 & s2 & s3 - n20 -- Raw Data [Globus Transfer] --> n21 - n21 -- "Metadata [SciCat Ingestion]" --> n22 - n32["Scheduled Pruning
[Prefect Workers]"] --> n35["NERSC CFS"] & n34["data733"] - n38 -- Raw Data [SFAPI Slurm htar Transfer] --> n39 - n39 -- "Metadata [SciCat Ingestion]" --> n40 - s2 --> n32 - s3 --> n32 - s1 --> n32 - n41 -- Raw Data [Globus Transfer] --> n42 - n42 -- Raw Data --> n43 - n43 -- Scratch Data --> n42 - n42 -- Scratch Data [Globus Transfer] --> n41 - n20@{ shape: internal-storage} - n21@{ shape: disk} - n22@{ shape: db} - n38@{ shape: disk} - n39@{ shape: paper-tape} - n40@{ shape: db} - n41@{ shape: internal-storage} - n42@{ shape: disk} - n23@{ shape: internal-storage} - n24@{ shape: rect} - n25@{ shape: rounded} - n35@{ shape: disk} - n34@{ shape: internal-storage} - n20:::storage - n20:::Peach - n21:::Sky - n22:::Sky - n38:::Sky - n39:::storage - n40:::Sky - n41:::Peach - n42:::Sky - n43:::compute - n23:::collection - n23:::storage - n23:::Peach - n24:::collection - n24:::Rose - n25:::Ash - n32:::Rose - n35:::Sky - n34:::Peach - classDef collection fill:#D3A6A1, stroke:#D3A6A1, stroke-width:2px, color:#000000 - classDef Rose stroke-width:1px, stroke-dasharray:none, stroke:#FF5978, fill:#FFDFE5, color:#8E2236 - classDef storage fill:#A3C1DA, stroke:#A3C1DA, stroke-width:2px, color:#000000 - classDef Ash stroke-width:1px, stroke-dasharray:none, stroke:#999999, fill:#EEEEEE, color:#000000 - classDef visualization fill:#E8D5A6, stroke:#E8D5A6, stroke-width:2px, color:#000000 - classDef Peach stroke-width:1px, stroke-dasharray:none, stroke:#FBB35A, fill:#FFEFDB, color:#8F632D - classDef Sky stroke-width:1px, stroke-dasharray:none, stroke:#374D7C, fill:#E2EBFF, color:#374D7C - classDef compute fill:#A9C0C9, stroke:#A9C0C9, stroke-width:2px, color:#000000 - style s1 stroke:#757575 - style s2 stroke:#757575 - style s3 stroke:#757575 - -``` \ No newline at end of file + %% Flow 3: HPC Analysis + rect rgb(255, 230, 230) + note over DISP,HPC: FLOW 3: HPC Downstream Analysis + DISP->>SFAPI: Submit compute job + activate SFAPI + SFAPI->>HPC: Execute job + activate HPC + HPC->>HPC: Process data + HPC-->>SFAPI: Compute complete + deactivate HPC + SFAPI-->>DISP: Job complete + deactivate SFAPI + + DISP->>CAT: Update metadata + end + + %% Flow 4: Scheduled Pruning + rect rgb(255, 255, 220) + note over DISP,CAT: FLOW 4: Scheduled Pruning + DISP->>DISP: Scheduled pruning trigger + + DISP->>D733: Prune old files + activate D733 + D733->>D733: Delete expired data + D733-->>DISP: Pruning complete + deactivate D733 + + DISP->>CFS: Prune old files + activate CFS + CFS->>CFS: Delete expired data + CFS-->>DISP: Pruning complete + deactivate CFS + + DISP->>CAT: Update metadata + end + ``` \ No newline at end of file From 6330d38f64bf2c9e395b473eb1f19e1b0a46dbe0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 11:32:44 -0700 Subject: [PATCH 079/128] Using the tmp_path fixture for pytest --- orchestration/_tests/test_prune_controller.py | 85 +++++++++---------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index 82d633fe..84efb19f 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -44,28 +44,22 @@ def mock_config() -> MockConfig: @pytest.fixture -def tmp_fs_path(tmp_path: Path) -> Path: - """Temporary directory for filesystem tests.""" - return tmp_path - - -@pytest.fixture -def fs_endpoint(tmp_fs_path: Path) -> FileSystemEndpoint: +def fs_endpoint(tmp_path: Path) -> FileSystemEndpoint: """A FileSystemEndpoint rooted at our tmp directory.""" return FileSystemEndpoint( name="fs_endpoint", - root_path=str(tmp_fs_path), - uri=str(tmp_fs_path), + root_path=str(tmp_path), + uri=str(tmp_path), ) @pytest.fixture -def globus_endpoint(tmp_fs_path: Path) -> GlobusEndpoint: +def globus_endpoint(tmp_path: Path) -> GlobusEndpoint: """A real GlobusEndpoint with a mock UUID.""" return GlobusEndpoint( uuid="mock-uuid", - uri=str(tmp_fs_path), - root_path=str(tmp_fs_path), + uri=str(tmp_path), + root_path=str(tmp_path), name="globus_endpoint", ) @@ -82,20 +76,6 @@ def globus_controller(mock_config: MockConfig) -> GlobusPruneController: return GlobusPruneController(config=mock_config) -def create_file_or_dir(root: Path, rel: str, mkdir: bool = False) -> Path: - """ - Create either a file or directory under root/rel. - If mkdir is True, makes a directory; otherwise creates an empty file. - """ - p = root / rel - if mkdir: - p.mkdir(parents=True, exist_ok=True) - else: - p.parent.mkdir(parents=True, exist_ok=True) - p.touch() - return p - - ############################################################################### # Mock Fixtures for External Calls ############################################################################### @@ -190,10 +170,12 @@ def test_get_prune_controller_factory_correct_types(mock_config): get_prune_controller("invalid", mock_config) # type: ignore -def test_fs_prune_immediate_deletes_file_directly(fs_controller, fs_endpoint, tmp_fs_path): +def test_fs_prune_immediate_deletes_file_directly(fs_controller, fs_endpoint, tmp_path: Path): """Immediate FileSystem prune should delete an existing file.""" rel = "subdir/foo.txt" - p = create_file_or_dir(tmp_fs_path, rel) + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() assert p.exists() fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore @@ -201,19 +183,21 @@ def test_fs_prune_immediate_deletes_file_directly(fs_controller, fs_endpoint, tm assert not p.exists() -def test_fs_prune_immediate_returns_false_if_missing(fs_controller, fs_endpoint, tmp_fs_path): +def test_fs_prune_immediate_returns_false_if_missing(fs_controller, fs_endpoint, tmp_path: Path): """Immediate FileSystem prune should return False for missing path.""" rel = "no/such/file.txt" - assert not (tmp_fs_path / rel).exists() + assert not (tmp_path / rel).exists() fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore assert fn(relative_path=rel, source_endpoint=fs_endpoint, check_endpoint=None, config=fs_controller.config) is False -def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoint, tmp_fs_path, mock_scheduler): +def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler): """Calling prune with days_from_now>0 should schedule a Prefect flow.""" rel = "to_schedule.txt" - create_file_or_dir(tmp_fs_path, rel) + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() result = fs_controller.prune( file_path=rel, @@ -228,10 +212,12 @@ def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoi assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 1.5 * 86400 -def test_fs_prune_returns_false_if_schedule_raises(fs_controller, fs_endpoint, tmp_fs_path, mock_scheduler_raises): +def test_fs_prune_returns_false_if_schedule_raises(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler_raises): """If scheduling fails, fs_controller.prune should return False.""" rel = "error.txt" - create_file_or_dir(tmp_fs_path, rel) + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() assert fs_controller.prune( file_path=rel, @@ -244,13 +230,14 @@ def test_fs_prune_returns_false_if_schedule_raises(fs_controller, fs_endpoint, t def test_globus_prune_immediate_calls_prune_one_safe_directly( globus_controller, globus_endpoint, - tmp_fs_path, + tmp_path: Path, mock_prune_one_safe ): """Immediate Globus prune should invoke prune_one_safe with correct arguments.""" rel = "data.bin" - create_file_or_dir(tmp_fs_path, rel) - assert (tmp_fs_path / rel).exists() + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() fn = globus_controller._prune_globus_endpoint.fn # type: ignore _ = fn( @@ -278,9 +265,9 @@ def test_globus_prune_rejects_missing_file_path_directly(globus_controller, glob ) is False -def test_globus_prune_rejects_missing_endpoint_directly(globus_controller, tmp_fs_path): +def test_globus_prune_rejects_missing_endpoint_directly(globus_controller, tmp_path: Path): """Globus prune should return False when source_endpoint is None.""" - (tmp_fs_path / "whatever").touch() + (tmp_path / "whatever").touch() assert globus_controller.prune( file_path="whatever", @@ -290,10 +277,16 @@ def test_globus_prune_rejects_missing_endpoint_directly(globus_controller, tmp_f ) is False -def test_globus_prune_schedules_when_days_from_now_positive(globus_controller, globus_endpoint, tmp_fs_path, mock_scheduler): +def test_globus_prune_schedules_when_days_from_now_positive( + globus_controller, + globus_endpoint, + tmp_path: Path, + mock_scheduler): """Calling Globus prune with days_from_now>0 should schedule a Prefect flow.""" rel = "sched.txt" - create_file_or_dir(tmp_fs_path, rel) + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() result = globus_controller.prune( file_path=rel, @@ -307,10 +300,16 @@ def test_globus_prune_schedules_when_days_from_now_positive(globus_controller, g assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 3.0 * 86400 -def test_globus_prune_returns_false_if_schedule_raises(globus_controller, globus_endpoint, tmp_fs_path, mock_scheduler_raises): +def test_globus_prune_returns_false_if_schedule_raises( + globus_controller, + globus_endpoint, + tmp_path: Path, + mock_scheduler_raises): """If scheduling fails, globus_controller.prune should return False.""" rel = "err.txt" - create_file_or_dir(tmp_fs_path, rel) + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() assert globus_controller.prune( file_path=rel, From 0f1008a911bad3be32048fa8b4643b7a8e545984 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 11:42:34 -0700 Subject: [PATCH 080/128] Adding checks in prune_controller (Globus and Filesystem) for days_from_now < 0, which throws an error if it is negative --- orchestration/prune_controller.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index b4d01056..d3668bee 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -108,7 +108,7 @@ def prune( file_path (str): The path to the file or directory to prune source_endpoint (FileSystemEndpoint): The file system endpoint containing the data check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning - days_from_now (float): Delay in days before pruning; if 0.0, prune immediately. + days_from_now (float): Delay in days before pruning; if 0.0, prune immediately. If <0, throws error. Returns: bool: True if pruning was successful or scheduled successfully, False otherwise @@ -121,6 +121,10 @@ def prune( logger.error("No source_endpoint provided for pruning operation") return False + if days_from_now < 0: + logger.error("days_from_now cannot be negative") + return False + flow_name = f"prune_from_{source_endpoint.name}" logger.info(f"Setting up pruning of '{file_path}' from '{source_endpoint.name}'") @@ -252,7 +256,7 @@ def prune( file_path (str): The path to the file or directory to prune source_endpoint (FileSystemEndpoint): The file system endpoint containing the data check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning - days_from_now (datetime.timedelta): Delay before pruning; if 0, prune immediately + days_from_now (float): Delay before pruning; if 0, prune immediately. If <0, throws error. Returns: bool: True if pruning was successful or scheduled successfully, False otherwise @@ -265,6 +269,10 @@ def prune( logger.error("No source_endpoint provided for pruning operation") return False + if days_from_now < 0: + logger.error("days_from_now cannot be negative") + return False + # globus_settings = JSON.load("globus-settings").value # max_wait_seconds = globus_settings["max_wait_seconds"] flow_name = f"prune_from_{source_endpoint.name}" From 70e7ab19334c5eaca21275ee919a879c0f3f84d5 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 11:48:25 -0700 Subject: [PATCH 081/128] Adding a new pytest functino called test_globus_prune_schedules_when_days_from_now_negative --- orchestration/_tests/test_prune_controller.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index 84efb19f..b05edb45 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -300,6 +300,25 @@ def test_globus_prune_schedules_when_days_from_now_positive( assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 3.0 * 86400 +def test_globus_prune_schedules_when_days_from_now_negative( + globus_controller, + globus_endpoint, + tmp_path: Path,): + """Calling Globus prune with days_from_now<0 should return False.""" + rel = "sched.txt" + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + + result = globus_controller.prune( + file_path=rel, + source_endpoint=globus_endpoint, + check_endpoint=None, + days_from_now=-4.0, + ) + assert result is False + + def test_globus_prune_returns_false_if_schedule_raises( globus_controller, globus_endpoint, From e898be00a54fc3f9ec830e983cf860774d7e9a57 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 11:50:20 -0700 Subject: [PATCH 082/128] updating test_prune_controller with a new function test_fs_prune_schedules_when_days_from_now_negative --- orchestration/_tests/test_prune_controller.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index b05edb45..d72db655 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -212,6 +212,22 @@ def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoi assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 1.5 * 86400 +def test_fs_prune_schedules_when_days_from_now_negative(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler): + """Calling prune with days_from_now<0 should return False.""" + rel = "to_schedule.txt" + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + + result = fs_controller.prune( + file_path=rel, + source_endpoint=fs_endpoint, + check_endpoint=None, + days_from_now=-4.0, + ) + assert result is False + + def test_fs_prune_returns_false_if_schedule_raises(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler_raises): """If scheduling fails, fs_controller.prune should return False.""" rel = "error.txt" From 26f78fa7638c35ccd744b2aeab0fdfe69bc5cf5a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 13:27:24 -0700 Subject: [PATCH 083/128] Fixing tests for when days_from_now == 0 --- orchestration/_tests/test_prune_controller.py | 62 +++++++++++++++++-- orchestration/prune_controller.py | 34 ++++++---- 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index d72db655..d72c059b 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -7,6 +7,7 @@ from prefect.blocks.system import JSON from prefect.testing.utilities import prefect_test_harness +from orchestration.config import BeamlineConfig from orchestration.prune_controller import ( PruneController, FileSystemPruneController, @@ -30,11 +31,23 @@ def prefect_test_fixture(): yield -class MockConfig: - """Minimal config stub for controllers (only beamline_id and tc).""" +# class MockConfig: +# """Minimal config stub for controllers (only beamline_id and tc).""" +# def __init__(self, beamline_id: str = "test_beamline") -> None: +# self.beamline_id = beamline_id +# self.tc: Any = None # transfer client stub + +class MockConfig(BeamlineConfig): + """Minimal concrete BeamlineConfig for tests (no real I/O).""" def __init__(self, beamline_id: str = "test_beamline") -> None: - self.beamline_id = beamline_id - self.tc: Any = None # transfer client stub + super().__init__(beamline_id=beamline_id) + # Test stubs that the controllers/flows expect to exist + self.tc = None + + def _beam_specific_config(self) -> None: + # Keep it no-op for tests; you can set other attributes here if needed + # e.g., self.some_endpoint = ... + pass @pytest.fixture @@ -212,6 +225,23 @@ def test_fs_prune_schedules_when_days_from_now_positive(fs_controller, fs_endpoi assert pytest.approx(mock_scheduler["duration"].total_seconds()) == 1.5 * 86400 +def test_fs_prune_schedules_when_days_from_now_zero(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler): + """Calling prune with days_from_now==0 should schedule a Prefect flow.""" + rel = "to_schedule.txt" + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + + result = fs_controller.prune( + file_path=rel, + source_endpoint=fs_endpoint, + check_endpoint=None, + days_from_now=0.0, + ) + assert result is True + assert not p.exists() + + def test_fs_prune_schedules_when_days_from_now_negative(fs_controller, fs_endpoint, tmp_path: Path, mock_scheduler): """Calling prune with days_from_now<0 should return False.""" rel = "to_schedule.txt" @@ -335,6 +365,30 @@ def test_globus_prune_schedules_when_days_from_now_negative( assert result is False +def test_globus_prune_schedules_when_days_from_now_zero( + globus_controller, + globus_endpoint, + tmp_path: Path, + mock_scheduler, + mock_prune_one_safe): + """Calling Globus prune with days_from_now==0 should schedule a Prefect flow.""" + rel = "sched.txt" + p = tmp_path / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + + result = globus_controller.prune( + file_path=rel, + source_endpoint=globus_endpoint, + check_endpoint=None, + days_from_now=0.0 + ) + assert result is True + assert mock_prune_one_safe["file"] == rel + assert mock_prune_one_safe["if_older_than_days"] == 0 + assert mock_prune_one_safe["source_endpoint"] is globus_endpoint + + def test_globus_prune_returns_false_if_schedule_raises( globus_controller, globus_endpoint, diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index d3668bee..fb36d178 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -134,12 +134,17 @@ def prune( # If days_from_now is 0, prune immediately if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") - return self._prune_filesystem_endpoint( - relative_path=file_path, - source_endpoint=source_endpoint, - check_endpoint=check_endpoint, - config=self.config - ) + try: + self._prune_filesystem_endpoint( + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + config=self.config + ) + return True + except Exception as e: + logger.error(f"Failed to prune file: {str(e)}", exc_info=True) + return False else: # Otherwise, schedule pruning for future execution logger.info(f"Scheduling pruning of '{file_path}' from '{source_endpoint.name}' " @@ -284,12 +289,17 @@ def prune( # If days_from_now is 0, prune immediately if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") - return self._prune_globus_endpoint( - relative_path=file_path, - source_endpoint=source_endpoint, - check_endpoint=check_endpoint, - config=self.config - ) + try: + self._prune_globus_endpoint( + relative_path=file_path, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + config=self.config + ) + return True + except Exception as e: + logger.error(f"Failed to prune file: {str(e)}", exc_info=True) + return False else: # Otherwise, schedule pruning for future execution logger.info(f"Scheduling pruning of '{file_path}' from '{source_endpoint.name}' " From 30b2542d272ecaebababb7604002883b8f4d41f9 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 14:22:47 -0700 Subject: [PATCH 084/128] Using Pathlib instead of os.path --- orchestration/_tests/test_sfapi_flow.py | 19 +++++-------------- orchestration/sfapi.py | 14 +++++--------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/orchestration/_tests/test_sfapi_flow.py b/orchestration/_tests/test_sfapi_flow.py index c9cc5ab4..6d331098 100644 --- a/orchestration/_tests/test_sfapi_flow.py +++ b/orchestration/_tests/test_sfapi_flow.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -from unittest.mock import MagicMock, patch, mock_open +from unittest.mock import MagicMock, patch from uuid import uuid4 from prefect.blocks.system import Secret @@ -48,20 +48,11 @@ def test_create_sfapi_client_success(): mock_client_secret = '{"key": "value"}' # Create separate mock_open instances for each file - mock_open_client_id = mock_open(read_data=mock_client_id) - mock_open_client_secret = mock_open(read_data=mock_client_secret) - - with patch("orchestration.sfapi.os.path.isfile") as mock_isfile, \ - patch("builtins.open", side_effect=[ - mock_open_client_id.return_value, - mock_open_client_secret.return_value - ]), \ + with patch("orchestration.sfapi.Path.is_file", return_value=True), \ + patch("orchestration.sfapi.Path.read_text", side_effect=[mock_client_id, mock_client_secret]), \ patch("orchestration.sfapi.JsonWebKey.import_key") as mock_import_key, \ patch("orchestration.sfapi.Client") as MockClient: - # Simulate that both credential files exist - mock_isfile.return_value = True - # Mock key import to return a fake secret mock_import_key.return_value = "mock_secret" @@ -93,8 +84,8 @@ def test_create_sfapi_client_missing_files(): fake_client_id_path = "/path/to/client_id" fake_client_secret_path = "/path/to/client_secret" - # Simulate missing credential files by patching os.path.isfile to return False. - with patch("orchestration.sfapi.os.path.isfile", return_value=False): + # Simulate missing credential files by patching Path.is_file to return False. + with patch("orchestration.sfapi.Path.is_file", return_value=False): with pytest.raises(FileNotFoundError, match="NERSC credential files are missing."): create_sfapi_client(fake_client_id_path, fake_client_secret_path) diff --git a/orchestration/sfapi.py b/orchestration/sfapi.py index a0e24e22..f845443f 100644 --- a/orchestration/sfapi.py +++ b/orchestration/sfapi.py @@ -2,6 +2,7 @@ import json import logging import os +from pathlib import Path from authlib.jose import JsonWebKey from sfapi_client import Client @@ -24,17 +25,12 @@ def create_sfapi_client( if not client_id_path or not client_secret_path: logger.error("NERSC credentials paths are missing.") raise ValueError("NERSC credentials paths are missing.") - if not os.path.isfile(client_id_path) or not os.path.isfile(client_secret_path): - logger.error("NERSC credential files are missing.") + if not Path(client_id_path).is_file() or not Path(client_secret_path).is_file(): raise FileNotFoundError("NERSC credential files are missing.") - client_id = None - client_secret = None - with open(client_id_path, "r") as f: - client_id = f.read() - - with open(client_secret_path, "r") as f: - client_secret = JsonWebKey.import_key(json.loads(f.read())) + client_id = Path(client_id_path).read_text(encoding="utf-8").strip() + secret_text = Path(client_secret_path).read_text(encoding="utf-8") + client_secret = JsonWebKey.import_key(json.loads(secret_text)) try: client = Client(client_id, client_secret) From c63ff168a6cd1bcdb90cb5c165b2c6cb1d1d3def Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 14:26:03 -0700 Subject: [PATCH 085/128] removing redundat MockClient assertion --- orchestration/_tests/test_sfapi_flow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/orchestration/_tests/test_sfapi_flow.py b/orchestration/_tests/test_sfapi_flow.py index 6d331098..ee424f47 100644 --- a/orchestration/_tests/test_sfapi_flow.py +++ b/orchestration/_tests/test_sfapi_flow.py @@ -57,12 +57,10 @@ def test_create_sfapi_client_success(): mock_import_key.return_value = "mock_secret" # Create the client using the provided fake paths - client = create_sfapi_client(fake_client_id_path, fake_client_secret_path) + create_sfapi_client(fake_client_id_path, fake_client_secret_path) # Verify that Client was instantiated with the expected arguments MockClient.assert_called_once_with("value", "mock_secret") - # Assert that the returned client is the mocked Client instance - assert client == MockClient.return_value, "Client should be the mocked sfapi_client.Client instance" def test_create_sfapi_client_missing_paths(): From 6109c9232b5db37d32f7bb9aa6508758479f0ba6 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 16:22:29 -0700 Subject: [PATCH 086/128] Rewriting the filesystem transfercontroller to follow Kate's suggestions to make it less brittle. --- .../_tests/test_transfer_controller.py | 130 ++++++++++++++---- 1 file changed, 101 insertions(+), 29 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index 4bc27395..c4d086c1 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockFixture import time -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, Mock from uuid import uuid4 import globus_sdk @@ -317,40 +317,112 @@ def test_simple_transfer_controller_no_source_or_destination(mock_config832, tra assert result is False, "Expected False when either source or destination is None." -def test_simple_transfer_controller_copy_success( - mock_config832, mock_file_system_endpoint, transfer_controller_module +def test_simple_transfer_controller_copy_success_with_real_files( + tmp_path, mock_config832, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("orchestration.transfer_controller.os.path.exists", return_value=True): # patch in module namespace - with patch("orchestration.transfer_controller.os.system", return_value=0) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - assert result is True, "Expected True when os.system returns 0." - mock_os_system.assert_called_once() - command_called = mock_os_system.call_args[0][0] - assert "cp -r" in command_called, "Expected cp command in os.system call." + # Create real directory structure + source_dir = tmp_path / "source" + dest_dir = tmp_path / "destination" + source_dir.mkdir() + dest_dir.mkdir() -def test_simple_transfer_controller_copy_failure( - mock_config832, mock_file_system_endpoint, transfer_controller_module + # Create actual source file + source_file = source_dir / "experiment" / "data.txt" + source_file.parent.mkdir(parents=True) + source_file.write_text("test content") + + # Setup endpoints with real paths + source_endpoint = Mock() + source_endpoint.name = "source_storage" + source_endpoint.root_path = str(source_dir) + + dest_endpoint = Mock() + dest_endpoint.name = "dest_storage" + dest_endpoint.root_path = str(dest_dir) + + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="experiment/data.txt", + source=source_endpoint, + destination=dest_endpoint, + ) + + # Verify the result and actual file operations + assert result is True + + # Check that the file actually exists at destination + dest_file = dest_dir / "experiment" / "data.txt" + assert dest_file.exists(), "File should be copied to destination" + assert dest_file.read_text() == "test content", "File content should match" + + +# def test_simple_transfer_controller_copy_failure( +# mock_config832, mock_file_system_endpoint, transfer_controller_module +# ): +# SimpleTransferController = transfer_controller_module["SimpleTransferController"] +# with patch("orchestration.transfer_controller.os.path.exists", return_value=True): # ensure source file exists +# with patch("orchestration.transfer_controller.os.system", return_value=1) as mock_os_system: +# controller = SimpleTransferController(mock_config832) +# result = controller.copy( +# file_path="some_dir/test_file.txt", +# source=mock_file_system_endpoint, +# destination=mock_file_system_endpoint, +# ) +# assert result is False, "Expected False when os.system returns non-zero." +# mock_os_system.assert_called_once() +# command_called = mock_os_system.call_args[0][0] +# assert "cp -r" in command_called, "Expected cp command in os.system call." + +def test_simple_transfer_controller_copy_command_failure( + tmp_path, mock_config832, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("orchestration.transfer_controller.os.path.exists", return_value=True): # ensure source file exists - with patch("orchestration.transfer_controller.os.system", return_value=1) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - assert result is False, "Expected False when os.system returns non-zero." - mock_os_system.assert_called_once() - command_called = mock_os_system.call_args[0][0] - assert "cp -r" in command_called, "Expected cp command in os.system call." + + # Create real directory structure and source file + source_dir = tmp_path / "source" + dest_dir = tmp_path / "destination" + source_dir.mkdir() + dest_dir.mkdir() + + # Create the actual source file + source_file = source_dir / "some_dir" / "test_file.txt" + source_file.parent.mkdir(parents=True) + source_file.write_text("test content") + + # Setup endpoints with real paths + source_endpoint = Mock() + source_endpoint.root_path = str(source_dir) + + dest_endpoint = Mock() + dest_endpoint.root_path = str(dest_dir) + + # Mock only os.system to simulate command failure + with patch("orchestration.transfer_controller.os.system", return_value=1) as mock_os_system: + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="some_dir/test_file.txt", + source=source_endpoint, + destination=dest_endpoint, + ) + + # Verify the copy failed + assert result is False, "Expected False when os.system returns non-zero." + + # Verify the command was called correctly + mock_os_system.assert_called_once() + command_called = mock_os_system.call_args[0][0] + + # More specific assertions about the command + expected_source = str(source_dir / "some_dir" / "test_file.txt") + expected_dest = str(dest_dir / "some_dir" / "test_file.txt") + expected_command = f"cp -r '{expected_source}' '{expected_dest}'" + assert command_called == expected_command, f"Expected exact command: {expected_command}" + + # Verify the destination file was NOT created (since command failed) + dest_file = dest_dir / "some_dir" / "test_file.txt" + assert not dest_file.exists(), "File should not exist when copy command fails" def test_simple_transfer_controller_copy_exception( From 2b046dd5f52da62cfd9104f6d025b8f101061832 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 16:25:02 -0700 Subject: [PATCH 087/128] Rewriting the filesystem transfercontroller exception handling to follow Kate's suggestions to make it less brittle. --- .../_tests/test_transfer_controller.py | 62 +++++++++++++++---- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index c4d086c1..5f7c461f 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -425,20 +425,58 @@ def test_simple_transfer_controller_copy_command_failure( assert not dest_file.exists(), "File should not exist when copy command fails" -def test_simple_transfer_controller_copy_exception( - mock_config832, mock_file_system_endpoint, transfer_controller_module +def test_simple_transfer_controller_copy_exception_handling( + tmp_path, mock_config832, transfer_controller_module ): SimpleTransferController = transfer_controller_module["SimpleTransferController"] - with patch("orchestration.transfer_controller.os.path.exists", return_value=True): - with patch("orchestration.transfer_controller.os.system", side_effect=Exception("Mocked cp error")) as mock_os_system: - controller = SimpleTransferController(mock_config832) - result = controller.copy( - file_path="some_dir/test_file.txt", - source=mock_file_system_endpoint, - destination=mock_file_system_endpoint, - ) - assert result is False, "Expected False when an exception is raised during copy." - mock_os_system.assert_called_once() + + # Create real directory structure and source file + source_dir = tmp_path / "source" + dest_dir = tmp_path / "destination" + source_dir.mkdir() + dest_dir.mkdir() + + # Create the actual source file + source_file = source_dir / "some_dir" / "test_file.txt" + source_file.parent.mkdir(parents=True) + source_file.write_text("test content") + + # Setup endpoints with real paths + source_endpoint = Mock() + source_endpoint.root_path = str(source_dir) + + dest_endpoint = Mock() + dest_endpoint.root_path = str(dest_dir) + + # Mock os.system to raise an exception + with patch("orchestration.transfer_controller.os.system", side_effect=Exception("Mocked cp error")) as mock_os_system: + controller = SimpleTransferController(mock_config832) + result = controller.copy( + file_path="some_dir/test_file.txt", + source=source_endpoint, + destination=dest_endpoint, + ) + + # Verify the copy failed due to exception + assert result is False, "Expected False when an exception is raised during copy." + + # Verify the system command was attempted + mock_os_system.assert_called_once() + + # Verify the command that would have been called + command_called = mock_os_system.call_args[0][0] + expected_source = str(source_dir / "some_dir" / "test_file.txt") + expected_dest = str(dest_dir / "some_dir" / "test_file.txt") + expected_command = f"cp -r '{expected_source}' '{expected_dest}'" + assert command_called == expected_command, f"Expected exact command: {expected_command}" + + # Verify the destination file was NOT created (since exception occurred) + dest_file = dest_dir / "some_dir" / "test_file.txt" + assert not dest_file.exists(), "File should not exist when copy operation raises exception" + + # Verify destination directory was still created (this happens before the exception) + dest_parent_dir = dest_dir / "some_dir" + assert dest_parent_dir.exists(), "Destination directory should have been created before exception" # -------------------------------------------------------------------------- From 67626ff6e7df2f623a2eaceb45cb5b9323bd5d65 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 16:26:24 -0700 Subject: [PATCH 088/128] Linting --- orchestration/_tests/test_transfer_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestration/_tests/test_transfer_controller.py b/orchestration/_tests/test_transfer_controller.py index 5f7c461f..fad96601 100644 --- a/orchestration/_tests/test_transfer_controller.py +++ b/orchestration/_tests/test_transfer_controller.py @@ -324,7 +324,7 @@ def test_simple_transfer_controller_copy_success_with_real_files( # Create real directory structure source_dir = tmp_path / "source" - dest_dir = tmp_path / "destination" + dest_dir = tmp_path / "destination" source_dir.mkdir() dest_dir.mkdir() @@ -339,7 +339,7 @@ def test_simple_transfer_controller_copy_success_with_real_files( source_endpoint.root_path = str(source_dir) dest_endpoint = Mock() - dest_endpoint.name = "dest_storage" + dest_endpoint.name = "dest_storage" dest_endpoint.root_path = str(dest_dir) controller = SimpleTransferController(mock_config832) From e7125cca3392a689b004da7e71b1be566a17c42a Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 13 Oct 2025 17:01:47 -0700 Subject: [PATCH 089/128] Set self.config = None after assigning beamline-specific configs --- orchestration/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/config.py b/orchestration/config.py index a13014fc..1615491f 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -68,9 +68,9 @@ def __init__( beamline_id: str ) -> None: self.beamline_id = beamline_id - # self.config = read_config() self.config = settings self._beam_specific_config() + self.config = None # Clear reference to config after beam-specific setup @abstractmethod def _beam_specific_config(self) -> None: From f04b0aeef9f87ca72e01253036786879a960733e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 09:25:22 -0700 Subject: [PATCH 090/128] Enforcing consistency for beamline_id to be a str with numbers delimited by a period --- orchestration/_tests/test_prune_controller.py | 10 ++-------- orchestration/config.py | 7 ++++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index d72c059b..84631253 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -31,15 +31,9 @@ def prefect_test_fixture(): yield -# class MockConfig: -# """Minimal config stub for controllers (only beamline_id and tc).""" -# def __init__(self, beamline_id: str = "test_beamline") -> None: -# self.beamline_id = beamline_id -# self.tc: Any = None # transfer client stub - class MockConfig(BeamlineConfig): """Minimal concrete BeamlineConfig for tests (no real I/O).""" - def __init__(self, beamline_id: str = "test_beamline") -> None: + def __init__(self, beamline_id: str = "0.0.0") -> None: super().__init__(beamline_id=beamline_id) # Test stubs that the controllers/flows expect to exist self.tc = None @@ -53,7 +47,7 @@ def _beam_specific_config(self) -> None: @pytest.fixture def mock_config() -> MockConfig: """Provides a fresh MockConfig per test.""" - return MockConfig(beamline_id="unittest_beamline") + return MockConfig(beamline_id="0.0.0") @pytest.fixture diff --git a/orchestration/config.py b/orchestration/config.py index 1615491f..39946302 100644 --- a/orchestration/config.py +++ b/orchestration/config.py @@ -3,6 +3,7 @@ import collections import os from pathlib import Path +import re import yaml from dynaconf import Dynaconf @@ -59,7 +60,7 @@ class BeamlineConfig(ABC): must override the _setup_specific_config() method to assign their own attributes. Attributes: - beamline_id (str): Beamline identifier (e.g. "832" or "733"). + beamline_id (str): Beamline number identifier with periods (e.g. "8.3.2" or "7.3.3"). config (dict): The loaded configuration dictionary. """ @@ -67,6 +68,10 @@ def __init__( self, beamline_id: str ) -> None: + pattern = r'^\d+(\.\d+)+$' + if not re.match(pattern, beamline_id): + raise ValueError(f"Invalid beamline_id format: '{beamline_id}'." + f"Expected format: digits separated by dots (e.g., '8.3.2', '7.0.1.2', '12.3')") self.beamline_id = beamline_id self.config = settings self._beam_specific_config() From 81915befd6f7e17b4b9e48c6e662f88317990d53 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 09:31:26 -0700 Subject: [PATCH 091/128] Adding env var check to the start of the main method --- scripts/test_controllers_end_to_end.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index 5ef43f29..6e32b3f2 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -662,6 +662,12 @@ def test_it_all( # test_hpss=False, # test_scicat=False # ) + try: + check_required_envvars() + except Exception as e: + logger.error(f"Error checking environment variables: {e}") + finally: + logger.info("Continuing with tests...") # Test individual transfer controllers directly test_transfer_controllers( From a9db74376285dcb1b537ecb401f84da0309b29b1 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 09:49:06 -0700 Subject: [PATCH 092/128] Making the check_required_envars() function more verbose with which variables are missing --- scripts/test_controllers_end_to_end.py | 79 +++++++++++--------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index 6e32b3f2..c2ee1a4a 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -4,21 +4,23 @@ """ +from collections import defaultdict from datetime import datetime +from dotenv import load_dotenv import logging import os import shutil -from typing import Optional +from typing import Dict, List, Optional, Tuple from pyscicat.client import ScicatClient from orchestration.config import BeamlineConfig from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController -from orchestration.globus import flows, transfer +from orchestration.globus import transfer from orchestration.globus.transfer import GlobusEndpoint from orchestration.hpss import cfs_to_hpss_flow, hpss_to_cfs_flow from orchestration.prune_controller import get_prune_controller, PruneMethod -from orchestration.transfer_controller import get_transfer_controller, CopyMethod +# from orchestration.transfer_controller import get_transfer_controller, CopyMethod from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint from globus_sdk import TransferClient @@ -26,6 +28,9 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +load_dotenv() + + # ---------------------------------------------------------------------------------------------------------------------- # Setup Environment Configuration and Test Classes # ---------------------------------------------------------------------------------------------------------------------- @@ -35,37 +40,21 @@ def check_required_envvars() -> bool: """ Check for required environment variables before running the end-to-end tests. """ - missing_vars = [] - - # Check Globus environment variables - globus_vars = ['GLOBUS_CLIENT_ID', 'GLOBUS_CLIENT_SECRET'] - for var in globus_vars: - if not os.getenv(var): - missing_vars.append(var) - - # Check Prefect environment variables - prefect_vars = ['PREFECT_API_URL', 'PREFECT_API_KEY'] - for var in prefect_vars: - if not os.getenv(var): - missing_vars.append(var) - - # Check SciCat environment variables - scicat_vars = ['SCICAT_API_URL', 'SCICAT_INGEST_USER', 'SCICAT_INGEST_PASSWORD'] - for var in scicat_vars: - if not os.getenv(var): - missing_vars.append(var) - - # Check NERSC SFAPI environment variables - nersc_vars = ['PATH_NERSC_CLIENT_ID', 'PATH_NERSC_PRI_KEY'] - for var in nersc_vars: - if not os.getenv(var): - missing_vars.append(var) - - if missing_vars: - logger.error(f"Missing required environment variables: {', '.join(missing_vars)}") - logger.error("Please set these variables before running the end-to-end tests.") + to_check = ( + "GLOBUS_CLIENT_ID", "GLOBUS_CLIENT_SECRET", + "PREFECT_API_URL", "PREFECT_API_KEY", + "SCICAT_API_URL", "SCICAT_INGEST_USER", "SCICAT_INGEST_PASSWORD", + "PATH_NERSC_CLIENT_ID", "PATH_NERSC_PRI_KEY", + ) + missing = [] + for var in to_check: + logger.info(f"Checking environment variable: {var}") + if var not in os.environ or os.environ.get(var) is None: + logger.warning(f"Environment variable {var} is not set.") + missing.append(var) + if missing: + logger.error("Missing required environment variables: %s", ", ".join(missing)) return False - logger.info("All required environment variables are set.") return True @@ -78,6 +67,7 @@ def __init__(self) -> None: super().__init__(beamline_id="0.0.0") def _beam_specific_config(self) -> None: + from orchestration.globus import flows self.endpoints = transfer.build_endpoints(self.config) self.apps = transfer.build_apps(self.config) self.tc: TransferClient = transfer.init_transfer_client(self.apps["als_transfer"]) @@ -274,6 +264,8 @@ def test_transfer_controllers( Returns: None """ + from orchestration.transfer_controller import get_transfer_controller, CopyMethod + logger.info("Testing transfer controllers...") logger.info(f"File path: {file_path}") logger.info(f"Test Globus: {test_globus}") @@ -662,18 +654,13 @@ def test_it_all( # test_hpss=False, # test_scicat=False # ) - try: - check_required_envvars() - except Exception as e: - logger.error(f"Error checking environment variables: {e}") - finally: - logger.info("Continuing with tests...") + check_required_envvars() # Test individual transfer controllers directly - test_transfer_controllers( - file_path="test.txt", - test_globus=False, - test_filesystem=False, - test_hpss=False, - config=TestConfig() - ) + # test_transfer_controllers( + # file_path="test.txt", + # test_globus=False, + # test_filesystem=False, + # test_hpss=False, + # config=TestConfig() + # ) From 13509cae01917e58d0b222258495ac4217cfa68e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 09:52:03 -0700 Subject: [PATCH 093/128] Removing unused imports --- scripts/test_controllers_end_to_end.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index c2ee1a4a..fdcd7857 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -4,23 +4,22 @@ """ -from collections import defaultdict from datetime import datetime from dotenv import load_dotenv import logging import os import shutil -from typing import Dict, List, Optional, Tuple +from typing import Optional from pyscicat.client import ScicatClient from orchestration.config import BeamlineConfig from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController -from orchestration.globus import transfer +from orchestration.globus import flows, transfer from orchestration.globus.transfer import GlobusEndpoint from orchestration.hpss import cfs_to_hpss_flow, hpss_to_cfs_flow from orchestration.prune_controller import get_prune_controller, PruneMethod -# from orchestration.transfer_controller import get_transfer_controller, CopyMethod +from orchestration.transfer_controller import get_transfer_controller, CopyMethod from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint from globus_sdk import TransferClient @@ -67,7 +66,6 @@ def __init__(self) -> None: super().__init__(beamline_id="0.0.0") def _beam_specific_config(self) -> None: - from orchestration.globus import flows self.endpoints = transfer.build_endpoints(self.config) self.apps = transfer.build_apps(self.config) self.tc: TransferClient = transfer.init_transfer_client(self.apps["als_transfer"]) @@ -264,7 +262,6 @@ def test_transfer_controllers( Returns: None """ - from orchestration.transfer_controller import get_transfer_controller, CopyMethod logger.info("Testing transfer controllers...") logger.info(f"File path: {file_path}") @@ -657,10 +654,10 @@ def test_it_all( check_required_envvars() # Test individual transfer controllers directly - # test_transfer_controllers( - # file_path="test.txt", - # test_globus=False, - # test_filesystem=False, - # test_hpss=False, - # config=TestConfig() - # ) + test_transfer_controllers( + file_path="test.txt", + test_globus=False, + test_filesystem=False, + test_hpss=False, + config=TestConfig() + ) From ec2afc630c6990bd816ef39014d8718b204360f6 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:02:34 -0700 Subject: [PATCH 094/128] Adding better checks to make sure env variables are set --- orchestration/globus/flows.py | 16 ++++++++++++++-- scripts/test_controllers_end_to_end.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/orchestration/globus/flows.py b/orchestration/globus/flows.py index a27b3889..ff7e465b 100644 --- a/orchestration/globus/flows.py +++ b/orchestration/globus/flows.py @@ -10,6 +10,10 @@ from globus_sdk.tokenstorage import SimpleJSONFileAdapter from pprint import pprint from prefect.blocks.system import Secret +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) MY_FILE_ADAPTER = SimpleJSONFileAdapter(os.path.expanduser("~/.sdk-manage-flow.json")) @@ -19,8 +23,16 @@ dotenv_file = load_dotenv() -GLOBUS_CLIENT_ID = Secret.load("globus-client-id") -GLOBUS_CLIENT_SECRET = Secret.load("globus-client-secret") +if os.getenv("PREFECT_API_URL") and os.getenv("PREFECT_API_KEY"): + try: + GLOBUS_CLIENT_ID = Secret.load("globus-client-id") + GLOBUS_CLIENT_SECRET = Secret.load("globus-client-secret") + except Exception as e: + logger.error(f"Error loading Globus client credentials: {e}") + raise e +else: + logger.error("Prefect environment variables are not set.") + raise EnvironmentError("Prefect environment variables are not set.") def get_flows_client(): diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index fdcd7857..d635a823 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -15,7 +15,7 @@ from orchestration.config import BeamlineConfig from orchestration.flows.scicat.ingestor_controller import BeamlineIngestorController -from orchestration.globus import flows, transfer +from orchestration.globus import transfer from orchestration.globus.transfer import GlobusEndpoint from orchestration.hpss import cfs_to_hpss_flow, hpss_to_cfs_flow from orchestration.prune_controller import get_prune_controller, PruneMethod @@ -66,6 +66,7 @@ def __init__(self) -> None: super().__init__(beamline_id="0.0.0") def _beam_specific_config(self) -> None: + from orchestration.globus import flows # Wait to import here to ensure checked env vars first self.endpoints = transfer.build_endpoints(self.config) self.apps = transfer.build_apps(self.config) self.tc: TransferClient = transfer.init_transfer_client(self.apps["als_transfer"]) From 2f89be9e157a311eac11d2444c761483b7981bb4 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:35:21 -0700 Subject: [PATCH 095/128] Using pathlib to build the full path --- orchestration/transfer_endpoints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/orchestration/transfer_endpoints.py b/orchestration/transfer_endpoints.py index 55062774..6c25311e 100644 --- a/orchestration/transfer_endpoints.py +++ b/orchestration/transfer_endpoints.py @@ -1,5 +1,6 @@ # orchestration/transfer_endpoints.py from abc import ABC +from pathlib import Path class TransferEndpoint(ABC): @@ -63,9 +64,7 @@ def full_path( Returns: str: The full absolute path. """ - if path_suffix.startswith("/"): - path_suffix = path_suffix[1:] - return f"{self.root_path.rstrip('/')}/{path_suffix}" + return str(Path(self.root_path) / path_suffix) class HPSSEndpoint(TransferEndpoint): From c365640995cfd9fb27fbea2477d34c643c8d2e31 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:35:47 -0700 Subject: [PATCH 096/128] Updating docstring --- orchestration/transfer_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/transfer_endpoints.py b/orchestration/transfer_endpoints.py index 6c25311e..40a04180 100644 --- a/orchestration/transfer_endpoints.py +++ b/orchestration/transfer_endpoints.py @@ -31,7 +31,7 @@ def root_path(self) -> str: def uri(self) -> str: """ - Root path or base directory for this endpoint. + Uri for this endpoint. """ return self.uri From 45b56a9abfa85d9a6c0b16a9a5bef5f9f5a924d2 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:37:25 -0700 Subject: [PATCH 097/128] Including comment about circular dependencies --- orchestration/transfer_controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 107c9389..0325d8b9 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -431,6 +431,7 @@ def get_transfer_controller( return SimpleTransferController(config) elif transfer_type == CopyMethod.CFS_TO_HPSS: logger.debug("Importing and returning CFSToHPSSTransferController") + # Import here to avoid circular dependencies from orchestration.hpss import CFSToHPSSTransferController from orchestration.sfapi import create_sfapi_client return CFSToHPSSTransferController( @@ -439,6 +440,7 @@ def get_transfer_controller( ) elif transfer_type == CopyMethod.HPSS_TO_CFS: logger.debug("Importing and returning HPSSToCFSTransferController") + # Import here to avoid circular dependencies from orchestration.hpss import HPSSToCFSTransferController from orchestration.sfapi import create_sfapi_client return HPSSToCFSTransferController( From 3f69853e0eb50e30c5274e5ae22f17bac6e3fe38 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:39:58 -0700 Subject: [PATCH 098/128] Removing main block (tests are handled in scripts/test_controllers_end_to_end.py) --- orchestration/transfer_controller.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/orchestration/transfer_controller.py b/orchestration/transfer_controller.py index 0325d8b9..809e957e 100644 --- a/orchestration/transfer_controller.py +++ b/orchestration/transfer_controller.py @@ -451,27 +451,3 @@ def get_transfer_controller( error_msg = f"Invalid transfer type: {transfer_type}" logger.error(error_msg) raise ValueError(error_msg) - - -if __name__ == "__main__": - from orchestration.flows.bl832.config import Config832 - config = Config832() - transfer_type = CopyMethod.GLOBUS - globus_transfer_controller = get_transfer_controller(transfer_type, config) - globus_transfer_controller.copy( - file_path="dabramov/test.txt", - source=config.alcf832_raw, - destination=config.alcf832_scratch - ) - - simple_transfer_controller = get_transfer_controller(CopyMethod.SIMPLE, config) - success = simple_transfer_controller.copy( - file_path="test.rtf", - source=FileSystemEndpoint("source", "/Users/david/Documents/copy_test/test_source/"), - destination=FileSystemEndpoint("destination", "/Users/david/Documents/copy_test/test_destination/") - ) - - if success: - logger.info("Simple transfer succeeded.") - else: - logger.error("Simple transfer failed.") From 4992204ad86179a5bebc375c59785274da3b001f Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 10:45:16 -0700 Subject: [PATCH 099/128] Prefect/Globus env variable checks throw warnings rather than errors to prevent tests from failing --- orchestration/globus/flows.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/orchestration/globus/flows.py b/orchestration/globus/flows.py index ff7e465b..bbf35780 100644 --- a/orchestration/globus/flows.py +++ b/orchestration/globus/flows.py @@ -23,16 +23,14 @@ dotenv_file = load_dotenv() -if os.getenv("PREFECT_API_URL") and os.getenv("PREFECT_API_KEY"): - try: - GLOBUS_CLIENT_ID = Secret.load("globus-client-id") - GLOBUS_CLIENT_SECRET = Secret.load("globus-client-secret") - except Exception as e: - logger.error(f"Error loading Globus client credentials: {e}") - raise e -else: - logger.error("Prefect environment variables are not set.") - raise EnvironmentError("Prefect environment variables are not set.") +if not os.getenv("PREFECT_API_URL") and not os.getenv("PREFECT_API_KEY"): + logger.warning("Prefect environment variables are not set.") + +try: + GLOBUS_CLIENT_ID = Secret.load("globus-client-id") + GLOBUS_CLIENT_SECRET = Secret.load("globus-client-secret") +except Exception as e: + logger.warning(f"Error loading Globus client credentials: {e}") def get_flows_client(): From 74bc670ab173b254ea0cd05ac24aaa859162277f Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 14 Oct 2025 13:46:04 -0700 Subject: [PATCH 100/128] Elevating pruning flows out of the controller classes. --- orchestration/_tests/test_prune_controller.py | 8 +- orchestration/prune_controller.py | 157 +++++++++--------- 2 files changed, 84 insertions(+), 81 deletions(-) diff --git a/orchestration/_tests/test_prune_controller.py b/orchestration/_tests/test_prune_controller.py index 84631253..6f72db4f 100644 --- a/orchestration/_tests/test_prune_controller.py +++ b/orchestration/_tests/test_prune_controller.py @@ -14,6 +14,8 @@ GlobusPruneController, get_prune_controller, PruneMethod, + prune_filesystem_endpoint, + prune_globus_endpoint, ) from orchestration.transfer_endpoints import FileSystemEndpoint from orchestration.globus.transfer import GlobusEndpoint @@ -185,7 +187,7 @@ def test_fs_prune_immediate_deletes_file_directly(fs_controller, fs_endpoint, tm p.touch() assert p.exists() - fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore + fn = prune_filesystem_endpoint.fn # type: ignore assert fn(relative_path=rel, source_endpoint=fs_endpoint, check_endpoint=None, config=fs_controller.config) assert not p.exists() @@ -195,7 +197,7 @@ def test_fs_prune_immediate_returns_false_if_missing(fs_controller, fs_endpoint, rel = "no/such/file.txt" assert not (tmp_path / rel).exists() - fn = fs_controller._prune_filesystem_endpoint.fn # type: ignore + fn = prune_filesystem_endpoint.fn # type: ignore assert fn(relative_path=rel, source_endpoint=fs_endpoint, check_endpoint=None, config=fs_controller.config) is False @@ -279,7 +281,7 @@ def test_globus_prune_immediate_calls_prune_one_safe_directly( p.parent.mkdir(parents=True, exist_ok=True) p.touch() - fn = globus_controller._prune_globus_endpoint.fn # type: ignore + fn = prune_globus_endpoint.fn # type: ignore _ = fn( relative_path=rel, source_endpoint=globus_endpoint, diff --git a/orchestration/prune_controller.py b/orchestration/prune_controller.py index fb36d178..8710b244 100644 --- a/orchestration/prune_controller.py +++ b/orchestration/prune_controller.py @@ -135,7 +135,7 @@ def prune( if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") try: - self._prune_filesystem_endpoint( + prune_filesystem_endpoint( relative_path=file_path, source_endpoint=source_endpoint, check_endpoint=check_endpoint, @@ -168,57 +168,57 @@ def prune( logger.error(f"Failed to schedule pruning task: {str(e)}", exc_info=True) return False - @staticmethod - @flow(name="prune_filesystem_endpoint") - def _prune_filesystem_endpoint( - relative_path: str, - source_endpoint: FileSystemEndpoint, - check_endpoint: Optional[FileSystemEndpoint] = None, - config: BeamlineConfig = None - ) -> None: - """ - Prefect flow that performs the actual filesystem pruning operation. - Args: - relative_path (str): The path of the file or directory to prune - source_endpoint (FileSystemEndpoint): The source endpoint to prune from - check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning - config (Optional[BeamlineConfig]): Configuration object, if needed +@flow(name="prune_filesystem_endpoint") +def prune_filesystem_endpoint( + relative_path: str, + source_endpoint: FileSystemEndpoint, + check_endpoint: Optional[FileSystemEndpoint] = None, + config: BeamlineConfig = None +) -> None: + """ + Prefect flow that performs the actual filesystem pruning operation. - Returns: - bool: True if pruning was successful, False otherwise - """ - logger.info(f"Running flow: prune_from_{source_endpoint.name}") - logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + Args: + relative_path (str): The path of the file or directory to prune + source_endpoint (FileSystemEndpoint): The source endpoint to prune from + check_endpoint (Optional[FileSystemEndpoint]): If provided, verify data exists here before pruning + config (Optional[BeamlineConfig]): Configuration object, if needed - # Check if the file exists at the source endpoint using os.path - source_full_path = source_endpoint.full_path(relative_path) - if not os.path.exists(source_full_path): - logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") + Returns: + bool: True if pruning was successful, False otherwise + """ + logger.info(f"Running flow: prune_from_{source_endpoint.name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + + # Check if the file exists at the source endpoint using os.path + source_full_path = source_endpoint.full_path(relative_path) + if not os.path.exists(source_full_path): + logger.warning(f"File {relative_path} does not exist at the source: {source_endpoint.name}.") + return False + + # If check_endpoint is provided, verify file exists there before pruning + if check_endpoint is not None: + check_full_path = check_endpoint.full_path(relative_path) + if os.path.exists(check_full_path): + logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") + logger.info("Safe to prune.") + else: + logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") + logger.warning("Not safe to prune.") return False - # If check_endpoint is provided, verify file exists there before pruning - if check_endpoint is not None: - check_full_path = check_endpoint.full_path(relative_path) - if os.path.exists(check_full_path): - logger.info(f"File {relative_path} exists on the check point: {check_endpoint.name}.") - logger.info("Safe to prune.") - else: - logger.warning(f"File {relative_path} does not exist at the check point: {check_endpoint.name}.") - logger.warning("Not safe to prune.") - return False - - # Now perform the pruning operation - if os.path.isdir(source_full_path): - logger.info(f"Pruning directory {relative_path}") - import shutil - shutil.rmtree(source_full_path) - else: - logger.info(f"Pruning file {relative_path}") - os.remove(source_full_path) + # Now perform the pruning operation + if os.path.isdir(source_full_path): + logger.info(f"Pruning directory {relative_path}") + import shutil + shutil.rmtree(source_full_path) + else: + logger.info(f"Pruning file {relative_path}") + os.remove(source_full_path) - logger.info(f"Successfully pruned {relative_path} from {source_endpoint.name}") - return True + logger.info(f"Successfully pruned {relative_path} from {source_endpoint.name}") + return True class GlobusPruneController(PruneController[GlobusEndpoint]): @@ -290,7 +290,7 @@ def prune( if days_from_now.total_seconds() == 0: logger.info(f"Executing immediate pruning of '{file_path}' from '{source_endpoint.name}'") try: - self._prune_globus_endpoint( + prune_globus_endpoint( relative_path=file_path, source_endpoint=source_endpoint, check_endpoint=check_endpoint, @@ -323,39 +323,39 @@ def prune( logger.error(f"Failed to schedule pruning task: {str(e)}", exc_info=True) return False - @staticmethod - @flow(name="prune_globus_endpoint") - def _prune_globus_endpoint( - relative_path: str, - source_endpoint: GlobusEndpoint, - check_endpoint: Optional[GlobusEndpoint] = None, - config: BeamlineConfig = None - ) -> None: - """ - Prefect flow that performs the actual Globus endpoint pruning operation. - Args: - relative_path (str): The path of the file or directory to prune - source_endpoint (GlobusEndpoint): The Globus endpoint to prune from - check_endpoint (Optional[GlobusEndpoint]): If provided, verify data exists here before pruning - config (BeamlineConfig): Configuration object with transfer client - """ - logger.info(f"Running Globus pruning flow for '{relative_path}' from '{source_endpoint.name}'") +@flow(name="prune_globus_endpoint") +def prune_globus_endpoint( + relative_path: str, + source_endpoint: GlobusEndpoint, + check_endpoint: Optional[GlobusEndpoint] = None, + config: BeamlineConfig = None +) -> None: + """ + Prefect flow that performs the actual Globus endpoint pruning operation. - globus_settings = JSON.load("globus-settings").value - max_wait_seconds = globus_settings["max_wait_seconds"] - flow_name = f"prune_from_{source_endpoint.name}" - logger.info(f"Running flow: {flow_name}") - logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") - prune_one_safe( - file=relative_path, - if_older_than_days=0, - transfer_client=config.tc, - source_endpoint=source_endpoint, - check_endpoint=check_endpoint, - logger=logger, - max_wait_seconds=max_wait_seconds - ) + Args: + relative_path (str): The path of the file or directory to prune + source_endpoint (GlobusEndpoint): The Globus endpoint to prune from + check_endpoint (Optional[GlobusEndpoint]): If provided, verify data exists here before pruning + config (BeamlineConfig): Configuration object with transfer client + """ + logger.info(f"Running Globus pruning flow for '{relative_path}' from '{source_endpoint.name}'") + + globus_settings = JSON.load("globus-settings").value + max_wait_seconds = globus_settings["max_wait_seconds"] + flow_name = f"prune_from_{source_endpoint.name}" + logger.info(f"Running flow: {flow_name}") + logger.info(f"Pruning {relative_path} from source endpoint: {source_endpoint.name}") + prune_one_safe( + file=relative_path, + if_older_than_days=0, + transfer_client=config.tc, + source_endpoint=source_endpoint, + check_endpoint=check_endpoint, + logger=logger, + max_wait_seconds=max_wait_seconds + ) class PruneMethod(Enum): @@ -402,6 +402,7 @@ def get_prune_controller( return FileSystemPruneController(config) elif prune_type == PruneMethod.HPSS: logger.debug("Importing and returning HPSSPruneController") + # Import here to avoid circular dependencies from orchestration.hpss import HPSSPruneController from orchestration.sfapi import create_sfapi_client return HPSSPruneController( From fbb1a18ff2fdbb476041bd5d5ca9c96cfe43c598 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 10:06:54 -0700 Subject: [PATCH 101/128] Moving HPSS slurm jobs into a folder orchestration/slurm/ --- orchestration/hpss.py | 541 +++----------------------- orchestration/slurm/cfs_to_hpss.slurm | 323 +++++++++++++++ orchestration/slurm/hpss_to_cfs.slurm | 111 ++++++ orchestration/slurm/ls_hpss.slurm | 13 + orchestration/slurm/prune_hpss.slurm | 30 ++ 5 files changed, 537 insertions(+), 481 deletions(-) create mode 100644 orchestration/slurm/cfs_to_hpss.slurm create mode 100644 orchestration/slurm/hpss_to_cfs.slurm create mode 100644 orchestration/slurm/ls_hpss.slurm create mode 100644 orchestration/slurm/prune_hpss.slurm diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 338c4242..549fb288 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -35,6 +35,33 @@ logger.setLevel(logging.INFO) +""" +HPSS SLURM Template Loader + +Provides utilities for loading SLURM job script templates for HPSS operations. +""" + +# Directory containing SLURM template files + + +def load_slurm_job(job_name: str, **variables) -> str: + """ + Load and render a SLURM template with variable substitution. + + Args: + job_name: Name of the job (without .slurm extension) + **variables: Variables to substitute using .format() + + Returns: + str: The rendered SLURM script + """ + # Read slurm files from orchestration/slurm/ + TEMPLATES_DIR = Path(__file__).parent / "slurm" + slurm_path = TEMPLATES_DIR / f"{job_name}.slurm" + job = slurm_path.read_text() + return job.format(**variables) + + # --------------------------------- # HPSS Prefect Flows # --------------------------------- @@ -245,20 +272,14 @@ def list_hpss_slurm( ls_flag = "-R" if recursive else "" cmd = f'hsi ls {ls_flag} "{full_hpss}"' - job_script = rf"""#!/bin/bash -#SBATCH -q xfer -#SBATCH -A als -#SBATCH -C cron -#SBATCH --time=00:10:00 -#SBATCH --job-name={job_name} -#SBATCH --output={out_pattern} -#SBATCH --error={err_pattern} - -set -euo pipefail - -echo "[LOG] Listing HPSS path: {full_hpss}" -{cmd} -""" + job_script = load_slurm_job( + "ls_hpss", + job_name=job_name, + out_pattern=out_pattern, + err_pattern=err_pattern, + full_hpss=full_hpss, + cmd=cmd + ) # submit & wait perlmutter = client.compute(Machine.perlmutter) @@ -392,38 +413,13 @@ def _prune_hpss_endpoint( beamline_id = self.config.beamline_id logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" + job_script = load_slurm_job( + "prune_hpss", + relative_path=relative_path, + logs_path=logs_path, + source_endpoint=source_endpoint + ) - job_script = rf"""#!/bin/bash -# ------------------------------------------------------------------ -# SLURM Job Script for Pruning Data from HPSS -# ------------------------------------------------------------------ - -#SBATCH -q xfer # Specify the SLURM queue to use. -#SBATCH -A als # Specify the account. -#SBATCH -C cron # Use the 'cron' constraint. -#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_to_HPSS_{relative_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/{relative_path}_prune_from_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{relative_path}_prune_from_hpss_%j.err # Standard error log file. -#SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. - -set -euo pipefail # Enable strict error checking. -echo "[LOG] Job started at: $(date)" - -# Check if the file exists on HPSS -if hsi "ls {source_endpoint.full_path(relative_path)}" &> /dev/null; then - echo "[LOG] File {relative_path} exists on HPSS. Proceeding to prune." - # Prune the file from HPSS - hsi "rm {source_endpoint.full_path(relative_path)}" - echo "[LOG] File {relative_path} has been pruned from HPSS." - hsi ls -R {source_endpoint.full_path(relative_path)} -else - echo "[LOG] Could not find File {relative_path} does not on HPSS. Check your file path again." - exit 0 -fi -echo "[LOG] Job completed at: $(date)" -""" try: logger.info("Submitting HPSS transfer job to Perlmutter.") perlmutter = self.client.compute(Machine.perlmutter) @@ -563,330 +559,15 @@ def copy( logs_path = f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{beamline_id}" logger.info(f"Logs will be saved to: {logs_path}") # Build the SLURM job script with detailed inline comments for clarity. - job_script = rf"""#!/bin/bash -# ------------------------------------------------------------------ -# SLURM Job Script for Transferring Data from CFS to HPSS -# This script will: -# 1. Define the source (CFS) and destination (HPSS) paths. -# 2. Create the destination directory on HPSS if it doesn't exist. -# 3. Determine if the source is a file or a directory. -# - If a file, transfer it using 'hsi cput'. -# - If a directory, group files by beam cycle and archive them. -# * Cycle 1: Jan 1 - Jul 15 -# * Cycle 2: Jul 16 - Dec 31 -# * If a group exceeds 2 TB, it is partitioned into multiple tar archives. -# * Archive names: -# [proposal_name]_[year]-[cycle].tar -# [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. -# 4. Echo directory trees for both source and destination for logging. -# ------------------------------------------------------------------ - -#SBATCH -q xfer # Specify the SLURM queue to use. -#SBATCH -A als # Specify the account. -#SBATCH -C cron # Use the 'cron' constraint. -#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_to_HPSS_{proposal_name} # Set a descriptive job name. -#SBATCH --output={logs_path}/{proposal_name}_to_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{proposal_name}_to_hpss_%j.err # Standard error log file. -#SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. - -set -euo pipefail # Enable strict error checking. -echo "[LOG] Job started at: $(date)" - -# ------------------------------------------------------------------ -# Define source and destination variables. -# ------------------------------------------------------------------ - -echo "[LOG] Defining source and destination paths." - -# SOURCE_PATH: Full path of the file or directory on CFS. -SOURCE_PATH="{full_cfs_path}" -echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" - -# DEST_ROOT: Root destination on HPSS built from configuration. -DEST_ROOT="{hpss_root_path}" -echo "[LOG] DEST_ROOT set to: $DEST_ROOT" - -# FOLDER_NAME: Proposal name (project folder) derived from the file path. -FOLDER_NAME="{proposal_name}" -echo "[LOG] FOLDER_NAME set to: $FOLDER_NAME" - -# DEST_PATH: Final HPSS destination directory. -DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" -echo "[LOG] DEST_PATH set to: $DEST_PATH" - -# ------------------------------------------------------------------ -# Create destination directory on HPSS recursively using hsi mkdir. -# This section ensures that the entire directory tree specified in DEST_PATH -# exists on HPSS. Since HPSS hsi does not support a recursive mkdir option, -# we split the path into its components and create each directory one by one. -# ------------------------------------------------------------------ - -echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." - -# Use 'hsi ls' to verify if the destination directory exists. -# The '-q' flag is used for quiet mode, and any output or errors are discarded. -if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then - echo "[LOG] Destination directory $DEST_PATH already exists." -else - # If the directory does not exist, begin the process to create it. - echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." - - # Initialize an empty variable 'current' that will store the path built so far. - current="" - - # Split the DEST_PATH using '/' as the delimiter. - # This creates an array 'parts' where each element is a directory level in the path. - IFS='/' read -ra parts <<< "$DEST_PATH" - - # Iterate over each directory component in the 'parts' array. - for part in "${{parts[@]}}"; do - # Skip any empty parts. An empty string may occur if the path starts with a '/'. - if [ -z "$part" ]; then - continue - fi - - # Append the current part to the 'current' path variable. - # This step incrementally reconstructs the full path one directory at a time. - current="$current/$part" - - # Check if the current directory exists on HPSS using 'hsi ls'. - if ! hsi -q "ls $current" >/dev/null 2>&1; then - # If the directory does not exist, attempt to create it using 'hsi mkdir'. - if hsi "mkdir $current" >/dev/null 2>&1; then - echo "[LOG] Created directory $current." - else - echo "[ERROR] Failed to create directory $current." - exit 1 - fi - else - echo "[LOG] Directory $current already exists." - fi - done -fi - -# List the final HPSS directory tree for logging purposes. -# For some reason this gets logged in the project.err file, not the .out file. -hsi ls $DEST_PATH - -# ------------------------------------------------------------------ -# Transfer Logic: Check if SOURCE_PATH is a file or directory. -# ------------------------------------------------------------------ - -echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" -if [ -f "$SOURCE_PATH" ]; then - # Case: Single file detected. - echo "[LOG] Single file detected. Transferring via hsi cput." - FILE_NAME=$(basename "$SOURCE_PATH") - echo "[LOG] File name: $FILE_NAME" - hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" - echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." -elif [ -d "$SOURCE_PATH" ]; then - # Case: Directory detected. - echo "[LOG] Directory detected. Initiating bundling process." - - # ------------------------------------------------------------------ - # Define thresholds - # - THRESHOLD: maximum total size per HTAR archive (2 TB). - # - MEMBER_LIMIT: maximum size per member file in an HTAR (set to 65 GB). - # ------------------------------------------------------------------ - - THRESHOLD=2199023255552 # 2 TB in bytes. - MEMBER_LIMIT=$((65*1024**3)) # 65 GB in bytes. 68 GB is the htar limit. Move files >65 GB than this using hsi cput. - echo "[LOG] Threshold set to 2 TB (bytes): $THRESHOLD" - echo "[LOG] Threshold for individual file transfer (bytes): $MEMBER_LIMIT" - - # ------------------------------------------------------------------ - # Generate a list of relative file paths in the project directory. - # This list will be used to group files by their modification date. - # ------------------------------------------------------------------ - # Create a temporary file to store the list of relative file paths. - # Explanation: - # 1. FILE_LIST=$(mktemp) - # - mktemp creates a unique temporary file and its path is stored in FILE_LIST. - # - # 2. (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') - # - The parentheses run the commands in a subshell, so the directory change does not affect the current shell. - # - cd "$SOURCE_PATH": Changes the working directory to the source directory. - # - find . -type f: Recursively finds all files starting from the current directory (which is now SOURCE_PATH), - # outputting paths prefixed with "./". - # - sed 's|^\./||': Removes the leading "./" from each file path, resulting in relative paths without the prefix. - # - # 3. The output is then redirected into the temporary file specified by FILE_LIST. - # ------------------------------------------------------------------ - - echo "[LOG] Grouping files by modification date." - - FILE_LIST=$(mktemp) - (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') > "$FILE_LIST" - - echo "[LOG] List of files stored in temporary file: $FILE_LIST" - - # ------------------------------------------------------------------ - # Filter out oversized files (>65GB) for immediate transfer - # - For each file: - # ā€¢ If fsize > MEMBER_LIMIT: transfer via hsi cput. - # ā€¢ Else: add path to new list for bundling. - # ------------------------------------------------------------------ - - echo "[LOG] Beginning oversized-file filtering (> $MEMBER_LIMIT bytes)" - FILTERED_LIST=$(mktemp) - echo "[LOG] Writing remaining file paths to $FILTERED_LIST" - - while IFS= read -r f; do - # Absolute local path and size - full_local="$SOURCE_PATH/$f" - fsize=$(stat -c %s "$full_local") - - if (( fsize > MEMBER_LIMIT )); then - # Relative subdirectory and filename - rel_dir=$(dirname "$f") - fname=$(basename "$f") - - # Compute HPSS directory under project (create if needed) - if [ "$rel_dir" = "." ]; then - dest_dir="$DEST_PATH" - else - dest_dir="$DEST_PATH/$rel_dir" - fi - - if ! hsi -q "ls $dest_dir" >/dev/null 2>&1; then - echo "[LOG] Creating HPSS directory $dest_dir" - hsi mkdir "$dest_dir" - fi - - # Full remote file path (directory + filename) - remote_file="$dest_dir/$fname" - - # Transfer via conditional put - echo "[LOG] Transferring oversized file '$f' ($fsize bytes) to HPSS path $remote_file" - echo "[DEBUG] hsi cput \"$full_local\" : \"$remote_file\"" - hsi cput "$full_local" : "$remote_file" - echo "[LOG] Completed hsi cput for '$f'." - else - # Keep for bundling later - echo "$f" >> "$FILTERED_LIST" - fi - done < "$FILE_LIST" - - # Swap in the filtered list and report - mv "$FILTERED_LIST" "$FILE_LIST" - remaining=$(wc -l < "$FILE_LIST") - echo "[LOG] Oversized-file transfer done. Remaining for bundling: $remaining files." - - # ------------------------------------------------------------------ - # Cycle-based grouping & tar-bundling logic (unchanged). - # ------------------------------------------------------------------ - - # Declare associative arrays to hold grouped file paths and sizes. - declare -A group_files - declare -A group_sizes - - # ------------------------------------------------------------------ - # Group files by modification date. - # ------------------------------------------------------------------ - - cd "$SOURCE_PATH" && \ - while IFS= read -r file; do - mtime=$(stat -c %Y "$file") - year=$(date -d @"$mtime" +%Y) - month=$(date -d @"$mtime" +%m | sed 's/^0*//') - day=$(date -d @"$mtime" +%d | sed 's/^0*//') - # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. - if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then - cycle=1 - else - cycle=2 - fi - key="${{year}}-${{cycle}}" - group_files["$key"]="${{group_files["$key"]:-}} $file" - fsize=$(stat -c %s "$file") - group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) - done < "$FILE_LIST" - rm "$FILE_LIST" - echo "[LOG] Completed grouping files." - - # Print the files in each group at the end - for key in "${{!group_files[@]}}"; do - echo "[LOG] Group $key contains files:" - for f in ${{group_files["$key"]}}; do - echo " $f" - done - done - - # ------------------------------------------------------------------ - # Bundle files into tar archives. - # ------------------------------------------------------------------ - for key in "${{!group_files[@]}}"; do - files=(${{group_files["$key"]}}) - total_group_size=${{group_sizes["$key"]}} - echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." - - part=0 - current_size=0 - current_files=() - for f in "${{files[@]}}"; do - fsize=$(stat -c %s "$f") - # If adding this file exceeds the threshold, process the current bundle. - if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Bundle reached threshold." - echo "[LOG] Files in current bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." - (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) - part=$((part+1)) - echo "[DEBUG] Resetting bundle variables." - current_files=() - current_size=0 - fi - current_files+=("$f") - current_size=$(( current_size + fsize )) - done - if [ ${{#current_files[@]}} -gt 0 ]; then - if [ $part -eq 0 ]; then - tar_name="${{FOLDER_NAME}}_${{key}}.tar" - else - tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" - fi - echo "[LOG] Final bundle for group $key:" - echo "[LOG] Files in final bundle:" - for file in "${{current_files[@]}}"; do - echo "$file" - done - echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." - echo "[LOG] Bundle size: $current_size bytes." - (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) - fi - echo "[LOG] Completed processing group $key." - done -else - echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." - exit 1 -fi - -# ------------------------------------------------------------------ -# Logging: Display directory trees for both source and destination. -# ------------------------------------------------------------------ -echo "[LOG] Listing Source (CFS) Tree:" -if [ -d "$SOURCE_PATH" ]; then - find "$SOURCE_PATH" -print -else - echo "[LOG] $SOURCE_PATH is a file." -fi - -echo "[LOG] Listing Destination (HPSS) Tree:" -hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" - -echo "[LOG] Job completed at: $(date)" -""" + + job_script = load_slurm_job( + "cfs_to_hpss", + full_cfs_path=full_cfs_path, + hpss_root_path=hpss_root_path, + proposal_name=proposal_name, + logs_path=logs_path + ) + try: logger.info("Submitting HPSS transfer job to Perlmutter.") perlmutter = self.client.compute(Machine.perlmutter) @@ -1026,118 +707,16 @@ def copy( # - if HPSS_PATH ends with .tar, then if FILES_TO_EXTRACT is nonempty, MODE becomes "partial", # else MODE is "tar". # - Otherwise, MODE is "single" and hsi get is used. - job_script = fr"""#!/bin/bash -#SBATCH -q xfer # Specify the SLURM queue -#SBATCH -A als # Specify the account. -#SBATCH -C cron # Use the 'cron' constraint. -#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. -#SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. -#SBATCH --output={logs_path}/{sanitized_path}_from_hpss_%j.out # Standard output log file. -#SBATCH --error={logs_path}/{sanitized_path}_from_hpss_%j.err # Standard error log file. -#SBATCH --licenses=SCRATCH # Request the SCRATCH license. -#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. -set -euo pipefail # Enable strict error checking. -echo "[LOG] Job started at: $(date)" - -# ------------------------------------------------------------------- -# Define source and destination variables. -# ------------------------------------------------------------------- - -echo "[LOG] Defining source and destination paths." - -# SOURCE_PATH: Full path of the file or directory on HPSS. -SOURCE_PATH="{hpss_path}" -echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" - -# DEST_ROOT: Root destination on CFS built from configuration. -DEST_ROOT="{dest_root}" -echo "[LOG] DEST_ROOT set to: $DEST_ROOT" - -# FILES_TO_EXTRACT: Specific files to extract from the tar archive, if any. -# If not provided, this will be empty. -FILES_TO_EXTRACT="{files_to_extract_str}" -echo "[LOG] FILES_TO_EXTRACT set to: $FILES_TO_EXTRACT" - -# ------------------------------------------------------------------- -# Verify that SOURCE_PATH exists on HPSS using hsi ls. -# ------------------------------------------------------------------- - -echo "[LOG] Verifying file existence with hsi ls." -if ! hsi ls "$SOURCE_PATH" >/dev/null 2>&1; then - echo "[ERROR] File not found on HPSS: $SOURCE_PATH" - exit 1 -fi - -# ------------------------------------------------------------------- -# Determine the transfer mode based on the type (file vs tar). -# ------------------------------------------------------------------- - -echo "[LOG] Determining transfer mode based on the type (file vs tar)." - -# Check if SOURCE_PATH ends with .tar -if [[ "$SOURCE_PATH" =~ \.tar$ ]]; then - # If FILES_TO_EXTRACT is nonempty, MODE becomes "partial", else MODE is "tar". - if [ -n "${{FILES_TO_EXTRACT}}" ]; then - MODE="partial" - else - MODE="tar" - fi -else - MODE="single" -fi - -echo "Transfer mode: $MODE" - -# ------------------------------------------------------------------- -# Transfer Logic: Based on the mode, perform the appropriate transfer. -# ------------------------------------------------------------------- - -if [ "$MODE" = "single" ]; then - echo "[LOG] Single file detected. Using hsi get." - mkdir -p "$DEST_ROOT" - hsi get "$SOURCE_PATH" "$DEST_ROOT/" -elif [ "$MODE" = "tar" ]; then - echo "[LOG] Tar archive detected. Extracting entire archive using htar." - ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") - ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" - DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" - echo "[LOG] Extracting to: $DEST_PATH" - mkdir -p "$DEST_PATH" - htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" -elif [ "$MODE" = "partial" ]; then - echo "[LOG] Partial extraction detected. Extracting selected files using htar." - ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") - ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" - DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" - - # Verify that each requested file exists in the tar archive. - echo "[LOG] Verifying requested files are in the tar archive." - ARCHIVE_CONTENTS=$(htar -tvf "$SOURCE_PATH") - echo "[LOG] List: $ARCHIVE_CONTENTS" - for file in $FILES_TO_EXTRACT; do - echo "[LOG] Checking for file: $file" - if ! echo "$ARCHIVE_CONTENTS" | grep -q "$file"; then - echo "[ERROR] Requested file '$file' not found in archive $SOURCE_PATH" - exit 1 - else - echo "[LOG] File '$file' found in archive." - fi - done - - echo "[LOG] All requested files verified. Proceeding with extraction." - mkdir -p "$DEST_PATH" - (cd "$DEST_PATH" && htar -xvf "$SOURCE_PATH" -Hnostage $FILES_TO_EXTRACT) - - echo "[LOG] Extraction complete. Listing contents of $DEST_PATH:" - ls -l "$DEST_PATH" - -else - echo "[ERROR]: Unknown mode: $MODE" - exit 1 -fi - -date -""" + + job_script = load_slurm_job( + job_name="hpss_to_cfs", + logs_path=logs_path, + sanitized_path=sanitized_path, + hpss_path=hpss_path, + dest_root=dest_root, + files_to_extract_str=files_to_extract_str + ) + logger.info("Submitting HPSS to CFS transfer job to Perlmutter.") try: perlmutter = self.client.compute(Machine.perlmutter) diff --git a/orchestration/slurm/cfs_to_hpss.slurm b/orchestration/slurm/cfs_to_hpss.slurm new file mode 100644 index 00000000..ef210aa4 --- /dev/null +++ b/orchestration/slurm/cfs_to_hpss.slurm @@ -0,0 +1,323 @@ +#!/bin/bash +# ------------------------------------------------------------------ +# SLURM Job Script for Transferring Data from CFS to HPSS +# This script will: +# 1. Define the source (CFS) and destination (HPSS) paths. +# 2. Create the destination directory on HPSS if it doesn't exist. +# 3. Determine if the source is a file or a directory. +# - If a file, transfer it using 'hsi cput'. +# - If a directory, group files by beam cycle and archive them. +# * Cycle 1: Jan 1 - Jul 15 +# * Cycle 2: Jul 16 - Dec 31 +# * If a group exceeds 2 TB, it is partitioned into multiple tar archives. +# * Archive names: +# [proposal_name]_[year]-[cycle].tar +# [proposal_name]_[year]-[cycle]_part0.tar, _part1.tar, etc. +# 4. Echo directory trees for both source and destination for logging. +# ------------------------------------------------------------------ + +#SBATCH -q xfer # Specify the SLURM queue to use. +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_to_HPSS_{proposal_name} # Set a descriptive job name. +#SBATCH --output={logs_path}/{proposal_name}_to_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{proposal_name}_to_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. + +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# ------------------------------------------------------------------ +# Define source and destination variables. +# ------------------------------------------------------------------ + +echo "[LOG] Defining source and destination paths." + +# SOURCE_PATH: Full path of the file or directory on CFS. +SOURCE_PATH="{full_cfs_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" + +# DEST_ROOT: Root destination on HPSS built from configuration. +DEST_ROOT="{hpss_root_path}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" + +# FOLDER_NAME: Proposal name (project folder) derived from the file path. +FOLDER_NAME="{proposal_name}" +echo "[LOG] FOLDER_NAME set to: $FOLDER_NAME" + +# DEST_PATH: Final HPSS destination directory. +DEST_PATH="${{DEST_ROOT}}/${{FOLDER_NAME}}" +echo "[LOG] DEST_PATH set to: $DEST_PATH" + +# ------------------------------------------------------------------ +# Create destination directory on HPSS recursively using hsi mkdir. +# This section ensures that the entire directory tree specified in DEST_PATH +# exists on HPSS. Since HPSS hsi does not support a recursive mkdir option, +# we split the path into its components and create each directory one by one. +# ------------------------------------------------------------------ + +echo "[LOG] Checking if HPSS destination directory exists at $DEST_PATH." + +# Use 'hsi ls' to verify if the destination directory exists. +# The '-q' flag is used for quiet mode, and any output or errors are discarded. +if hsi -q "ls $DEST_PATH" >/dev/null 2>&1; then + echo "[LOG] Destination directory $DEST_PATH already exists." +else + # If the directory does not exist, begin the process to create it. + echo "[LOG] Destination directory $DEST_PATH does not exist. Attempting to create it recursively." + + # Initialize an empty variable 'current' that will store the path built so far. + current="" + + # Split the DEST_PATH using '/' as the delimiter. + # This creates an array 'parts' where each element is a directory level in the path. + IFS='/' read -ra parts <<< "$DEST_PATH" + + # Iterate over each directory component in the 'parts' array. + for part in "${{parts[@]}}"; do + # Skip any empty parts. An empty string may occur if the path starts with a '/'. + if [ -z "$part" ]; then + continue + fi + + # Append the current part to the 'current' path variable. + # This step incrementally reconstructs the full path one directory at a time. + current="$current/$part" + + # Check if the current directory exists on HPSS using 'hsi ls'. + if ! hsi -q "ls $current" >/dev/null 2>&1; then + # If the directory does not exist, attempt to create it using 'hsi mkdir'. + if hsi "mkdir $current" >/dev/null 2>&1; then + echo "[LOG] Created directory $current." + else + echo "[ERROR] Failed to create directory $current." + exit 1 + fi + else + echo "[LOG] Directory $current already exists." + fi + done +fi + +# List the final HPSS directory tree for logging purposes. +# For some reason this gets logged in the project.err file, not the .out file. +hsi ls $DEST_PATH + +# ------------------------------------------------------------------ +# Transfer Logic: Check if SOURCE_PATH is a file or directory. +# ------------------------------------------------------------------ + +echo "[LOG] Determining type of SOURCE_PATH: $SOURCE_PATH" +if [ -f "$SOURCE_PATH" ]; then + # Case: Single file detected. + echo "[LOG] Single file detected. Transferring via hsi cput." + FILE_NAME=$(basename "$SOURCE_PATH") + echo "[LOG] File name: $FILE_NAME" + hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" + echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." +elif [ -d "$SOURCE_PATH" ]; then + # Case: Directory detected. + echo "[LOG] Directory detected. Initiating bundling process." + + # ------------------------------------------------------------------ + # Define thresholds + # - THRESHOLD: maximum total size per HTAR archive (2 TB). + # - MEMBER_LIMIT: maximum size per member file in an HTAR (set to 65 GB). + # ------------------------------------------------------------------ + + THRESHOLD=2199023255552 # 2 TB in bytes. + MEMBER_LIMIT=$((65*1024**3)) # 65 GB in bytes. 68 GB is the htar limit. Move files >65 GB than this using hsi cput. + echo "[LOG] Threshold set to 2 TB (bytes): $THRESHOLD" + echo "[LOG] Threshold for individual file transfer (bytes): $MEMBER_LIMIT" + + # ------------------------------------------------------------------ + # Generate a list of relative file paths in the project directory. + # This list will be used to group files by their modification date. + # ------------------------------------------------------------------ + # Create a temporary file to store the list of relative file paths. + # Explanation: + # 1. FILE_LIST=$(mktemp) + # - mktemp creates a unique temporary file and its path is stored in FILE_LIST. + # + # 2. (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') + # - The parentheses run the commands in a subshell, so the directory change does not affect the current shell. + # - cd "$SOURCE_PATH": Changes the working directory to the source directory. + # - find . -type f: Recursively finds all files starting from the current directory (which is now SOURCE_PATH), + # outputting paths prefixed with "./". + # - sed 's|^\./||': Removes the leading "./" from each file path, resulting in relative paths without the prefix. + # + # 3. The output is then redirected into the temporary file specified by FILE_LIST. + # ------------------------------------------------------------------ + + echo "[LOG] Grouping files by modification date." + + FILE_LIST=$(mktemp) + (cd "$SOURCE_PATH" && find . -type f | sed 's|^\./||') > "$FILE_LIST" + + echo "[LOG] List of files stored in temporary file: $FILE_LIST" + + # ------------------------------------------------------------------ + # Filter out oversized files (>65GB) for immediate transfer + # - For each file: + # ā€¢ If fsize > MEMBER_LIMIT: transfer via hsi cput. + # ā€¢ Else: add path to new list for bundling. + # ------------------------------------------------------------------ + + echo "[LOG] Beginning oversized-file filtering (> $MEMBER_LIMIT bytes)" + FILTERED_LIST=$(mktemp) + echo "[LOG] Writing remaining file paths to $FILTERED_LIST" + + while IFS= read -r f; do + # Absolute local path and size + full_local="$SOURCE_PATH/$f" + fsize=$(stat -c %s "$full_local") + + if (( fsize > MEMBER_LIMIT )); then + # Relative subdirectory and filename + rel_dir=$(dirname "$f") + fname=$(basename "$f") + + # Compute HPSS directory under project (create if needed) + if [ "$rel_dir" = "." ]; then + dest_dir="$DEST_PATH" + else + dest_dir="$DEST_PATH/$rel_dir" + fi + + if ! hsi -q "ls $dest_dir" >/dev/null 2>&1; then + echo "[LOG] Creating HPSS directory $dest_dir" + hsi mkdir "$dest_dir" + fi + + # Full remote file path (directory + filename) + remote_file="$dest_dir/$fname" + + # Transfer via conditional put + echo "[LOG] Transferring oversized file '$f' ($fsize bytes) to HPSS path $remote_file" + echo "[DEBUG] hsi cput \"$full_local\" : \"$remote_file\"" + hsi cput "$full_local" : "$remote_file" + echo "[LOG] Completed hsi cput for '$f'." + else + # Keep for bundling later + echo "$f" >> "$FILTERED_LIST" + fi + done < "$FILE_LIST" + + # Swap in the filtered list and report + mv "$FILTERED_LIST" "$FILE_LIST" + remaining=$(wc -l < "$FILE_LIST") + echo "[LOG] Oversized-file transfer done. Remaining for bundling: $remaining files." + + # ------------------------------------------------------------------ + # Cycle-based grouping & tar-bundling logic (unchanged). + # ------------------------------------------------------------------ + + # Declare associative arrays to hold grouped file paths and sizes. + declare -A group_files + declare -A group_sizes + + # ------------------------------------------------------------------ + # Group files by modification date. + # ------------------------------------------------------------------ + + cd "$SOURCE_PATH" && \ + while IFS= read -r file; do + mtime=$(stat -c %Y "$file") + year=$(date -d @"$mtime" +%Y) + month=$(date -d @"$mtime" +%m | sed 's/^0*//') + day=$(date -d @"$mtime" +%d | sed 's/^0*//') + # Determine cycle: Cycle 1 if month < 7 or (month == 7 and day <= 15), else Cycle 2. + if [ "$month" -lt 7 ] || {{ [ "$month" -eq 7 ] && [ "$day" -le 15 ]; }}; then + cycle=1 + else + cycle=2 + fi + key="${{year}}-${{cycle}}" + group_files["$key"]="${{group_files["$key"]:-}} $file" + fsize=$(stat -c %s "$file") + group_sizes["$key"]=$(( ${{group_sizes["$key"]:-0}} + fsize )) + done < "$FILE_LIST" + rm "$FILE_LIST" + echo "[LOG] Completed grouping files." + + # Print the files in each group at the end + for key in "${{!group_files[@]}}"; do + echo "[LOG] Group $key contains files:" + for f in ${{group_files["$key"]}}; do + echo " $f" + done + done + + # ------------------------------------------------------------------ + # Bundle files into tar archives. + # ------------------------------------------------------------------ + for key in "${{!group_files[@]}}"; do + files=(${{group_files["$key"]}}) + total_group_size=${{group_sizes["$key"]}} + echo "[LOG] Processing group $key with ${{#files[@]}} files; total size: $total_group_size bytes." + + part=0 + current_size=0 + current_files=() + for f in "${{files[@]}}"; do + fsize=$(stat -c %s "$f") + # If adding this file exceeds the threshold, process the current bundle. + if (( current_size + fsize > THRESHOLD && ${{#current_files[@]}} > 0 )); then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Bundle reached threshold." + echo "[LOG] Files in current bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating archive $tar_name with ${{#current_files[@]}} files; bundle size: $current_size bytes." + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) + part=$((part+1)) + echo "[DEBUG] Resetting bundle variables." + current_files=() + current_size=0 + fi + current_files+=("$f") + current_size=$(( current_size + fsize )) + done + if [ ${{#current_files[@]}} -gt 0 ]; then + if [ $part -eq 0 ]; then + tar_name="${{FOLDER_NAME}}_${{key}}.tar" + else + tar_name="${{FOLDER_NAME}}_${{key}}_part${{part}}.tar" + fi + echo "[LOG] Final bundle for group $key:" + echo "[LOG] Files in final bundle:" + for file in "${{current_files[@]}}"; do + echo "$file" + done + echo "[LOG] Creating final archive $tar_name with ${{#current_files[@]}} files." + echo "[LOG] Bundle size: $current_size bytes." + (cd "$SOURCE_PATH" && htar -cvf "${{DEST_PATH}}/${{tar_name}}" $(printf "%s " "${{current_files[@]}}")) + fi + echo "[LOG] Completed processing group $key." + done +else + echo "[ERROR] $SOURCE_PATH is neither a file nor a directory. Exiting." + exit 1 +fi + +# ------------------------------------------------------------------ +# Logging: Display directory trees for both source and destination. +# ------------------------------------------------------------------ +echo "[LOG] Listing Source (CFS) Tree:" +if [ -d "$SOURCE_PATH" ]; then + find "$SOURCE_PATH" -print +else + echo "[LOG] $SOURCE_PATH is a file." +fi + +echo "[LOG] Listing Destination (HPSS) Tree:" +hsi ls -R "$DEST_PATH" || echo "[ERROR] Failed to list HPSS tree at $DEST_PATH" + +echo "[LOG] Job completed at: $(date)" \ No newline at end of file diff --git a/orchestration/slurm/hpss_to_cfs.slurm b/orchestration/slurm/hpss_to_cfs.slurm new file mode 100644 index 00000000..8dc6b180 --- /dev/null +++ b/orchestration/slurm/hpss_to_cfs.slurm @@ -0,0 +1,111 @@ +#!/bin/bash +#SBATCH -q xfer # Specify the SLURM queue +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_from_HPSS_{file_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/{sanitized_path}_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{sanitized_path}_from_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# ------------------------------------------------------------------- +# Define source and destination variables. +# ------------------------------------------------------------------- + +echo "[LOG] Defining source and destination paths." + +# SOURCE_PATH: Full path of the file or directory on HPSS. +SOURCE_PATH="{hpss_path}" +echo "[LOG] SOURCE_PATH set to: $SOURCE_PATH" + +# DEST_ROOT: Root destination on CFS built from configuration. +DEST_ROOT="{dest_root}" +echo "[LOG] DEST_ROOT set to: $DEST_ROOT" + +# FILES_TO_EXTRACT: Specific files to extract from the tar archive, if any. +# If not provided, this will be empty. +FILES_TO_EXTRACT="{files_to_extract_str}" +echo "[LOG] FILES_TO_EXTRACT set to: $FILES_TO_EXTRACT" + +# ------------------------------------------------------------------- +# Verify that SOURCE_PATH exists on HPSS using hsi ls. +# ------------------------------------------------------------------- + +echo "[LOG] Verifying file existence with hsi ls." +if ! hsi ls "$SOURCE_PATH" >/dev/null 2>&1; then + echo "[ERROR] File not found on HPSS: $SOURCE_PATH" + exit 1 +fi + +# ------------------------------------------------------------------- +# Determine the transfer mode based on the type (file vs tar). +# ------------------------------------------------------------------- + +echo "[LOG] Determining transfer mode based on the type (file vs tar)." + +# Check if SOURCE_PATH ends with .tar +if [[ "$SOURCE_PATH" =~ \.tar$ ]]; then + # If FILES_TO_EXTRACT is nonempty, MODE becomes "partial", else MODE is "tar". + if [ -n "${{FILES_TO_EXTRACT}}" ]; then + MODE="partial" + else + MODE="tar" + fi +else + MODE="single" +fi + +echo "Transfer mode: $MODE" + +# ------------------------------------------------------------------- +# Transfer Logic: Based on the mode, perform the appropriate transfer. +# ------------------------------------------------------------------- + +if [ "$MODE" = "single" ]; then + echo "[LOG] Single file detected. Using hsi get." + mkdir -p "$DEST_ROOT" + hsi get "$SOURCE_PATH" "$DEST_ROOT/" +elif [ "$MODE" = "tar" ]; then + echo "[LOG] Tar archive detected. Extracting entire archive using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + echo "[LOG] Extracting to: $DEST_PATH" + mkdir -p "$DEST_PATH" + htar -xvf "$SOURCE_PATH" -C "$DEST_PATH" +elif [ "$MODE" = "partial" ]; then + echo "[LOG] Partial extraction detected. Extracting selected files using htar." + ARCHIVE_BASENAME=$(basename "$SOURCE_PATH") + ARCHIVE_NAME="${{ARCHIVE_BASENAME%.tar}}" + DEST_PATH="${{DEST_ROOT}}/${{ARCHIVE_NAME}}" + + # Verify that each requested file exists in the tar archive. + echo "[LOG] Verifying requested files are in the tar archive." + ARCHIVE_CONTENTS=$(htar -tvf "$SOURCE_PATH") + echo "[LOG] List: $ARCHIVE_CONTENTS" + for file in $FILES_TO_EXTRACT; do + echo "[LOG] Checking for file: $file" + if ! echo "$ARCHIVE_CONTENTS" | grep -q "$file"; then + echo "[ERROR] Requested file '$file' not found in archive $SOURCE_PATH" + exit 1 + else + echo "[LOG] File '$file' found in archive." + fi + done + + echo "[LOG] All requested files verified. Proceeding with extraction." + mkdir -p "$DEST_PATH" + (cd "$DEST_PATH" && htar -xvf "$SOURCE_PATH" -Hnostage $FILES_TO_EXTRACT) + + echo "[LOG] Extraction complete. Listing contents of $DEST_PATH:" + ls -l "$DEST_PATH" + +else + echo "[ERROR]: Unknown mode: $MODE" + exit 1 +fi + +date \ No newline at end of file diff --git a/orchestration/slurm/ls_hpss.slurm b/orchestration/slurm/ls_hpss.slurm new file mode 100644 index 00000000..a47157fb --- /dev/null +++ b/orchestration/slurm/ls_hpss.slurm @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH -q xfer +#SBATCH -A als +#SBATCH -C cron +#SBATCH --time=00:10:00 +#SBATCH --job-name={job_name} +#SBATCH --output={out_pattern} +#SBATCH --error={err_pattern} + +set -euo pipefail + +echo "[LOG] Listing HPSS path: {full_hpss}" +{cmd} \ No newline at end of file diff --git a/orchestration/slurm/prune_hpss.slurm b/orchestration/slurm/prune_hpss.slurm new file mode 100644 index 00000000..bb01f11e --- /dev/null +++ b/orchestration/slurm/prune_hpss.slurm @@ -0,0 +1,30 @@ +#!/bin/bash +# ------------------------------------------------------------------ +# SLURM Job Script for Pruning Data from HPSS +# ------------------------------------------------------------------ + +#SBATCH -q xfer # Specify the SLURM queue to use. +#SBATCH -A als # Specify the account. +#SBATCH -C cron # Use the 'cron' constraint. +#SBATCH --time=12:00:00 # Maximum runtime of 12 hours. +#SBATCH --job-name=transfer_to_HPSS_{relative_path} # Set a descriptive job name. +#SBATCH --output={logs_path}/{relative_path}_prune_from_hpss_%j.out # Standard output log file. +#SBATCH --error={logs_path}/{relative_path}_prune_from_hpss_%j.err # Standard error log file. +#SBATCH --licenses=SCRATCH # Request the SCRATCH license. +#SBATCH --mem=20GB # Request #GB of memory. Default 2GB. + +set -euo pipefail # Enable strict error checking. +echo "[LOG] Job started at: $(date)" + +# Check if the file exists on HPSS +if hsi "ls {source_endpoint.full_path(relative_path)}" &> /dev/null; then + echo "[LOG] File {relative_path} exists on HPSS. Proceeding to prune." + # Prune the file from HPSS + hsi "rm {source_endpoint.full_path(relative_path)}" + echo "[LOG] File {relative_path} has been pruned from HPSS." + hsi ls -R {source_endpoint.full_path(relative_path)} +else + echo "[LOG] Could not find File {relative_path} does not on HPSS. Check your file path again." + exit 0 +fi +echo "[LOG] Job completed at: $(date) \ No newline at end of file From fcb1cb22112dab960970f3df94bf688f8b65df8d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 10:22:36 -0700 Subject: [PATCH 102/128] Ensuring all variables are passed into hpss_to_cfs.slurm --- orchestration/hpss.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/orchestration/hpss.py b/orchestration/hpss.py index 549fb288..dafd3cee 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -709,8 +709,9 @@ def copy( # - Otherwise, MODE is "single" and hsi get is used. job_script = load_slurm_job( - job_name="hpss_to_cfs", + "hpss_to_cfs", logs_path=logs_path, + file_path=file_path, sanitized_path=sanitized_path, hpss_path=hpss_path, dest_root=dest_root, From 2cf5e44c4966f56615a771243ac4ba58b019de9d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 14:57:00 -0700 Subject: [PATCH 103/128] moving hpss main method to it's own check_hpss script --- orchestration/hpss.py | 128 +-------------------------------------- scripts/check_hpss.py | 137 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 127 deletions(-) create mode 100644 scripts/check_hpss.py diff --git a/orchestration/hpss.py b/orchestration/hpss.py index dafd3cee..ab236f16 100644 --- a/orchestration/hpss.py +++ b/orchestration/hpss.py @@ -27,7 +27,7 @@ from orchestration.config import BeamlineConfig from orchestration.prefect import schedule_prefect_flow -from orchestration.prune_controller import get_prune_controller, PruneController, PruneMethod +from orchestration.prune_controller import PruneController from orchestration.transfer_controller import get_transfer_controller, CopyMethod, TransferController from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint @@ -753,129 +753,3 @@ def copy( return False else: return False - - -if __name__ == "__main__": - TEST_HPSS_PRUNE = False - TEST_CFS_TO_HPSS = False - TEST_HPSS_TO_CFS = False - TEST_HPSS_LS = True - - # ------------------------------------------------------ - # Test pruning from HPSS - # ------------------------------------------------------ - if TEST_HPSS_PRUNE: - from orchestration.flows.bl832.config import Config832 - config = Config832() - file_name = "8.3.2/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" - source = HPSSEndpoint( - name="HPSS", - root_path=config.hpss_alsdev["root_path"], - uri=config.hpss_alsdev["uri"] - ) - - days_from_now = datetime.timedelta(days=0) # Prune immediately - - prune_controller = get_prune_controller( - prune_type=PruneMethod.HPSS, - config=config - ) - prune_controller.prune( - file_path=f"{file_name}", - source_endpoint=source, - check_endpoint=None, - days_from_now=days_from_now - ) - # ------------------------------------------------------ - # Test transfer from CFS to HPSS - # ------------------------------------------------------ - if TEST_CFS_TO_HPSS: - from orchestration.flows.bl832.config import Config832 - config = Config832() - project_name = "ALS-11193_nbalsara" - source = FileSystemEndpoint( - name="CFS", - root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", - uri="nersc.gov" - ) - destination = HPSSEndpoint( - name="HPSS", - root_path=config.hpss_alsdev["root_path"], - uri=config.hpss_alsdev["uri"] - ) - cfs_to_hpss_flow( - file_path=f"{project_name}", - source=source, - destination=destination, - config=config - ) - - # ------------------------------------------------------ - # Test transfer from HPSS to CFS - # ------------------------------------------------------ - if TEST_HPSS_TO_CFS: - from orchestration.flows.bl832.config import Config832 - config = Config832() - relative_file_path = f"{config.beamline_id}/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" - source = HPSSEndpoint( - name="HPSS", - root_path=config.hpss_alsdev["root_path"], # root_path: /home/a/alsdev/data_mover - uri=config.hpss_alsdev["uri"] - ) - destination = FileSystemEndpoint( - name="CFS", - root_path="/global/cfs/cdirs/als/data_mover/8.3.2/retrieved_from_tape", - uri="nersc.gov" - ) - - files_to_extract = [ - "20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", - "20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", - ] - - hpss_to_cfs_flow( - file_path=f"{relative_file_path}", - source=source, - destination=destination, - files_to_extract=files_to_extract, - config=config - ) - - # ------------------------------------------------------ - # Test listing HPSS files - # ------------------------------------------------------ - if TEST_HPSS_LS: - from orchestration.flows.bl832.config import Config832 - - # Build client, config, endpoint - config = Config832() - endpoint = HPSSEndpoint( - name="HPSS", - root_path=config.hpss_alsdev["root_path"], - uri=config.hpss_alsdev["uri"] - ) - - # Instantiate controller - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.CFS_TO_HPSS, - config=config - ) - - # Directory listing - project_path = f"{config.beamline_id}/raw/BLS-00564_dyparkinson" - logger.info("Controller-based directory listing on HPSS:") - output_file = transfer_controller.list_hpss( - endpoint=endpoint, - remote_path=project_path, - recursive=True - ) - - # TAR archive listing - archive_name = project_path.split("/")[-1] - tar_path = f"{project_path}/{archive_name}_2023-1.tar" - logger.info("Controller-based tar archive listing on HPSS:") - output_file = transfer_controller.list_hpss( - endpoint=endpoint, - remote_path=tar_path, - recursive=False - ) diff --git a/scripts/check_hpss.py b/scripts/check_hpss.py new file mode 100644 index 00000000..d625da70 --- /dev/null +++ b/scripts/check_hpss.py @@ -0,0 +1,137 @@ +from orchestration.hpss import ( + cfs_to_hpss_flow, + hpss_to_cfs_flow, + get_prune_controller, + get_transfer_controller, + PruneMethod, + CopyMethod, +) +from orchestration.transfer_endpoints import FileSystemEndpoint, HPSSEndpoint +import datetime +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) +TEST_HPSS_PRUNE = False +TEST_CFS_TO_HPSS = False +TEST_HPSS_TO_CFS = False +TEST_HPSS_LS = True + +# ------------------------------------------------------ +# Test pruning from HPSS +# ------------------------------------------------------ +if TEST_HPSS_PRUNE: + from orchestration.flows.bl832.config import Config832 + config = Config832() + file_name = "8.3.2/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" + source = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + + days_from_now = datetime.timedelta(days=0) # Prune immediately + + prune_controller = get_prune_controller( + prune_type=PruneMethod.HPSS, + config=config + ) + prune_controller.prune( + file_path=f"{file_name}", + source_endpoint=source, + check_endpoint=None, + days_from_now=days_from_now + ) +# ------------------------------------------------------ +# Test transfer from CFS to HPSS +# ------------------------------------------------------ +if TEST_CFS_TO_HPSS: + from orchestration.flows.bl832.config import Config832 + config = Config832() + project_name = "ALS-11193_nbalsara" + source = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/raw/", + uri="nersc.gov" + ) + destination = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + cfs_to_hpss_flow( + file_path=f"{project_name}", + source=source, + destination=destination, + config=config + ) + +# ------------------------------------------------------ +# Test transfer from HPSS to CFS +# ------------------------------------------------------ +if TEST_HPSS_TO_CFS: + from orchestration.flows.bl832.config import Config832 + config = Config832() + relative_file_path = f"{config.beamline_id}/raw/ALS-11193_nbalsara/ALS-11193_nbalsara_2022-2.tar" + source = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], # root_path: /home/a/alsdev/data_mover + uri=config.hpss_alsdev["uri"] + ) + destination = FileSystemEndpoint( + name="CFS", + root_path="/global/cfs/cdirs/als/data_mover/8.3.2/retrieved_from_tape", + uri="nersc.gov" + ) + + files_to_extract = [ + "20221109_012020_MSB_Book1_Proj33_Cell5_2pFEC_LiR2_6C_Rest3.h5", + "20221012_172023_DTH_100722_LiT_r01_cell3_10x_0_19_CP2.h5", + ] + + hpss_to_cfs_flow( + file_path=f"{relative_file_path}", + source=source, + destination=destination, + files_to_extract=files_to_extract, + config=config + ) + +# ------------------------------------------------------ +# Test listing HPSS files +# ------------------------------------------------------ +if TEST_HPSS_LS: + from orchestration.flows.bl832.config import Config832 + + # Build client, config, endpoint + config = Config832() + endpoint = HPSSEndpoint( + name="HPSS", + root_path=config.hpss_alsdev["root_path"], + uri=config.hpss_alsdev["uri"] + ) + + # Instantiate controller + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.CFS_TO_HPSS, + config=config + ) + + # Directory listing + project_path = f"{config.beamline_id}/raw/BLS-00564_dyparkinson" + logger.info("Controller-based directory listing on HPSS:") + output_file = transfer_controller.list_hpss( + endpoint=endpoint, + remote_path=project_path, + recursive=True + ) + + # TAR archive listing + archive_name = project_path.split("/")[-1] + tar_path = f"{project_path}/{archive_name}_2023-1.tar" + logger.info("Controller-based tar archive listing on HPSS:") + output_file = transfer_controller.list_hpss( + endpoint=endpoint, + remote_path=tar_path, + recursive=False + ) From 05712adb53f88aa3c222988b7cddd08a96cd5896 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 15:11:48 -0700 Subject: [PATCH 104/128] Adding a note about encode_thumbnail being part of scicat --- orchestration/flows/scicat/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index 7dcf0ebb..b4d0bc30 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -133,7 +133,12 @@ def encode_image_2_thumbnail( filebuffer, imType="jpg" ) -> str: - """Encode an image file to a base 64 string for use as a thumbnail.""" + """Encode an image file to a base 64 string for use as a thumbnail. + + "encode_thumbnail()" is now part of SciCat. + Not sure if this would conflict with the current production version we have deployed. + https://www.scicatproject.org/pyscicat/howto/ingest.html?highlight=encode + """ logging.info("Creating thumbnail for dataset") header = "data:image/{imType};base64,".format(imType=imType) dataBytes = base64.b64encode(filebuffer.read()) From ed35aeb96bfc251d480023f023cb0bb252603254 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 15:14:25 -0700 Subject: [PATCH 105/128] capitalizing severity enum options --- orchestration/flows/bl832/scicat_ingestor.py | 2 +- orchestration/flows/scicat/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index cc7c6f1a..573b165e 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -554,7 +554,7 @@ def _extract_fields( dataset = file.get(md_key) if not dataset: issues.append( - Issue(msg=f"dataset not found {md_key}", severity=Severity.warning) + Issue(msg=f"dataset not found {md_key}", severity=Severity.WARNING) ) continue metadata[md_key] = self._get_dataset_value(file[md_key]) diff --git a/orchestration/flows/scicat/utils.py b/orchestration/flows/scicat/utils.py index b4d0bc30..205675bd 100644 --- a/orchestration/flows/scicat/utils.py +++ b/orchestration/flows/scicat/utils.py @@ -23,8 +23,8 @@ class Severity( Enum ): """Enum for issue severity.""" - warning = "warning" - error = "error" + WARNING = "warning" + ERROR = "error" @dataclass From 30aa7c0685e54cefc5e77fa0de714a2bd4e12336 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:16:42 -0700 Subject: [PATCH 106/128] Adding comment about cput not overwriting --- orchestration/slurm/cfs_to_hpss.slurm | 1 + 1 file changed, 1 insertion(+) diff --git a/orchestration/slurm/cfs_to_hpss.slurm b/orchestration/slurm/cfs_to_hpss.slurm index ef210aa4..2726a144 100644 --- a/orchestration/slurm/cfs_to_hpss.slurm +++ b/orchestration/slurm/cfs_to_hpss.slurm @@ -115,6 +115,7 @@ if [ -f "$SOURCE_PATH" ]; then echo "[LOG] Single file detected. Transferring via hsi cput." FILE_NAME=$(basename "$SOURCE_PATH") echo "[LOG] File name: $FILE_NAME" + # Note about hsi cput: If the file already exists on HPSS, hsi cput will skip the transfer. hsi cput "$SOURCE_PATH" "$DEST_PATH/$FILE_NAME" echo "[LOG] (Simulated) File transfer completed for $FILE_NAME." elif [ -d "$SOURCE_PATH" ]; then From 80a95bc434b2cd843b4b16f6b86772674ce9a594 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:17:09 -0700 Subject: [PATCH 107/128] Deleting unused sfapi key variable --- orchestration/flows/bl832/move_refactor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index 7a57cf87..db75d0ee 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -18,8 +18,6 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -API_KEY = os.getenv("API_KEY") - @flow(name="new_832_file_flow") def process_new_832_file( From 08a1dcc6cadd326651034bd2b15bbb5cfe4161f1 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:25:06 -0700 Subject: [PATCH 108/128] Docstring --- orchestration/flows/bl832/move_refactor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index db75d0ee..c28fc3e1 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -110,6 +110,15 @@ def process_new_832_file( @flow(name="test_832_transfers") def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): + """Test transfers between spot832, data832, and NERSC. + Note that the file must already exist on spot832. + Uses Globus transfer. + This flow is scheduled to run periodically to verify that transfers are working. + + :param file_path: path to file on spot832 + :return: None + + """ config = Config832() logger.info(f"{str(uuid.uuid4())}{file_path}") # copy file to a uniquely-named file in the same folder From 6ebe83931c378990b79e0fb5a44300767808175e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:29:43 -0700 Subject: [PATCH 109/128] Docstring --- orchestration/flows/bl832/scicat_ingestor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 573b165e..3344955d 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -38,7 +38,7 @@ class TomographyIngestorController(BeamlineIngestorController): """ - Ingestor for Tomo832 beamline. + Ingestor for 8.3.2 Microtomography beamline. """ DEFAULT_USER = "8.3.2" # In case there's not proposal number INGEST_SPEC = "als832_dx_3" # Where is this spec defined? From 21e8d8237b6c5daca8c93dc3e6a7d653ce4e3bff Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:30:15 -0700 Subject: [PATCH 110/128] renaming login_to_scicat to get_scicat_client --- orchestration/flows/bl832/move_refactor.py | 8 ++++---- orchestration/flows/bl832/scicat_ingestor.py | 6 +++--- orchestration/flows/scicat/ingestor_controller.py | 4 ++-- scripts/test_controllers_end_to_end.py | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py index c28fc3e1..a5b49da4 100644 --- a/orchestration/flows/bl832/move_refactor.py +++ b/orchestration/flows/bl832/move_refactor.py @@ -69,8 +69,8 @@ def process_new_832_file( logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") try: ingestor = TomographyIngestorController(config) - # login_to_scicat assumes that the environment variables are set in the environment - ingestor.login_to_scicat() + # get_scicat_client assumes that the environment variables are set in the environment + ingestor.get_scicat_client() ingestor.ingest_new_raw_dataset(file_path) except Exception as e: logger.error(f"SciCat ingest failed with {e}") @@ -157,9 +157,9 @@ def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): config = Config832() file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" ingestor = TomographyIngestorController(config) - # login_to_scicat assumes that the environment variables are set in the environment + # get_scicat_client assumes that the environment variables are set in the environment # in this test, just using the scicatlive backend defaults (admin user) - ingestor.login_to_scicat( + ingestor.get_scicat_client( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="admin", scicat_password="2jf70TPNZsS" diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 3344955d..32f43195 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -699,12 +699,12 @@ def _upload_raw_dataset( proposal_id = "test832" ingestor = TomographyIngestorController(config) - # login_to_scicat assumes that the environment variables are set in the environment + # get_scicat_client assumes that the environment variables are set in the environment # in this test, just using the scicatlive (3.2.5) backend defaults (admin user) logger.info("Setting up metadata SciCat ingestion") - ingestor.login_to_scicat( + ingestor.get_scicat_client( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="admin", scicat_password="2jf70TPNZsS" @@ -753,7 +753,7 @@ def _upload_raw_dataset( ) admin_ingestor = TomographyIngestorController(config) - admin_ingestor.login_to_scicat( + admin_ingestor.get_scicat_client( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="archiveManager", scicat_password="aman" diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 2eb17735..817e0f9e 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -33,7 +33,7 @@ def __init__( self.config = config self.scicat_client = scicat_client - def login_to_scicat( + def get_scicat_client( self, scicat_base_url: Optional[str] = None, scicat_user: Optional[str] = None, @@ -321,7 +321,7 @@ def ingest_new_derived_dataset(self, file_path: str = "", raw_dataset_id: str = if __name__ == "__main__": logger.info("Testing SciCat ingestor controller") test_ingestor = ConcreteBeamlineIngestorController(BeamlineConfig) - test_ingestor.login_to_scicat( + test_ingestor.get_scicat_client( scicat_base_url="http://localhost:3000/api/v3/", scicat_user="ingestor", scicat_password="aman" diff --git a/scripts/test_controllers_end_to_end.py b/scripts/test_controllers_end_to_end.py index d635a823..66ab0cec 100644 --- a/scripts/test_controllers_end_to_end.py +++ b/scripts/test_controllers_end_to_end.py @@ -104,8 +104,8 @@ def ingest_new_raw_dataset( str: The SciCat ID of the created dataset. """ if not self.scicat_client: - logger.error("SciCat client not initialized. Call login_to_scicat first.") - raise ValueError("SciCat client not initialized. Call login_to_scicat first.") + logger.error("SciCat client not initialized. Call get_scicat_client first.") + raise ValueError("SciCat client not initialized. Call get_scicat_client first.") # Create minimal metadata for the dataset from pyscicat.model import CreateDatasetOrigDatablockDto, DataFile, RawDataset @@ -177,8 +177,8 @@ def ingest_new_derived_dataset( str: The SciCat ID of the created dataset. """ if not self.scicat_client: - logger.error("SciCat client not initialized. Call login_to_scicat first.") - raise ValueError("SciCat client not initialized. Call login_to_scicat first.") + logger.error("SciCat client not initialized. Call get_scicat_client first.") + raise ValueError("SciCat client not initialized. Call get_scicat_client first.") # Create minimal metadata for the dataset from pyscicat.model import CreateDatasetOrigDatablockDto, DataFile, DerivedDataset @@ -553,7 +553,7 @@ def test_scicat_ingest( # SCICAT_INGEST_USER="admin" # SCICAT_INGEST_PASSWORD="2jf70TPNZsS" - test_ingestor.login_to_scicat( + test_ingestor.get_scicat_client( scicat_base_url=os.getenv("SCICAT_API_URL"), scicat_user=os.getenv("SCICAT_INGEST_USER"), scicat_password=os.getenv("SCICAT_INGEST_PASSWORD") From 6a4fcf67137c293ea9b133cce318c31aa7f4fd19 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:31:22 -0700 Subject: [PATCH 111/128] removing redundant error message --- orchestration/flows/scicat/ingestor_controller.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index 817e0f9e..ba42166c 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -93,9 +93,6 @@ def get_scicat_client( # logger.info(f"SciCat token: {response.json()['access_token']}") return self.scicat_client - except requests.exceptions.RequestException as e: - logger.error(f"Failed to log in to SciCat: {e}") - raise e except Exception as e: logger.error(f"Failed to log in to SciCat: {e}") raise e From b0a7d1ee37cd1c026212601542bf45a1857fbae2 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:34:53 -0700 Subject: [PATCH 112/128] Docstring --- orchestration/flows/scicat/ingestor_controller.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index ba42166c..b669b6d7 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -187,6 +187,12 @@ def remove_dataset_location( ) -> bool: """ Remove a location from an existing dataset in SciCat. + We might want to do this after data was moved to a new location, + and has been pruned from the previous location. + + :param dataset_id: SciCat ID of the dataset. + :param source_folder_host: The source folder host to identify the location to remove. + :return: True if the location was successfully removed, False otherwise. """ logger.info(f"Removing location with host {source_folder_host} from dataset {dataset_id}") From 354e151b80e4f2e83459f1badb216cd76a0bb678 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:37:13 -0700 Subject: [PATCH 113/128] removing test code --- .../flows/scicat/ingestor_controller.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/orchestration/flows/scicat/ingestor_controller.py b/orchestration/flows/scicat/ingestor_controller.py index b669b6d7..fbaee240 100644 --- a/orchestration/flows/scicat/ingestor_controller.py +++ b/orchestration/flows/scicat/ingestor_controller.py @@ -308,24 +308,3 @@ def _find_dataset( raise ValueError("The dataset returned does not have a valid 'pid' field.") return dataset_id - - -# Concrete implementation for testing and instantiation. -class ConcreteBeamlineIngestorController(BeamlineIngestorController): - def ingest_new_raw_dataset(self, file_path: str = "") -> str: - # Dummy implementation for testing. - return "raw_dataset_id_dummy" - - def ingest_new_derived_dataset(self, file_path: str = "", raw_dataset_id: str = "") -> str: - # Dummy implementation for testing. - return "derived_dataset_id_dummy" - - -if __name__ == "__main__": - logger.info("Testing SciCat ingestor controller") - test_ingestor = ConcreteBeamlineIngestorController(BeamlineConfig) - test_ingestor.get_scicat_client( - scicat_base_url="http://localhost:3000/api/v3/", - scicat_user="ingestor", - scicat_password="aman" - ) From 4a7030f8c2f79fef820bae5f093b830cf3250558 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:44:09 -0700 Subject: [PATCH 114/128] Fixing project path endpoint to nersc cfs endpoint. Updating the log to use that path --- orchestration/flows/bl832/dispatcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 1867519a..873305c2 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -267,14 +267,14 @@ def archive_832_projects_from_previous_cycle_dispatcher( # config.nersc832_alsdev_scratch.path: the SCRATCH directory path. projects = config.tc.operation_ls( endpoint_id=config.nersc832.endpoint_id, - path=config.nersc832_alsdev_scratch.path, + path=config.nersc832_alsdev_raw.path, orderby=["name", "last_modified"], ).get("DATA", []) except Exception as e: logger.error(f"Failed to list projects: {e}") return - logger.info(f"Found {len(projects)} items in the SCRATCH directory.") + logger.info(f"Found {len(projects)} items in the {projects.path} directory.") # Process each project: check its modification time and trigger transfer if within the archive window. for project in projects: From a64745b0d4662c24a0abe5ab5721d99962b1b34d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 15 Oct 2025 16:50:48 -0700 Subject: [PATCH 115/128] adding log paths to error message --- orchestration/flows/bl832/dispatcher.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 873305c2..dbe03c9e 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -305,7 +305,11 @@ def archive_832_projects_from_previous_cycle_dispatcher( ) except Exception as e: - logger.error(f"Error archiving project {project_name}: {e}") + logger.error( + f"Error archiving project {project_name}: {e}. Logs are available on NERSCT at " + f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project_name}_to_hpss_*.log" + f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project_name}_to_hpss_*.err" + ) try: # Ingest the project into SciCat. logger.info("Ingesting new file path into SciCat...") From 7ce99bb682f831760c5ebeafb813c099187b975b Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 21 Oct 2025 14:25:48 -0700 Subject: [PATCH 116/128] Addressing comments regarding hpss dispatcher flows. --- orchestration/flows/bl832/dispatcher.py | 238 ++++++++++++++---------- 1 file changed, 144 insertions(+), 94 deletions(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index dbe03c9e..e47a5b92 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -184,6 +184,7 @@ async def dispatcher( def archive_832_project_dispatcher( config: Config832, file_path: Union[str, List[str]] = None, + scicat_id: Optional[Union[str, List[str]]] = None ) -> None: """ Flow to archive one or more beamline 832 projects to tape. @@ -192,19 +193,28 @@ def archive_832_project_dispatcher( Parameters ---------- - file_path : Union[str, List[str]] - A single file path or a list of file paths to be archived. config : Config832 Configuration object containing endpoint details. + file_path : Union[str, List[str]] + A single file path or a list of file paths to be archived (path on CFS). + scicat_id : Optional[Union[str, List[str]]] + Optional SciCat ID(s) for the project(s). Must be in the same order as file_path(s). """ # Normalize file_path into a list if it's a single string. - if isinstance(file_path, str): - file_paths = [file_path] + logger = get_run_logger() + + # Build pairs with strict 1:1 length check (only if scicat_id provided) + if scicat_id is None: + projects = zip(file_path, [None] * len(file_path)) else: - file_paths = file_path + if len(file_path) != len(scicat_id): + raise ValueError( + f"Length mismatch: file_path({len(file_path)}) != scicat_id({len(scicat_id)})" + ) + projects = zip(file_path, scicat_id) - for fp in file_paths: + for fp, scid in projects: try: run_specific_flow( "cfs_to_hpss_flow/cfs_to_hpss_flow", @@ -220,6 +230,25 @@ def archive_832_project_dispatcher( except Exception as e: logger.error(f"Error scheduling transfer for {fp}: {e}") + # Ingest the project into SciCat if scicat_id is provided. + ingestor = TomographyIngestorController( + config=config, + scicat_client=config.scicat + ) + + if scid: + logger.info("Ingesting new file path into SciCat...") + + try: + ingestor.add_new_dataset_location( + dataset_id=scid, + datafile_path=config.hpss_alsdev.root_path + "/" + fp.split("/")[-1], + source_folder_host="HPSS" + ) + logger.info(f"Updated SciCat dataset {scicat_id} with new location for project: {fp}") + except Exception as e: + logger.error(f"Error updating dataset location for project {fp} into SciCat: {e}") + # --------------------------------------------------------------------------- # Tape Transfer Flow: Process pending projects @@ -231,11 +260,13 @@ def archive_832_projects_from_previous_cycle_dispatcher( config: Config832, ) -> None: """ - Archives the previous cycle's projects from the NERSC / CFS / 8.3.2 / SCRATCH directory. + Archives the previous cycle's projects from the NERSC / CFS / 8.3.2 / RAW directory. The schedule is as follows: - - On January 2: Archive projects with modification dates between January 1 and July 15 (previous year) - - On July 4: Archive projects with modification dates between July 16 and December 31 (previous year) + - On/around January 2 (assuming NERSC is up): + Archive projects with modification dates between January 1 and July 15 (previous year) + - On/around July 4 (assuming NERSC is up): + Archive projects with modification dates between July 16 and December 31 (previous year) The flow lists projects via Globus Transfer's operation_ls, filters them based on modification times, and then calls the cfs_to_hpss_flow for each eligible project. @@ -244,30 +275,32 @@ def archive_832_projects_from_previous_cycle_dispatcher( now = datetime.now() # Validate that today is a scheduled trigger day and set the archive window accordingly. - if now.month == 1 and now.day == 2: - archive_start = datetime(now.year - 1, 1, 1, 0, 0, 0) - archive_end = datetime(now.year - 1, 7, 15, 23, 59, 59) - logger.info(f"Archiving Cycle 1 ({archive_start.strftime('%b %d, %Y %H:%M:%S')} - " - f"{archive_end.strftime('%b %d, %Y %H:%M:%S')})") - elif now.month == 7 and now.day == 4: - archive_start = datetime(now.year - 1, 7, 16, 0, 0, 0) - archive_end = datetime(now.year - 1, 12, 31, 23, 59, 59) - logger.info(f"Archiving Cycle 2 ({archive_start.strftime('%b %d, %Y %H:%M:%S')} - " - f"{archive_end.strftime('%b %d, %Y %H:%M:%S')})") + # ------------------------- + # Compute "last complete cycle" inline (no helpers). + # ------------------------- + if (now.month < 7) or (now.month == 7 and now.day <= 15): + # Before or on Jul 15: most recent completed cycle is previous year's H2. + y = now.year - 1 + label = "Cycle 2" + archive_start = datetime(y, 7, 16, 0, 0, 0) + archive_end = datetime(y, 12, 31, 23, 59, 59) else: - logger.info("Today is not a scheduled day for archiving.") - return + # Jul 16 or later: most recent completed cycle is current year's H1. + y = now.year + label = "Cycle 1" + archive_start = datetime(y, 1, 1, 0, 0, 0) + archive_end = datetime(y, 7, 15, 23, 59, 59) - logger.info(f"Archive window: {archive_start} to {archive_end}") + logger.info(f"Archive window for {label}: {archive_start} to {archive_end}") # List projects using Globus Transfer's operation_ls. try: # config.tc: configured Globus Transfer client. # config.nersc832.endpoint_id: the NERSC endpoint ID. - # config.nersc832_alsdev_scratch.path: the SCRATCH directory path. + # config.nersc832_alsdev_raw.root_path: the NERSC CFS directory path. projects = config.tc.operation_ls( - endpoint_id=config.nersc832.endpoint_id, - path=config.nersc832_alsdev_raw.path, + endpoint_id=config.nersc832.uuid, + path=config.nersc832_alsdev_raw.root_path, orderby=["name", "last_modified"], ).get("DATA", []) except Exception as e: @@ -287,47 +320,67 @@ def archive_832_projects_from_previous_cycle_dispatcher( try: last_mod = isoparse(last_mod_str) except Exception as e: - logger.error(f"Error parsing modification time for project {project_name}: {e}") + logger.warning(f"Error parsing modification time for project {project_name}: {e}") continue if archive_start <= last_mod <= archive_end: logger.info(f"Project {project_name} last modified at {last_mod} is within the archive window.") try: # Call the transfer flow for this project. + # This should be blocking to ensure sequential processing (HPSS has limitations for concurrent transfers). run_specific_flow( "cfs_to_hpss_flow/cfs_to_hpss_flow", { - "project": project, - "source_endpoint": config.nersc832, - "destination_endpoint": config.hpss_alsdev, + "file_path": config.nersc832_alsdev_raw.root_path + "/" + project['name'], + "source": config.nersc832, + "destination": config.hpss_alsdev, "config": config } ) except Exception as e: logger.error( - f"Error archiving project {project_name}: {e}. Logs are available on NERSCT at " + f"Error archiving project {project_name}: {e}. Logs are available on NERSC at " f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project_name}_to_hpss_*.log" f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project_name}_to_hpss_*.err" ) - try: - # Ingest the project into SciCat. - logger.info("Ingesting new file path into SciCat...") - ingestor = TomographyIngestorController( - config=config, - scicat_client=config.scicat - ) - scicat_id = ingestor._find_dataset( - proposal_id="proposal_id", - file_name="file_name" - ) - ingestor.add_new_dataset_location( - dataset_id=scicat_id, - source_folder="source_folder", - source_folder_host="source_folder_host" - ) - except Exception as e: - logger.error(f"Error ingesting project {project_name} into SciCat: {e}") + # Ingest the project into SciCat. + logger.info("Ingesting new file path into SciCat...") + ingestor = TomographyIngestorController( + config=config, + scicat_client=config.scicat + ) + + # Add a loop to get each file name in projects, and upate path in SciCat. + for scan in config.tc.operation_ls( + endpoint_id=config.nersc832.uuid, + path=config.nersc832_alsdev_raw.root_path + "/" + project['name'], + orderby=["name", "last_modified"], + ): + logger.info(f"Found scan: {scan['name']}") + + logger.info("Looking for dataset in SciCat...") + + try: + scicat_id = ingestor._find_dataset( + file_name=scan['name'] + ) + logger.info(f"Found existing dataset in SciCat with ID: {scicat_id}") + except Exception as e: + logger.warning(f"Error finding dataset in SciCat for scan {scan['name']}: {e}") + + logger.info("Updating dataset location in SciCat...") + try: + if scicat_id: + ingestor.add_new_dataset_location( + dataset_id=scicat_id, + datafile_path=config.hpss_alsdev.root_path + "/" + project['name'] + "/" + scan['name'], + source_folder_host="HPSS" + ) + else: + logger.warning(f"Skipping dataset location update for scan {scan['name']} as SciCat ID was not found.") + except Exception as e: + logger.warning(f"Error updating dataset location for project {project} into SciCat: {e}") else: logger.info(f"Project {project_name} last modified at {last_mod} is outside the archive window.") @@ -345,68 +398,65 @@ def archive_all_832_projects_dispatcher( """ logger = get_run_logger() - logger.info(f"Checking for projects at {config.nersc832_alsdev_scratch.path} to archive to tape...") + logger.info(f"Checking for projects at {config.nersc832_alsdev_raw.root_path} to archive to tape...") - # ARCHIVE ALL PROJECTS IN THE NERSC / CFS / 8.3.2 / SCRATCH DIRECTORY + # ARCHIVE ALL PROJECTS IN THE NERSC / CFS / 8.3.2 / RAW DIRECTORY + # Use the Globus SDK transfer controller (config.tc) to list all projects. + # Note this is different from the controller classes in this repo. for project in config.tc.operation_ls( - endpoint_id=config.nersc832.endpoint_id, - path=config.nersc832_alsdev_scratch.path, + endpoint_id=config.nersc832.uuid, + path=config.nersc832_alsdev_raw.root_path, orderby=["name", "last_modified"], ): - logger.info(f"Found project: {project}") + logger.info(f"Found project: {project['name']}") try: + # Call the transfer flow for this project. + # This should be blocking to ensure sequential processing (HPSS has limitations for concurrent transfers). run_specific_flow( "cfs_to_hpss_flow/cfs_to_hpss_flow", { - "file_path": project, + "file_path": config.nersc832_alsdev_raw.root_path + "/" + project['name'], "source": config.nersc832, # NERSC FileSystem Endpoint (not globus) "destination": config.hpss_alsdev, # HPSS Endpoint "config": config } ) + except Exception as e: + logger.error( + f"Error archiving project {project['name']}: {e}. Logs are available on NERSC at " + f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project['name']}_to_hpss_*.log" + f"/global/cfs/cdirs/als/data_mover/hpss_transfer_logs/{config.beamline_id}/{project['name']}_to_hpss_*.err" + ) + ingestor = TomographyIngestorController( + config=config, + scicat_client=config.scicat + ) + + # Update the path for each scan within the project into SciCat. + for scan in config.tc.operation_ls( + endpoint_id=config.nersc832.uuid, + path=config.nersc832_alsdev_raw.root_path + "/" + project['name'], + orderby=["name", "last_modified"], + ): try: - # Ingest the project into SciCat. + logger.info(f"Found scan: {scan['name']}") + logger.info("Ingesting new file path into SciCat...") - ingestor = TomographyIngestorController( - config=config, - scicat_client=config.scicat - ) scicat_id = ingestor._find_dataset( - proposal_id="proposal_id", - file_name="file_name" - ) - ingestor.add_new_dataset_location( - dataset_id=scicat_id, - source_folder="source_folder", - source_folder_host="source_folder_host" + file_name=scan['name'] ) except Exception as e: - logger.error(f"Error ingesting project {project} into SciCat: {e}") - - except Exception as e: - logger.error(e) - - -if __name__ == "__main__": - """ - This script defines the flow for the decision making process of the BL832 beamline. - It first sets up the decision settings, then executes the decision flow to run specific sub-flows as needed. - """ - try: - # Setup decision settings based on input parameters - setup_decision_settings(alcf_recon=True, nersc_recon=True, new_file_832=True) - # Run the main decision flow with the specified parameters - # asyncio.run(dispatcher( - # config={}, # PYTEST, ALCF, NERSC - # is_export_control=False, # ALCF & MOVE - # folder_name="folder", # ALCF - # file_name="file", # ALCF - # file_path="/path/to/file", # MOVE - # send_to_alcf=True, # ALCF - # send_to_nersc=True, # MOVE - # ) - # ) - except Exception as e: - logger = get_run_logger() - logger.error(f"Failed to execute main flow: {e}") + logger.warning(f"Error finding dataset for scan {scan['name']}: {e}") + try: + if scicat_id: + logger.info(f"Found existing dataset in SciCat with ID: {scicat_id}") + ingestor.add_new_dataset_location( + dataset_id=scicat_id, + datafile_path=config.hpss_alsdev.root_path + "/" + project['name'] + "/" + scan['name'], + source_folder_host="HPSS" + ) + else: + logger.warning(f"Skipping dataset location update for scan {scan['name']} as SciCat ID was not found.") + except Exception as e: + logger.error(f"Error updating dataset location for project {project} into SciCat: {e}") From 0faae48bb649441b1f42a83206ae8fbd25358582 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 21 Oct 2025 14:28:49 -0700 Subject: [PATCH 117/128] Adjusting comments to be clearer --- orchestration/flows/bl832/scicat_ingestor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 32f43195..25d9ce7d 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -199,7 +199,7 @@ def ingest_new_raw_dataset( ) # Generate and upload a thumbnail attachment - # The "/exchange/data" key is specific to the Tomo832 HDF5 file structure. + # The "/exchange/data" key is specific to the Microtomography (8.3.2) HDF5 file structure. thumbnail_file = build_thumbnail(file["/exchange/data"][0]) encoded_thumbnail = encode_image_2_thumbnail(thumbnail_file) self._upload_attachment( @@ -220,7 +220,7 @@ def ingest_new_derived_dataset( raw_dataset_id: str = "", ) -> str: """ - Ingest a new derived dataset from the Tomo832 beamline. + Ingest a new derived dataset from the Microtomography (8.3.2) beamline. This method handles ingestion of derived datasets generated during tomography reconstruction: 1. A directory of TIFF slices From 2749500228f55c23b8f2957454f721219e159eb0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 21 Oct 2025 14:54:42 -0700 Subject: [PATCH 118/128] Adding typing throughout --- orchestration/flows/bl832/scicat_ingestor.py | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 25d9ce7d..65bb1791 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -1,4 +1,5 @@ # import importlib +import io import json from logging import getLogger import os @@ -445,7 +446,7 @@ def ingest_new_derived_dataset( return dataset_id - def _generate_zarr_thumbnail(self, zarr_path: Path): + def _generate_zarr_thumbnail(self, zarr_path: Path) -> io.BytesIO | None: """ Generate a thumbnail image from an NGFF Zarr dataset using ngff_zarr. This implementation extracts a mid-slice and returns a BytesIO buffer. @@ -457,7 +458,6 @@ def _generate_zarr_thumbnail(self, zarr_path: Path): import ngff_zarr as nz from PIL import Image import numpy as np - import io # Load the multiscale image from the Zarr store multiscales = nz.from_ngff_zarr(str(zarr_path)) @@ -496,9 +496,9 @@ def _generate_zarr_thumbnail(self, zarr_path: Path): def _calculate_access_controls( self, - username, - beamline, - proposal + username: str, + beamline: str, + proposal: str ) -> Dict: """ Calculate access controls for a dataset. @@ -545,9 +545,9 @@ def _create_data_files( def _extract_fields( self, - file, - keys, - issues + file: h5py.File, + keys: List[str], + issues: List[Issue] ) -> Dict[str, Any]: metadata = {} for md_key in keys: @@ -562,8 +562,8 @@ def _extract_fields( def _get_dataset_value( self, - data_set - ): + data_set: h5py.Dataset + ) -> Any: """ Extracts the value of a dataset from an HDF5 file. """ @@ -589,9 +589,9 @@ def _get_dataset_value( def _get_data_sample( self, - file, - sample_size=10 - ): + file: h5py.File, + sample_size: int = 10 + ) -> Dict[str, Any]: """ Extracts a sample of the data from the HDF5 file. """ data_sample = {} for key in self.DATA_SAMPLE_KEYS: @@ -628,14 +628,14 @@ def _upload_data_block( def _upload_attachment( self, - encoded_thumnbnail: str, + encoded_thumbnail: str, dataset_id: str, ownable: Ownable, - ) -> Attachment: + ) -> None: "Creates a thumbnail png" attachment = Attachment( datasetId=dataset_id, - thumbnail=encoded_thumnbnail, + thumbnail=encoded_thumbnail, caption="raw image", **ownable.dict(), ) From fba4b5de4d4414a22edff6263cf3573f8e59aad0 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 21 Oct 2025 15:07:27 -0700 Subject: [PATCH 119/128] Docstrings --- orchestration/flows/bl832/scicat_ingestor.py | 46 +++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/orchestration/flows/bl832/scicat_ingestor.py b/orchestration/flows/bl832/scicat_ingestor.py index 65bb1791..b824b453 100644 --- a/orchestration/flows/bl832/scicat_ingestor.py +++ b/orchestration/flows/bl832/scicat_ingestor.py @@ -1,4 +1,3 @@ -# import importlib import io import json from logging import getLogger @@ -39,7 +38,8 @@ class TomographyIngestorController(BeamlineIngestorController): """ - Ingestor for 8.3.2 Microtomography beamline. + Ingestor for 8.3.2 Microtomography beamline. Handles ingestion of raw and derived datasets. + Extends the BeamlineIngestorController with beamline-specific (8.3.2) metadata extraction """ DEFAULT_USER = "8.3.2" # In case there's not proposal number INGEST_SPEC = "als832_dx_3" # Where is this spec defined? @@ -123,6 +123,9 @@ def __init__( config: Config832, scicat_client: Optional[ScicatClient] = None ) -> None: + """Initializes the TomographyIngestorController with beamline-specific settings. + :param config: Configuration object (Config832) for the 8.3.2 beamline. + :param scicat_client: An optional SciCat client instance. If not provided, it will be created.""" super().__init__(config, scicat_client) def ingest_new_raw_dataset( @@ -502,6 +505,11 @@ def _calculate_access_controls( ) -> Dict: """ Calculate access controls for a dataset. + + :param username: Username of the dataset owner. + :param beamline: Beamline name. + :param proposal: Proposal number. + :return: Dictionary with 'owner_group' and 'access_groups'. """ # make an access group list that includes the name of the proposal and the name of the beamline @@ -531,7 +539,11 @@ def _create_data_files( storage_path: str ) -> List[DataFile]: """ - Collects all fits files + Builds a list of DataFile objects for SciCat from a single file. + + :param file_path: Path to the file. + :param storage_path: Path where the file is stored. + :return: List of DataFile objects. """ datafiles = [] datafile = DataFile( @@ -566,6 +578,9 @@ def _get_dataset_value( ) -> Any: """ Extracts the value of a dataset from an HDF5 file. + + :param data_set: HDF5 dataset object. + :return: The value of the dataset, or None if extraction fails. """ logger.debug(f"{data_set} {data_set.dtype}") try: @@ -592,7 +607,13 @@ def _get_data_sample( file: h5py.File, sample_size: int = 10 ) -> Dict[str, Any]: - """ Extracts a sample of the data from the HDF5 file. """ + """ + Extracts a sample of the data from the HDF5 file. + + :param file: HDF5 file object. + :param sample_size: Number of samples to extract. + :return: Dictionary of sampled data arrays. + """ data_sample = {} for key in self.DATA_SAMPLE_KEYS: data_array = file.get(key) @@ -614,7 +635,13 @@ def _upload_data_block( source_root_path: str ) -> Datablock: """ - Creates a datablock of files + Creates a datablock of files associated with a dataset and uploads it to SciCat. + + :param file_path: Path to the file to ingest. + :param dataset_id: SciCat ID of the dataset. + :param storage_root_path: Root path where files are stored. + :param source_root_path: Root path of the source files. + :return: Uploaded Datablock object. """ # calculate the path where the file will as known to SciCat storage_path = str(file_path).replace(source_root_path, storage_root_path) @@ -632,7 +659,14 @@ def _upload_attachment( dataset_id: str, ownable: Ownable, ) -> None: - "Creates a thumbnail png" + """ + Creates a thumbnail png attachment and uploads it to SciCat. + + :param encoded_thumbnail: Base64 encoded thumbnail image. + :param dataset_id: SciCat ID of the dataset. + :param ownable: Ownable object. + :return: None + """ attachment = Attachment( datasetId=dataset_id, thumbnail=encoded_thumbnail, From 2c548f2b389e71384e546eb1b00bc8fe2c7ec448 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 16:37:24 -0700 Subject: [PATCH 120/128] replacing old move.py with move_refactor.py --- orchestration/flows/bl832/move.py | 260 ++++++++------------- orchestration/flows/bl832/move_refactor.py | 171 -------------- 2 files changed, 98 insertions(+), 333 deletions(-) delete mode 100644 orchestration/flows/bl832/move_refactor.py diff --git a/orchestration/flows/bl832/move.py b/orchestration/flows/bl832/move.py index b547a5c7..a5b49da4 100644 --- a/orchestration/flows/bl832/move.py +++ b/orchestration/flows/bl832/move.py @@ -1,114 +1,30 @@ import datetime +import logging import os from pathlib import Path import uuid -from globus_sdk import TransferClient -from prefect import flow, task, get_run_logger +from prefect import flow, task from prefect.blocks.system import JSON -from orchestration.flows.scicat.ingest import ingest_dataset +# from orchestration.flows.scicat.ingest import ingest_dataset +from orchestration.flows.bl832.scicat_ingestor import TomographyIngestorController from orchestration.flows.bl832.config import Config832 -from orchestration.globus.transfer import GlobusEndpoint, start_transfer -from orchestration.prefect import schedule_prefect_flow -from orchestration.prometheus_utils import PrometheusMetrics +from orchestration.globus.transfer import start_transfer +from orchestration.prune_controller import get_prune_controller, PruneMethod +from orchestration.transfer_controller import get_transfer_controller, CopyMethod -API_KEY = os.getenv("API_KEY") -TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832" - - -@task(name="transfer_spot_to_data") -def transfer_spot_to_data( - file_path: str, - transfer_client: TransferClient, - spot832: GlobusEndpoint, - data832: GlobusEndpoint, -): - logger = get_run_logger() - - # if source_file begins with "/", it will mess up os.path.join - if file_path[0] == "/": - file_path = file_path[1:] - - source_path = os.path.join(spot832.root_path, file_path) - dest_path = os.path.join(data832.root_path, file_path) - success, _ = start_transfer( - transfer_client, - spot832, - source_path, - data832, - dest_path, - max_wait_seconds=600, - logger=logger, - ) - logger.info(f"spot832 to data832 globus task_id: {task}") - return success - - -@task(name="transfer_data_to_nersc") -def transfer_data_to_nersc( - file_path: str, - transfer_client: TransferClient, - data832: GlobusEndpoint, - nersc832: GlobusEndpoint, -): - logger = get_run_logger() - - # if source_file begins with "/", it will mess up os.path.join - if file_path[0] == "/": - file_path = file_path[1:] - - # Initialize config - config = Config832() - - # Import here to avoid circular imports - from orchestration.transfer_controller import get_transfer_controller, CopyMethod - - # Change prometheus_metrics=None if do not want to push metrics - # prometheus_metrics = None - prometheus_metrics = PrometheusMetrics() - # Get a Globus transfer controller - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.GLOBUS, - config=config, - prometheus_metrics=prometheus_metrics - ) - - # Use transfer controller to copy the file - # The controller automatically handles metrics collection and pushing - logger.info(f"Transferring {file_path} from data832 to nersc") - success = transfer_controller.copy( - file_path=file_path, - source=data832, - destination=nersc832 - ) - - return success +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) @flow(name="new_832_file_flow") -def process_new_832_file_flow( +def process_new_832_file( file_path: str, - is_export_control=False, send_to_nersc=True, - config=None -): - process_new_832_file_task( - file_path=file_path, - is_export_control=is_export_control, - send_to_nersc=send_to_nersc, - config=config - ) - - -@task(name="new_832_file_task") -def process_new_832_file_task( - file_path: str, - is_export_control=False, - send_to_nersc=True, - config=None -): + config: Config832 = None +) -> None: """ Sends a file along a path: - Copy from spot832 to data832 @@ -117,17 +33,11 @@ def process_new_832_file_task( - Schedule a job to delete from spot832 in the future - Schedule a job to delete from data832 in the future - The is_export_control and send_to_nersc flags are functionally identical, but - they are separate options at the beamlines, so we leave them as separate parameters - in case the desired behavior changes in the future. - :param file_path: path to file on spot832 - :param is_export_control: if True, do not send to NERSC ingest into SciCat :param send_to_nersc: if True, send to NERSC and ingest into SciCat """ - logger = get_run_logger() - logger.info("starting flow") + logger.info("Starting New 832 File Flow") if not config: config = Config832() @@ -136,100 +46,126 @@ def process_new_832_file_task( # to all 3 systems. logger.info(f"Transferring {file_path} from spot to data") relative_path = file_path.split("/global")[1] - transfer_spot_to_data(relative_path, config.tc, config.spot832, config.data832) - logger.info(f"Transferring {file_path} to spot to data") + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.GLOBUS, + config=config + ) + + data832_transfer_success = transfer_controller.copy( + file_path=relative_path, + source=config.spot832, + destination=config.data832, + ) - if not is_export_control and send_to_nersc: - transfer_data_to_nersc( - relative_path, config.tc, config.data832, config.nersc832 + if send_to_nersc and data832_transfer_success: + nersc_transfer_success = transfer_controller.copy( + file_path=relative_path, + source=config.data832, + destination=config.nersc832 ) - logger.info( - f"File successfully transferred from data832 to NERSC {file_path}. Task {task}" - ) - flow_name = f"ingest scicat: {Path(file_path).name}" - logger.info(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}") - try: - ingest_dataset(file_path, TOMO_INGESTOR_MODULE) - except Exception as e: - logger.error(f"SciCat ingest failed with {e}") - - # schedule_prefect_flow( - # "ingest_scicat/ingest_scicat", - # flow_name, - # {"relative_path": relative_path}, - # datetime.timedelta(0.0), - # ) + + if nersc_transfer_success: + logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") + try: + ingestor = TomographyIngestorController(config) + # get_scicat_client assumes that the environment variables are set in the environment + ingestor.get_scicat_client() + ingestor.ingest_new_raw_dataset(file_path) + except Exception as e: + logger.error(f"SciCat ingest failed with {e}") bl832_settings = JSON.load("bl832-settings").value - flow_name = f"delete spot832: {Path(file_path).name}" schedule_spot832_delete_days = bl832_settings["delete_spot832_files_after_days"] schedule_data832_delete_days = bl832_settings["delete_data832_files_after_days"] - schedule_prefect_flow( - "prune_spot832/prune_spot832", - flow_name, - { - "relative_path": relative_path, - "source_endpoint": config.spot832, - "check_endpoint": config.data832, - }, - - datetime.timedelta(days=schedule_spot832_delete_days), + + prune_controller = get_prune_controller( + prune_type=PruneMethod.GLOBUS, + config=config + ) + + prune_controller.prune( + file_path=relative_path, + source_endpoint=config.spot832, + check_endpoint=config.data832, + days_from_now=schedule_spot832_delete_days ) logger.info( f"Scheduled delete from spot832 at {datetime.timedelta(days=schedule_spot832_delete_days)}" ) - flow_name = f"delete data832: {Path(file_path).name}" - schedule_prefect_flow( - "prune_data832/prune_data832", - flow_name, - { - "relative_path": relative_path, - "source_endpoint": config.data832, - "check_endpoint": config.nersc832, - }, - datetime.timedelta(days=schedule_data832_delete_days), + prune_controller.prune( + file_path=relative_path, + source_endpoint=config.data832, + check_endpoint=config.nersc832, + days_from_now=schedule_data832_delete_days ) logger.info( f"Scheduled delete from data832 at {datetime.timedelta(days=schedule_data832_delete_days)}" ) + return @flow(name="test_832_transfers") def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): - logger = get_run_logger() + """Test transfers between spot832, data832, and NERSC. + Note that the file must already exist on spot832. + Uses Globus transfer. + This flow is scheduled to run periodically to verify that transfers are working. + + :param file_path: path to file on spot832 + :return: None + + """ config = Config832() - # test_scicat(config) logger.info(f"{str(uuid.uuid4())}{file_path}") # copy file to a uniquely-named file in the same folder file = Path(file_path) new_file = str(file.with_name(f"test_{str(uuid.uuid4())}.txt")) logger.info(new_file) - success, _ = start_transfer( + + success = start_transfer( config.tc, config.spot832, file_path, config.spot832, new_file, logger=logger ) + logger.info(success) - spot832_path = transfer_spot_to_data( - new_file, config.tc, config.spot832, config.data832 + + transfer_controller = get_transfer_controller( + transfer_type=CopyMethod.GLOBUS, + config=config ) - logger.info(f"Transferred {spot832_path} to spot to data") - task = transfer_data_to_nersc(new_file, config.tc, config.data832, config.nersc832) - logger.info( - f"File successfully transferred from data832 to NERSC {spot832_path}. Task {task}" + dat832_success = transfer_controller.copy( + file_path=new_file, + source=config.spot832, + destination=config.data832, ) + logger.info(f"Transferred {new_file} from spot to data. Success: {dat832_success}") + nersc_success = transfer_controller.copy( + file_path=new_file, + source=config.data832, + destination=config.nersc832, + ) + logger.info(f"File successfully transferred from data832 to NERSC {new_file}. Success: {nersc_success}") + pass -@flow(name="test_832_transfers_grafana") -def test_transfers_832_grafana(file_path: str = "/raw/transfer_tests/test/"): - logger = get_run_logger() - config = Config832() - task = transfer_data_to_nersc(file_path, config.tc, config.data832, config.nersc_alsdev) +if __name__ == "__main__": + config = Config832() + file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" + ingestor = TomographyIngestorController(config) + # get_scicat_client assumes that the environment variables are set in the environment + # in this test, just using the scicatlive backend defaults (admin user) + ingestor.get_scicat_client( + scicat_base_url="http://localhost:3000/api/v3/", + scicat_user="admin", + scicat_password="2jf70TPNZsS" + ) + # INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set + os.environ["INGEST_STORAGE_ROOT_PATH"] = "/global/cfs/cdirs/als/data_mover/8.3.2" + os.environ["INGEST_SOURCE_ROOT_PATH"] = "/data832-raw" - logger.info( - f"File successfully transferred from data832 to NERSC {file_path}. Task {task}" - ) \ No newline at end of file + ingestor.ingest_new_raw_dataset(file_path) diff --git a/orchestration/flows/bl832/move_refactor.py b/orchestration/flows/bl832/move_refactor.py deleted file mode 100644 index a5b49da4..00000000 --- a/orchestration/flows/bl832/move_refactor.py +++ /dev/null @@ -1,171 +0,0 @@ -import datetime -import logging -import os -from pathlib import Path -import uuid - -from prefect import flow, task -from prefect.blocks.system import JSON - -# from orchestration.flows.scicat.ingest import ingest_dataset -from orchestration.flows.bl832.scicat_ingestor import TomographyIngestorController -from orchestration.flows.bl832.config import Config832 -from orchestration.globus.transfer import start_transfer -from orchestration.prune_controller import get_prune_controller, PruneMethod -from orchestration.transfer_controller import get_transfer_controller, CopyMethod - - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -@flow(name="new_832_file_flow") -def process_new_832_file( - file_path: str, - send_to_nersc=True, - config: Config832 = None -) -> None: - """ - Sends a file along a path: - - Copy from spot832 to data832 - - Copy from data832 to NERSC - - Ingest into SciCat - - Schedule a job to delete from spot832 in the future - - Schedule a job to delete from data832 in the future - - :param file_path: path to file on spot832 - :param send_to_nersc: if True, send to NERSC and ingest into SciCat - """ - - logger.info("Starting New 832 File Flow") - if not config: - config = Config832() - - # paths come in from the app on spot832 as /global/raw/... - # remove 'global' so that all paths start with 'raw', which is common - # to all 3 systems. - logger.info(f"Transferring {file_path} from spot to data") - relative_path = file_path.split("/global")[1] - - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.GLOBUS, - config=config - ) - - data832_transfer_success = transfer_controller.copy( - file_path=relative_path, - source=config.spot832, - destination=config.data832, - ) - - if send_to_nersc and data832_transfer_success: - nersc_transfer_success = transfer_controller.copy( - file_path=relative_path, - source=config.data832, - destination=config.nersc832 - ) - - if nersc_transfer_success: - logger.info(f"File successfully transferred from data832 to NERSC {file_path}. Task {task}") - try: - ingestor = TomographyIngestorController(config) - # get_scicat_client assumes that the environment variables are set in the environment - ingestor.get_scicat_client() - ingestor.ingest_new_raw_dataset(file_path) - except Exception as e: - logger.error(f"SciCat ingest failed with {e}") - - bl832_settings = JSON.load("bl832-settings").value - - schedule_spot832_delete_days = bl832_settings["delete_spot832_files_after_days"] - schedule_data832_delete_days = bl832_settings["delete_data832_files_after_days"] - - prune_controller = get_prune_controller( - prune_type=PruneMethod.GLOBUS, - config=config - ) - - prune_controller.prune( - file_path=relative_path, - source_endpoint=config.spot832, - check_endpoint=config.data832, - days_from_now=schedule_spot832_delete_days - ) - logger.info( - f"Scheduled delete from spot832 at {datetime.timedelta(days=schedule_spot832_delete_days)}" - ) - - prune_controller.prune( - file_path=relative_path, - source_endpoint=config.data832, - check_endpoint=config.nersc832, - days_from_now=schedule_data832_delete_days - ) - logger.info( - f"Scheduled delete from data832 at {datetime.timedelta(days=schedule_data832_delete_days)}" - ) - - return - - -@flow(name="test_832_transfers") -def test_transfers_832(file_path: str = "/raw/transfer_tests/test.txt"): - """Test transfers between spot832, data832, and NERSC. - Note that the file must already exist on spot832. - Uses Globus transfer. - This flow is scheduled to run periodically to verify that transfers are working. - - :param file_path: path to file on spot832 - :return: None - - """ - config = Config832() - logger.info(f"{str(uuid.uuid4())}{file_path}") - # copy file to a uniquely-named file in the same folder - file = Path(file_path) - new_file = str(file.with_name(f"test_{str(uuid.uuid4())}.txt")) - logger.info(new_file) - - success = start_transfer( - config.tc, config.spot832, file_path, config.spot832, new_file, logger=logger - ) - - logger.info(success) - - transfer_controller = get_transfer_controller( - transfer_type=CopyMethod.GLOBUS, - config=config - ) - - dat832_success = transfer_controller.copy( - file_path=new_file, - source=config.spot832, - destination=config.data832, - ) - logger.info(f"Transferred {new_file} from spot to data. Success: {dat832_success}") - - nersc_success = transfer_controller.copy( - file_path=new_file, - source=config.data832, - destination=config.nersc832, - ) - logger.info(f"File successfully transferred from data832 to NERSC {new_file}. Success: {nersc_success}") - pass - - -if __name__ == "__main__": - config = Config832() - file_path = "/Users/david/Documents/data/tomo/raw/20241216_153047_ddd.h5" - ingestor = TomographyIngestorController(config) - # get_scicat_client assumes that the environment variables are set in the environment - # in this test, just using the scicatlive backend defaults (admin user) - ingestor.get_scicat_client( - scicat_base_url="http://localhost:3000/api/v3/", - scicat_user="admin", - scicat_password="2jf70TPNZsS" - ) - # INGEST_STORAGE_ROOT_PATH and INGEST_SOURCE_ROOT_PATH must be set - os.environ["INGEST_STORAGE_ROOT_PATH"] = "/global/cfs/cdirs/als/data_mover/8.3.2" - os.environ["INGEST_SOURCE_ROOT_PATH"] = "/data832-raw" - - ingestor.ingest_new_raw_dataset(file_path) From d7c353b2c09fa79720505aa52b30a77cc2d6fddf Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 16:41:29 -0700 Subject: [PATCH 121/128] Documentation --- docs/mkdocs/docs/hpss.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs/docs/hpss.md b/docs/mkdocs/docs/hpss.md index df180b40..b0603b75 100644 --- a/docs/mkdocs/docs/hpss.md +++ b/docs/mkdocs/docs/hpss.md @@ -30,8 +30,8 @@ It is important to clarify who users are when we talk about transferring to tape - Perform transfers that bundle data in a way that is optimized for tape storage and retrieval. - Apply it across different beamlines. -**Service Users** - - We use use a service account at NERSC for automating our transfers. This "service" user can perform the same sets of tasks as other NERSC users, but has wider access to data systems. ALS Users benefit from, but do not directly interact with this account. +**Collaboration Account** + - We use use a collaboration account at NERSC for automating our transfers. This "service" user can perform the same sets of tasks as other NERSC users, but has wider access to data systems. ALS Users benefit from, but do not directly interact with this account. For more information, see the NERSC [documentation](https://docs.nersc.gov/accounts/collaboration_accounts/). In `orchestration/transfer_controller.py` we have included two transfer classes for moving data from CFS to HPSS and vice versa (HPSS to CFS). We are following the [HPSS best practices](https://docs.nersc.gov/filesystems/HPSS-best-practices/) outlined in the NERSC documentation. From 0a4e74b69319f53937c0707ad46d285783b913df Mon Sep 17 00:00:00 2001 From: David Abramov Date: Wed, 22 Oct 2025 17:02:33 -0700 Subject: [PATCH 122/128] Adding screenshots to SFAPI documentation --- docs/mkdocs/docs/assets/images/sfapi_step1.png | Bin 0 -> 166931 bytes docs/mkdocs/docs/assets/images/sfapi_step2.png | Bin 0 -> 104296 bytes docs/mkdocs/docs/assets/images/sfapi_step3.png | Bin 0 -> 279194 bytes docs/mkdocs/docs/assets/images/sfapi_step4.png | Bin 0 -> 114143 bytes docs/mkdocs/docs/nersc832.md | 15 ++++++++++++--- 5 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 docs/mkdocs/docs/assets/images/sfapi_step1.png create mode 100644 docs/mkdocs/docs/assets/images/sfapi_step2.png create mode 100644 docs/mkdocs/docs/assets/images/sfapi_step3.png create mode 100644 docs/mkdocs/docs/assets/images/sfapi_step4.png diff --git a/docs/mkdocs/docs/assets/images/sfapi_step1.png b/docs/mkdocs/docs/assets/images/sfapi_step1.png new file mode 100644 index 0000000000000000000000000000000000000000..7c1c3d7f7fc9fd62ca1c1ad63cf25719f4ddc296 GIT binary patch literal 166931 zcmZ^J1z4Or(>AU}ine%*ySo>6hvHD&DegszLveR^hsCwHE$;5_w&>6CzTbZiJbP_6 z$z(E3O@*8u{KqVAWUx7>n6+4>0P=v5r-GKQFxW;7;0HPe-_!O-9+tchl{GY3|Is zu-?+^1BUMzpuMHd06bLv2~m-W-5p3^EDT^!nTRZ|k2LSeKYn~e#H9a1cWMo8I;B&O zKXQHA{-QRH0atSeCh!)n0IvEj364PbxGrNtk?{Xq}%Q--Tj!R-q3AFQk(7z=8bAb!-Nzb5I465RJt=^Gk{FSjN!8 za+Mx}vBj#zOA*YSJRn1D5x3GodMrB?&}BE{IeIE_a0+Xu`u=!U45JrSyVl#hg?4$I zEpByo`Du6K8RsX-hT^*UjYK__)0TlJ#JtpjRCV~`R`~DF;4zF<#V+cAisE=0;!l|L z!*GhA8oO*2i<&4z&PKLRjsa#ZkwNe<`b2OH0t{!c3PFxA;1g9Q4EoPFXzq4S>oxoykp0cP#HxGr@j^BKO~ zJ5T{EFCaT7OaVZFHHd=wL-JEz_EzC`;Ya27#W>U6<|2++UBFSTZ8aV*pb* z2L>*z5ixpUda(*2wM>gKnLXJnM+bU0xF<}Z-aP|L27-5VWZ=YITVD_ujS@W*4H9b- zQxd&~xoNPZ`ZV;GD?8_eEpeRSo$#Hw0dE%rjyrGk1HW>l(v8CG`Xlz8+s-$9T4GxQ zUou-_)#5*a)IqZO?%WN#5p$;K!PoXiH;~`IJ>Xji;RkuR9VnDf9D)L$rRN0Zq)bV* zK3t)T1|(n?b#p;erT0UAcD(F0tFOl7n9dtp+Q}R3-(PwV0Lc(~Q%0phrwJ zIB4@#z2gN+1S}$E-k0%Fg_uiT=Ufz(6uB3f6g{!fjQIBE<&B4!!I~}Cez2moGCAfr z7Fmc}Ag_yaz(|wWonYR>-3#2W-^<8mk3ul7rP_h zK9d8a!=xkoKHDMRl+dC2p~%7Z!SNV(2|Gs|+q6B^H}mnSpGv3uKksMm74D_3ZufL& z2-z4|;Fu!cTQYMPFqn$78{+1_kH<-8!ehwBamAGglMI)|^nnkqDgz3dttmz<=3@a_`F#|pmGgFEgV5)TGD4nx2)rgL5)qKV1 zIo+%?%%ohOjX6Vo@3SrK1Dx~ z9=&x$z<+ialtZxbl%R=-ty4O{dmtw-wEV2;zVDNA|sdFX680~RdGXi z^Zv&D_>|j}^E<~DcP&?oQ?1h$w<_0)?MH5Ct}mGgn2TYSj;ZlJ^ z*lO%*jTU}~-YH3%)tYgOR#@Sfof>^yr$4^lI_j=_bv@+-mWToR^ds7NtavAR4|y_1 zDic4XRym7iBm`pvE$R~{Qr6RsT8A|E4UT?Z7TvfGvl$`IJ}BQhah(;BLan@~`)aMgai`kLsN$W!V_#=~s5vy+5koCJ&>HAjYBUpgCEM;laS3n|F6+h{{D zI3xGd1r;ramC3@x)!Eh4{wmW3GY<24(m+yQp2mcO*)GIS=uoVyeoI4trtMD={+d|G z147#m)tRM@>a_DLh73)mmCBEqdn4zgMZ=&-+yr;SsGX1d*+Zda*cIB3Ec6VgyQF)h zkEHlg;R)p$3(a9QmbZ4Mu7ejpF!QMqRIe)DXsFkOG*Ed?+kkFBq#zr$z6xtv28z$> zW4A(MDA)c6{uf<~7Cf^@k8huHw)#%$WG(8;o-{}(nm$|VYNn;l^8@#Bz zh_^CcWV3$I`cU4WzOOm|IZw9^aPZ{Rd1Jvn$BpBJb40TOGD|R=OPO^KrorseGu`r%yb z1|WEtKAC+Y50g_)pWDskUIm~ZgRIO@xanBqtEIafZOAe{rmXz@_yUA)_ zHqLl0zuY(CF5xCvk2cNQQ{T*_E!taCpTeE+G^c#Moo-EaC%#qV>)@eq6Lk}O z7sJXkxe@*n9gQ}gd+djbAt897r;4qLwpN{6(c6*d`cWJ10OA0`)*%n}i?%(#lfdMN z(g^L04Srtyfv2zst>>Ntsl(Z^3Y0b@uLrWt0at?@Fa!k-R!r~N2XG>C2!U1vu&+0F zJiKIMckCVSd|tXYnQs#tAjUdih+2@qUgE%}KE?-sM$672R(ONkAuCIjmgw2HbEUv2 z+=gyP43;XlXXRdn_9)bAa(8<3B!{T1{$l(BO+^obx5Ij4@`_%+7=4y7mXQIYdM(3& zL4c!yLB5v2U;n`Z&A_03FN1-Ry?(y}-sk`@*w=5g*N=QA#9ysY6Pb{Il_7tl1`|{g zl8|`)Rx+?NGP1TewQ<ljTN1~q0JW~Iu|S3-$=lCT)19~ zRz?o`L@rjA*7jU3ydQt>!S!1Ht(yKL(eGUxEOI1MkO=zcKpz?~i#JxtRUK$=d!;v0eqF z|E-0dk&c1>-^d)yjQ@k|x0XN1ewXVHc09ig#wBm&Vq~cV+7*D!;D-*$gj~old*cTCj##E4Z<+)aW<`ec*J)#g-0EdPGHO2%rlG3%~9^G0nGMsb?T3%WnPDe|28qW)k4}i&T z`Lxhb77!@_2K8Uh1ibLe}7azBDiCwJY-W z82B$e&)$+on%d;JNTDleQj>nEirAxBWBX_FNA3^`^>y~E%@8QR%I}seP^u40n#>s# z0QJjYm5Wf>%~s$rzjnQmf~dl0ja^eQ75`NZUZIZWx*C(=PK1A$d3QVu2&iR$2mnHgs=h5d{2b-)DP10~7j@wMrJ4?c zkG5TuePh9o(`h99{A(tGCy~By{%Qq-JHmxf(1|1qowzZz;}>XJn@E3W*&pen+~va} z;ZB^mvqRF(8}C~784#U&1QDlFU9~BXa`p7_yZ?x zH1;d3BZ;rm@51|d3JBCY1>E6IrHj+8k#v&6a~$uJbk?4uBTwihht(CR<~!c957F5M z#hJO^5B%i&Tby##NU(({bd+=Dyyukm<=tubCL+N=B@^i!J`a*U=8E>T@yGc-VD4r< zfn)(NRhE7AMO^efi<8V+;Q+en{nq)_-etw_(rk(%lf$2RG?*iesMi`|(~y*ikBR#h z0NYvFiwx#6skn>>4U;YUIF8mB9iA!s6jZyRDl`@r_G}{f1PMz&m$|86ZeOitG;$0C zD6i-Oe67Fj@h?12%caQ)-K*sfXgvfj78d3Cdp)n}=G#&{$pV^sMeo;TzNxh4prnWw zj3k$Poe6T*CicA?`bCh^%ER6Bh3?v(on>8JS=cBD87M8x6(JGBpkz|hafBvoY8DnX za@OHG{;5B4A}U4}PV%yX$ra>m&X-#UhLJKldqF04Yq3VqarHMgN#|>>>cBtL{rkb# zqlB2%vIBqGIenR0ODdIazP&$WO7?ncz1^>KJvBL6OzMq0WS^*&(-zb$u6HiH(Y)Dx zF32dTO941g`28q{4|~s*$Ywo1Q@2JZ*}~*ZFrr%>>%_uxlI?F;SG0+e9c&<$2R%We zhb`A_w!A|-RnZd0_jN(y>gFMI|2#n_N1WtSLb@l8=Sk}1esg=X{}+z|QS~a$gY8@A zWuPGK%L|9&HPVWAb%^zNh|1MeiH)0;qUjD%a_*`fgSuri#s)r*n%4&2f|hoUj6ncW ze$9^*iBqq$54`tFLoCVax{p=p`=4~OoKMkf(r}^P#tf9xdJHC$-Prn>K z-(iRJY^KZnX+&;#eF{C?_xCZ)9;kwLY)=OCk~PD3dJm}D9|wVMAF~ff6)$lV{xngq zI`r`r7VtGdBZ-+p_o+US-t=!@68N?qC&>RWqOGiwcyMP>BjNo+K-y&8N|KN$B*LDEF zq_NpBgXb%`)U8Lw^eTbFhOt?f_4lmtI)?ppsU>}jW z4W$){X!jnzis8vthaAJQKl_Wm!4c5qc#(G7sjJ_Ar{kHF5L0&AzIgI27BFS0R{oKg z$Lp$YXg}?iKNFUtVQK!-w(a8|Z>zaWD2I{i#>Poc?uAytTSwFpS3(|U8SAT~;4SOj zB{h@8t8P5KUoZ=v&n%}cY021@#dS;z(@42&%yC+0Mf#iSZDSK9$_yJ)2~`vcb=9A| zUDWQ+&!+_}aOm690*!cDkq)=O@@d*{?QNGjO_x#Rrc-NzSG$L8d7ee|{&*d4wc! zT}lk3EK*>`bFC;cn8^j-(YqwZiS)^FhvB4Xa`fK@3=U#x!0Sj*^$+DlcYVU)cY|u> zF(GK>{FVhnN6&d_h!7}LT=PQ>j(~exo@>9bs5oYX&B#4(@e5^VK6*szAb9D>)fTl) zbgKT15%SsgC3BPm4QyEMH+0zL^ew&5n(o}5_rpr2dhH0FvS=#S}OkR{P1 zM04#ng!g*e=4GKMHJm|YpIKt>z6C0n*tt<+LT;diYK#h<5 zUN_`~tTNZjp?5mt1+mY!QVGRjca%PT(v*0}_F!jl#?3yS6{=BfMtLne%Vr#Yka+YF_v|PHFsAT$5}^k^VC27+l`G7W(LBTY=vAnvJDAsE01> zF!+Danjl2y<(St){kM}k6*ED?T#tje)u}jRMLtr}y$?QE^E;r2X2M8ox9^Y=Xbgum zPy20wjy);;JdT$YEVg7u@WKO5&5MLyspYShGMeO!*r%-X>A_5WVT-41vV8;gkn2)V z)wjp>aJed@eL-{j<~GA4qL||@j?yz2*cweVF)Y>7eIMD5WCu+l$+5jfEqExOttih>lmp7SJ6<__u@2+okwnDtE4J% z>qBnxsppn-dp(MKO(lGCeKOpsh9c=w?M+XFD|m)OhTJxBkF50roQA*)Dqc=aX;oyt zrvpvdo*}ulX>T;kd&CTMGBf0giTji7jI03zvB{FtQF8UDI{iboCQw^otuBO}`0uD-ju^$=nt6N)0q_(&)VAUC^>0*q zLl4oZr%`NDt$5Tgv02;5+^F1|RwP9v1h8<5)!98RKpX!2c(%H~+(S4~hQcFRv&y20 z#XaH5HpXwVj>cs=hl^m#`Ioz7;UTBwhw$k9&I4;;hCSIC~4J-K920<6hD&( za7dz#kgBhVj`zEl5`4S&)HiQ zfjDqcwSH=9`bqfO4sm=YrJl(D!JpXX++(u8fNMR8`Q?z^X8L7uX=IU38!cn&0vbNo zNUVlsjl5*Xkh+9Kz41;LPn_FhX)c#}!H{)90&gx3Y+V!?qG@V|AO}xJ{k?ScgrXv3 zgGLKX zX$IZNxhG(0jnXWT_wtLH<<)m&)$D61k-5Kd%v=i87L45d1!cY4U>=m>DmtYLkq6lW zTI~HQ|8tV6)ttDPG{^Vn`<&CUjJDfr>1>^pRHYrov&E~P`ve+N5CBEJ>=!0%Hc_EDd=Wp$b~0W38)Gkq$D zhPo*3p~S-3Mc0DOYDSZn_y%TJnNYbXCP5lp@NI(k>L(r6yR#blao!i#WJvdUHwB+7 z-af%8Jye+m60npJO%tOOGvZZmkl4kk z5l{hekwleI%_&oF{h?ZMa95wo&1{5?%Duj2;2OGv&z*yy{+Fx!>YLKTL#d@f76J`3Pm`5L}|@sX2myJqvMHJQ=(q9(3_|cHeTA zzfkO|$gIW5g&uQiB^^2K_@#zB!e!KuIcXdp&^Ag7_{z+#ddVeo!rRJZu;@U>+KHxF zvLjiW#7#TQdV-j^!qvPQnMb!gXgHgX@pZJx&FEkQwgAIJcQg%tqHm^E+^jPNTSQog zeW>ACpeHF-=nkSEr}!(Y2{VAIo5>RwNSc5HbqoDxeqY3k{&f%WP{j1Ny(m4e;CZ7vli2(YHRZH3FiD%{X!vrj=R+pU?{Bj) z8HmS2Vvy=!JNW)aJ<)P)!I$bZ1z)oVbEAcHxa>wyGp4g$5yaoFGZ@^JF$9`G49Oz5 zwyVvkny*sCszYn~2PYbv9t^@QRQGylg@yR12ZE-J#|Y;kT6j0BMIEWtmMiqDIJwjk z@KO_N5bIMYJyGN$4Rc<(M}uFzpJe1=Fm6QwGbh7u4nwI)xkQJkVsXT*)(fl-{*3Ic z_6wxjX~ws#`qM^Mv(NA+mPUjXO?PCpp%{>j-jV_i8)$JS_(z@S+q; zd3i3cDzLGn#%TX4PVT;FRelf8tax3)vD?E6m195TbQ)-=1>oGs* z`ONIQ)IUF7mRgN)-0$iki}N~tTwZssJZo}{hfX&2^=r>IjJ1cBkfTw;7k>OUK${QU zOXsyl{35y$>o}NBBT4W#*3@|4o7~IVk~Wcul{2M#LgajjXn|1_7yrqFSo?L`O6qdlN+XR93|6^=u*RkLpUE>eh3*jV zApC@`1LWa%TDub7G%yluc*mBmahYGZAc0A~yarXat*SgyfIi$mY_ND8d!z7TsiyUc zV4y%eoC${4$xwh%9_xS=II837g=3&cr;lZQ?A06H?_=#PT7C{EpL~%t>nE%{0O=&I z5lb$>c5&aYYG+2U(cc3t-^N`|lStSqdPE&}m&m*b}#Us)2*$tM&_F*kP~u4E>m zt|v((125!l9kG{#9bZZ%alVWuO41@Dy0n+@5+LBSOJuiE@4-59w(I{J-MI1lWIW$% zJiQn9!m{P(7gzT+y z+odrj&{a&kwO)I2z#(1=tn^2HG>j0pQZK$RcA6W+(-ok7Kgm>9*Go)yV&L6`uGZ(s z8@#kO%c|I#x$=!^7gx^ttI+nP3 zgNUCve9a5K^w`EsZh5Ma`ZCsAJb!DHbZBZ;T@x}RfmD|}hU=np{_{}ajESTm;VzBN zr&(>+_pKg%gl5bc3aXtY6MIfxwQJ(++r&i%lgZQM* zi~?d$%Vr6Sa!N{SZ3Tm;mb}-lN2PpQe{3G72!UMNvHn9e4_Tp%&YyP-V_SYg&cND= z)-OrwWvqb}H|0eqAQc;?Xe}C|1O&?!!+67ooYThW;UVa$c>BfxoBb${8zuU&)cTdl z3SM?Dn!jz82L_yY2;1dluH#c2p|jOh;!^gyKEL!?VK18{OYT)zd|TQ<3Fo zd~0F7V4%r;yS>HmYuW+@%8)b}YJf{>Ici^;e{NPuliQC?0p@qu^auW2t6_1}G(lOJ zgD?CiO8^Ej0M&$$9qiz27+vP1g?Ju9@K=!hH!7cWa@2TSZcuS%di|TwU&$T<1XOR@ z-ec$Z{$<*|1?b69oxZZrNFHv>;Y|L4*j@+W(1ZJKe}SCX_$B+F0V>(y&q7%2%}n4* zCT_@_!=V51dfmeWD3V38w4f$$-s!#jTLKO}_(bN3JFg7kk6s<0I=cV19Riv4Z-X!} zl;AXZ+!qacJpN8o+%Wo7st03j7);p8F7SC_y(9Q10p%7eklXNvdeSYHBFyJ63Gnuy zvf-sVwqdzYhY5e_iw>@y6u-tixj`!Vjb4UOT5Hme~b3FfAD9)XGPZA=D1se`?tOsIW{=GE+GvIH_t^E7WH1Q*HY>1#TzoJT9HoD9O22HnR}OQ$5c-YgU|F@xesEU* zIZ-(xJgIKiczDahqbDHFFA1vuGZ+#?OU-1t#H0s1jUeVfYxI9+=Emq_OM*exJ;ubp zzBA)w9y$=xI$V^M7x(4-#{;uP)_+m1<%kpE6>z{#7^>hXMhZx|hdtjcX}MjEh;$lP zXw6Q2;%|R(bu*cFt3dj1ol6l(W8}Mh^wHB8WR~qlzgoGtb~p_^HRj>B2Ib?pJ&yF$ zwl&d-psW0+GQ_=s;>00)%W17&-O_-Ro#5TQ@qjR&>eM5O>r*A+5>7qU*(J%c7Qaj9 z)0y$Z6piVI_(wcPsh^w?ZY%8_aanVVhS8QU(+A_~I zMZ=*b4;P@|#3Bhf6k60qD;QB>nLi@4(^Bh(+@kF{M10lvtzGmuEww5?;0)rQ)BTSL zcujYnMGR+Ps!%SWpw6xC;&Dsyv-y^!CU=PQrK#z-b9SY`b!cXMPVfnN zZrY3YI!q7t-}lN5Hbhfhj5CMv(n>=+M`OVw#V9Z~o5e#b}=uL0TrJ^z3FoP&l?;3T9SwW#^hdRGk&Z>#OT zi_Q}5r>U0V$Z}0PTHrTSz2{7QP)I~AoWYsv?PWVK(8NTK13y{S(sUH(SXT?+6NR)} zplq>XQ)pKpSdkxsK{!nmvI4={SC)nc#2YHSJi^tP?2ga!d9j4!&#BQ9G}!AyVs3vg zidD;%U^uNqnHdZBAXtp86530yF?P9eDj*7lOW_P5y-|uOWg3ncbo-eurwRJv|B%2o z5p6}r!5LOx=k>)BTxAR2#pL*!NBi@?8q7IDc;TNOy=rn+hf6K4H4DXsg>(CjGgDLY z5)$zKw;g$~Pg02ccRzc_G6=uGl88y#(`h9g7U{_c z5(@XH)>!^MyJW^nakbpwCt+2}xB!cNTqRg}e)X?)tpOCx^sHxJO0)33bLYx^&(@m^yD6Qb~5p|Y~DjMcY29TV`^ z$aF~;;Bh5WNnY%$p|pwfKljyK^-%-!L;H3n0|K0yLhJS8rRj3uJ&hG^dfoRMd~~zp zh2wQPBKtQMxfjhTW-5a}auQQuSY*1)h##b27tq^}74ozNd`el#4bayucqz5ao1JB! zEa=4?yDG+Wdps7oT7=cju8F{rOGtC!P>IKM^fzojO|V~I;0l^3E3qgLW3GgkoEu%2s83F|~d zt&y8avw zMoauK8F9LIpZIdGHmfmKq>RP$Nbi0mVf1ySnPVhF5%+b6lK1%dG>>vp?TGPV2kwG6 zqv?MI&c9f#Ju4`VEq%X_?~z|K7KYmZIe2eY7PbSBsbJcuuNhqsR%}9M`dm>-2_oC6 zQ)9ZCxRpiF_^s};Y<=sh0N#D}`@lCjrk9c-rBBS5^|S@2vefxXfx8Nce!Ay_HI%ha zw8d1#nT^!xc`mh)1{T*i0y!?s=%JJGz2;@R_2t_ZJoZJjHY9Poz5CRGSmTX9g)qvz z;y>b^+1!K-nk^jtljlZP~mYg9}R@m3C)-e2i zIqBCz<;&ZkRJ*alLmZ7o)197-1*6j~JD-aSyXE90bHn|KJQn8@t){gu#MW|YL6%w2 zknVG|@ksJ^adT|_B|`lyC}21_fQ*W=Kd%j>kUT}70u)3%2Ml4k`JwbwSjfn?PqZmV z#)w7HzGOw0Iu%pkOP^M_$8BWcQDwOeh8s_ABws5>>54)JyJ@Jd3OpgIlLO**lx=dx zpHOlx@?ED?Inq~FD!nByBuyOzxwoxB*fuYgx$$iUlaC8pZjnGF_e8bqfe(+n5d#Ix zV&WZig45SBR~Ol@E}VaSbw+cv;m`rGWpl8fe-UK_z33fB#g9#+z#{Et75WXU6n_u& z{bGDZbMsy1n>h~&>%>vKB)I~61}5sIn$6?*8u@<*_&x+5I}(u2l4}XzYEzw8T;0;= zcIldFMp)~aWJaYAX@YJN+MnaPM^Ew>0*sq3lblUKy;dxlEs3K%c6ezlt{7?DTp2%b zB7eeeuunir;c#Tme9GW)p^?G=ED{xtBHz&ZBOsnolAg}X`yxO#tF6vvd@HMOB47cf zkf}YK3@J&(XkFFZ%DiC4B{5(pfMR?~{POj(oVz=9oG2QG&Npn*5Hcl$^C$;k;PF*g zGFKa%_g?k6KT95yl$u4^ilWeM+Z{n`t0C(v_!-YDkj?`VIM1EPx>xM)p3RJI!tRcB zx2f00)sOVLvKpvcf4AY@mcp(qv#;QVee>(xxxY73r8E3@FV7RR;N$?&`<HQZ)f)juL-Apo;;6@umq6fLJ#~0pIITmUz5U)irlA&XV2FFC9fi# zFYTh5xYp1;^_ivOhiHQ6{>3>6!@MHz2qzSp?%R_3)o`OsRKkABlE}G%{(Q2@%)hmh$b#3@enYF>oD_n1Vfy)n?+T=pzt3sj++;2-oAA)3G+kR3RdFIM z;dw#quqa{V0i$z)z^L7frPV>!IfbD;zrQ%h?gKantL~8YHR;S5pRoM=6q}`3dVy59 z^btF3kTxe;3awdz019#b4NA*D&h{^Fj-w2y?2ULRgOBSBJx5tD&6~xD0k6tCE+b05 z(~Zs$(%sCQ-zJyYwn{JAr(HRCzs?}5WDkisHIh>`4skypW#F;_&2ws;Tz`Xv^Vx$1 z$Jc{0 zM+fr~U>46aY#N%ouq$17om{m{DKJ|p+lJg4n^y<-s`s|;xMli{kEN(GhZ!<8FguDL z5)K<)%EHU0BK5iV$NlUIX~;d*le>r6>?&=}0Zt1O7&#-4b$<1;l3;RM59di1n0R-% zz>+1&#&n#yLvx+IUoR^DgDib`WnSZGd14pSqwF>}-B@y88*2144!*m&k!p;2h2^kY z;qh_1Ex~aaKf;+?L@3C{?rN%iTJJ@n{IJ9yD(JYeIyj zbb}~l3YgPX2DAgC3x)g|XxecSRHfoD*Y6XH8NO#;5#W@aU7FeF-bO<`oUS|Br8^?w zhtJEA(B|s$wrr=+cWZf-Fgc+M&36AKFNnyWC+PE|q%7>`Im*=P zN#Rc0GlZLYLeDp#CI8wu5RISXKckOAhpnx#G>@QjEA8O0trOQewOESxzKLg zB+2->JQ!!+$%pJBZwSm8wVc-Z_?d2lqj2nDZeW^KE3gz0l1k>wH&}xs+31MZT4B~< z5J+F=e3O=`;3`76$5^#cSyE>V)xEk|)2ULiqQ2_k zNOiTYeN-z83%#1ifn(o`C6pJM0_ZXCynzBK0>Mik@S8B`(*)P8fM`9*6r*>daxPDF1^xH|iodPT%ryXYtK@?Tk zjZrH_OMw}Rtd8oyjP-X8&E6SLx$z;=vLbCt5#+1v$Ya62Q@dnN8lv7f;7lYhBFu<$ zI;hn0Kh6r%!jMiSX>d22&? zT>9>`MaH`nIfKEQi}kOReAx*99+D}7%0>fYO5s*&b}Cy@M?DUqF3(J{Y2zHRm4lgd zVqu+}xYjIc%%gQp-NvO@S?OXoZR^%#T$KwU7!)0$;k%Vq%@g^k?q-bVUqh+tTD}5E zvZRpTRI$mzw+E%xaW-|+BwUk<{KRv(AXX6l(0xNCn>DQ|Wx710F^T1(-ISI5r6uF5 zF6=}IsCU?&BhqoFlIOLQt5Y%%;N6fi5oB?$>Ub+gAd%60R9$=SMwQ4`s{8(FQ!``Y zxY!ysc(#8=0#r=P?KflcEzH~%dWxh?-)`~8c1xW->rfwMH+b6kQ-QpGVKAiOB}YiC zYe`Pc5QJ8kV#_JK$Io41jT%pAtL3VI#jci*v?-!jDdLNZh^Dlw-#H`I^i{S*Ob^7f z1IwREk7%x=FXiYw<`Rm+fHut@if?6h5#KMd$-+B{IDH5ry9_$>=rX6^(A=<}Ti9DS zl3|gd_X^7G!Mi1e3`6>NG4bC9H7#!>YsU8K5KbDo%*5=FDjK&VZnsWq^1L=bG?+8-=+$&z zoM{A&S$dBzQjKuuQ9FPKKe)?iglN1Yt{7zVByD(`XS<2k)_KWT92Tn}d+uP4j7>FU zOCnjL#xkk&NGezyeS}(ZgrLA#(T~$7*mAzDJ{73ueN1m=*KV1-Ag8)@9VW(!mq;_T zjVq;eLT|cjWbI6AnTb8aqGTkPn2*$-ms3*he9z>UevYR4e*fT8a9G#(KB`S zHATdt$@E7A%?!Swd?#$boCZDF{bLT7->U_Eb^GyzH2kV#1Ljokg=a<91r@eJc4v`? zT+PrPHMHKicc?J?87ZXHJ<`tvy~uNKdup`)Yrw)GW%UL-Mg>P4JzCvjgI`toBSr>f`(`>)b^|(~2>htg-*ZVj9MhMxQi%Gs_)(=KTMtSIq zWsh4Q8kLg>^A$XV48N_YQVM2^2i4oMmF3HCav@l0i)QUyx6s-{Oou(gmldmvlwiN4 z{J?R6yEb!4@G~SoD@*g^>~|thh}fJq*~Tzm?hTzfplC7R>k)mS_n%!AT9=6!OL2s6 zbFB6ky=f1D*?J8!#G8CBkLvO_Ff)|Je7*V@!p8{+1LAJl|6Je*>R#lv06kWIK_m zx2$wBOE6v2-IwCBvb_9!GVSG?nv9pF#oqZSD{Se9oBFmqpN5teo~HEx+`{baY~r{y z^V0q0_`ag@URf+S23=$9#pyHZ_ael_|2v8B!QDG>bv+o7rhIy3X3-Vk&b>)68*$-myyk zpImXPNu%Mf;oAJZ)y#UVitGb?B$m?R)KQ)dJ*qwCP>l`!CX27=n7iHkM{ipo%(p0& zwfLyQfqa|OMpJ;ajVEA1f3$^}42V>D zGFC1yj!*=WjZa|Pk57(P9kaHTRgk?6I~zR^Z?L31aLWTm)6kM$|);XVg99+ z0`t;I=|4gW6b$cbBGOa>4r>U8ba9~*U$qPgGH zoROu5MN_~h0%GxnG+lk@4tCfJ0-x$-H*c%13V-HlkFQrpOB>j=^9N0Mci~r@4^~Z2 zF{~-Nfqs&kIb?iLo_fTxDr)dPU~3inDa5I~IwS6??#WRIh{Uo2MfP5E#{>ZA3Z`k_ z-d7eyi=s3HRkQPpEAY{m(Q<`sqhB35ADBX9^f?W5LOXPAF*(YaeN zfHk*qG)3@V7G)JRND{1!d~y_Kd})psQyAU7n(Cx73exm(Ofs3%3kqpp^HY><;>yyL zB(Q9UHehjK@74dYn_0t4R@M?y$0abxg9uU4mcCSN7lj-Qm<&1+)$w$C*V=kO%2gO! zpL}tgZbSDine2xD`@P+omC(qi19lfO6+mfTk;>%WHmiLR%?s2@yc?H)J)eXTw8ef( zAfXI>S?G9;2lqCL?xJkp8QUcK-4Xzp#^Fzw%ow$zSr@gUQyoAhg@_-ZlW{4M&XP&= zDFH85Zr2v|5(znf8RP{yKvk<}zO$$Tg5&(JEnx#EY$Rne*F0<2er{2ZHvk;S4mFmY zxU3_Q|2(p5{p2W!Ke9L>i3xh!WN!CC<{(7|y6PHZ-u4xs7bsYqW0)6M;%Kw_OS+I7 zC)S&tQ1GS$W?DrcVIp#bhuq9Jtt!vuu@=L}`rC;2S{-dBBp`bg^e>sX+=kU#O6fzB zKSEL#eB%^h=a=+c1dPA`IhsxR6uSQ#NQYXT0^J0ta=8lArSZ{37~`>g|AF`a==!Rl zxS}lD5E2LpfsjCOcL*+xOK=bF?$)?N6M_YIcbCT9-K~+v-CY~k$JESwZ>DPI?|$8? zd(Syr)>?ZVmcPeWC&wXlmG91VhZ3lgWfY>7E&3g7?f7i&Jb}v9hm$-ZH-4fXfcV_I z?Egx`RlIy#`z7*s15f&JKhFHY{QwQ6r-9RYqtx%z#yam?!1g3+1Wgt-hP-4AtnTVzubtj?{V3WbJtNy9hn3{Avy!(%p%=7r`r@fs!NGz*EZ9t|5&cFHLNAl*;%) ze`Rv6__#}4W$5ZGyTdDN1y`5ds;w9NraVg84;&w*ve=V=>hdt`sDvT;Xf$vb&UclzVF9L}3avi=S?3>N_BL?n1~ke$B5LAl^6UuB1!(m7fImZMf~)UnNA|b zh)&3vpu>7mdcdXb00%dIWtj75U;_Ju3W|}Wdw^EXOz9{6mOl6$8i$>v^(^`HxEOXI8ZNq@tOA|8Nn{n~{oKuwUgAwl@7yu|_y zCrEVYu(gN1szOg+i5_DN_x{^=$ zxT$Utfsa+Nr@9kJ#T|X8O@72ou3mKGALP%P&q}KZSorUis$K@#hhg?6jzGVyNuQPE zx+Am9)5q;qXoOAw<9dNyR<|f}X6=<-lchX-7){6|#^$HO{IV$a+Rxcot4)nzzIsx{ zi{_HXDE=FsANVq&W>Tqg!n?qHei@y_vf^@coJO-)QeOQb8h|YB*NT#P)iGsbHA>-lqZr!OZ}r$%-N7&(}O{ zRqFL9I`+13A{?=|QxEAmQOetwy@~}BxVVd#qJwC?p45UZL0`!g-M9@bt5g`-~Fze!$%9CIky93|<{?Z_7~AE`g` zv5l;Q94R2oC`C6#OE@LP;F8#8Q%jdH=YHm0E$l%yUTCK(!oIS@jIST4W+!|oOUU^@ zAT+sm*?1NczeUNuW;yeXg| zF;6iKU3SU~jB6ARkSea`jUu+!;9{iJ(bJCu(QQ_-a95=ZCIm`|lMsniNfAp75M=PR9#+ z;kWxBWWy!sK zd5hchWXFoQRrMsIA>r&^Od?nq-o}MP8sjNeg2?)g@+0JI8~OBydN&-g!MxQ<)xd7?IbB*{7@j~sAy;rZ$b)HgTN(HtDzJYK zZmqt*R~D#u%_Ibl1*9zNrzcYgOLlPCC?~Q!;e39C;dJdfrm>Sm(7;G)tTx$2zhaIylBDtB^@2ocF$UYn*3&Xw8E<|1Q0hdh7r1 zwgdl09$b>Va?T`UrvAoU$HkbVRlY^YGz?j%T8=#UJPkh~`(G;^M0db7ei*925(-Xk zp!ljnHGb5MS^U{o)*w!l-4LX7&j#%~?=FB8XUtv54c;RG>SKN;Ozl@;nW6K01Nd1rykvdY^cN-Bw9pSnZS zxUURKQ>@Nd*Q$L*O`ECe}}O3z6xU!epcSNU%G^#A^j|9BqHdDo?{6GP!`|o z+xB{RRViqiP}lrdis~Qh-T@~|gcH#((bgA181V@asVzW1h0x#2*ft0FL>(+$L6sX6%jxdAl6bzFk#}nB=SeZ41c2 z0ant{;hY6DoLF5z?7o(Uq0^aiw;ub11Y?VYEh*qUB1C%;@U6>9OSmU>zs8?)+? z^mz8fEGMe6hzp8dtk-5VeP3(J8|8o3$2aUC2;9APn7O$R+cjy)TwZhSD)6;ib~Sgq zxl&g~WPj!oYHMF$809%XvEleIHlRM2#3Qn*oQ9-&%D5Yz<}jyT=QeIDsai0dneI_g z@iTu%z$E~2e4lDREeSz9;h>_om3Ue{e#mI-E+rW}k@+()$7t`v0cSUMk3vdnvIF$4 zobyvNXi!^{nGjw4z;;r5OYn(aIBzkCd*bBw&?p8PMttLDglx(P;bNvuh~92KckAkI z@aChq;YW>L9OXE`Kutty&%%A=xp;C?;~MjIPrFUqDTl3|+Jd+#isX13Pa+nhS}Y-{ zh^DcEBy-0IGfOwK9&fOrQz^{Y(g_C*2#HqGa#e<)+`5l{+Uv_{@#8=ALMia?&ao!?R~uk5DGj`^pZ%@*D5jJET` z_w{eyEY{-#u#yGcZeZUJ-!IvoLpW3-eRHHSXB{&mb%Hu8Ei_X zr2fvT)3P>z<<;*qo48!IYkN|^kEPW>bCerjnRTI0Jku(~V)=eQ`BiN5m)M52)>>p5 z)0>AD7i-q;%QsPZDId(rbmnY!PR&#g5H_b`oHG-({Y8I#*kAJ~&{R%&nMrartE_W2#n?BU_P{3LpHI>d5joBI@b7OMYf{dqV2E^#p?)=*#uO<|#|8X>YozcAgvCcy?vi4&?UWs+)(`HBmrKN(Sn{(hT*T=Mt= z`w{N z-S4tY;DZH|`%Cji(K_Sc`(n~tazKkEQ{zOWYLTWFgPhn}`%Ov9J;Yt3ws9{7_!>48 z*cyxW#NoVaGv0f0?v@Y7kd@@dxwtd?p$$jks&Dcq7+XCtOJ*17GN;Ct(oIh`#m&lE zPs=OIN~!c0;(E~`8v7bY?ea?)zVBzRBGL0iQzt!oj4#gJTAEWjT(G$sF%#9H!|_(u z)zn@NEx0!^nr5|k_B020vuvs8KBC-$FAh+_Mk69C;T!hy8(Tb+!80<{jj9$q!wI`h z&uonrjU>DIz)uI@;~LOIIM=)$0b?%3mdGccr6Ee=B4%p$g;9p2o*vmR>ATf&H}^-s zCD>C(1=88YqjQo%=gxa8o}#6Pk{3-D$<&kE%VefY6$Dg6B%I1z!%hade5AXZ_+kz{ zB0t|Yaz$>%J4h7hV<+E~h4wo)$dtM@o<^Ia-V{ByC`)@?nxiY#cDP1`jj8xk))U=DF&+3H;36uB3a6 zrD)MJ(|R|=$dz7sSPO<<$_G_NSLa*|!W=h){+=d$TuAucw(xCivNze857T5c#b->1(juWbe&O_>&n|M$& zn(sT`Zp5v3S`_z7O1QV-TYvwM-5g5zS2tTU|_QOkFkO)vXx0lgKbB^{PS7 zlWCKad+fc?>hwqrl-znFIwiwWdRW`nrkg=RSFcBJn$6>z$%I}zj)diz47fr`gd(zL zh#`H|nTT@7UJhdUi!`62yriEf4IA(ob}HSZF8bW$rgjXaG(IXSq$GatmY=K%clxCN zd8qw;oc$9G{y6L)KhxmArfPMJpWjphrixMOw)=V4c}e=ZakModv|R!9<3-7mK&J^E zt{}i>)~Kfu(q?z%AQQ?}Mqf`*44bLT2(qR%3qW$5N7yYqoK4kLc)Y^U9;CL_4jv;M zZR~tNOqe`{^0v1I2oRf%C%bgpm9>u9nCd*#20m_@p8mMg zJc)9P4>Q#%2Ie|49D4XDHJa9D?Ew_e8hkrI-U#Rv%E7J%%XX{UFKR_YYycNQUkxo%o--~stb;$MyOSmD}7oy^A= zd~V8F2N~(jdG&^&+6>^Tb{r*z_B;aHj3;&{w*VjWiY7Oj6NjPgN%~Ex@`1%v+Wh5% z%J$;Z?K<6Kgv%E9%na-1jQS=1hs)Gb*lz;oTGCUJ-GX6XP*9{R^>d1|Q?hOggys12 z5qv>vE0_ zOVi22EQfc_4Z0$!sg>J0B4gKB;Fa4~!HJ=Pd0Dbu8sQkjVSDg!$OO#3e0i|(+-mG8 zf2%$um&tI!aY-8QHl-AlbAzD<39|`vby_?LOJJrSzDuStO0zAcz;uv!Z0z*l%g#3t zossT}mkD$)K2A>^>AXE)b4#}kDWEh~l)=fsb@Gg_UJVReO=rp}7&5^+^ewGO5554S zY)^}HU%JAB^Q}J_U3Aw!+JO?`-o70RN6z$}tDHK*#>bz9b-U?1U{ux_E5*ZZt9f6Z z4i=8p($l9g_(Qu7{KqD~u`&0do=gym@*)9n35LT{V?$8;ay@LCmQc3#H4z(tM~O}R z;ks1_z6O)fPE-R6S;T60U{7rEvl5lpNx>l$RIs}ZM4(6IdUZdj5AaJtZ(VYBv(Osf zCHmZHbj2H42Vv)inLj_^Q-2g(RJg`ltKuqrdy&C~KY4#R8Xf76b7=gl3MO^d6$L@i$Ca-cYQuD%gVy!gC!Zre6>ZtPpy89nYTDp|#5n9kt0k zwezj(Er>?%@otT*%knrQe@ytdOZY_nW2Zcw7C5qz*z zJAY7=m6;$48IN;TRkf5&gKgO@@0z<4j2kNMzywQzL3&ycIqvkq)Q8m5^!Ah0X586x zr)LT~Hu}ZaBIoYr@B!C$WI)|JC$%9WuyU0##wzQdcz&d`Ks)AH2rK52LG8{JRwc5FTUxwnvafvol*_+3q9LW@R|8>TwFh^-Ao} zKe zrz+}gH#{dwaS6hF!vZERwQZS`=6C$sgh7V=vxhJPfz<^7>^sWy?ao~xfDe`1W|Kdn zJk#?boAeo*U=rA`>fN~H^Af%WR zcRXm|y6K%6IwtCq#th$R2lLsesjKO9cgubWchkgk^@Nw=W(v%5j{T5TdYQ=qzq@H? z`w-Zwb9KfNI6!IX{`g&Ex3q9COZ#oz)I%f0&c(pI(NcW2j%6E#mNY5lcY=dqzB3!< zk$5eGSF{$;QezwQupa3`oOkZ@aNcBU@TbJ-7lcEZ)%!yg=p856eaFkow157_LGj|C zE)(*r@8(hVdNwAZb0l)V@qRem;-eSTl?b6SU78K7J`#+(yy@v#=Hwo zcb}14*ekbJGIW4}l3w2Lc+M~3H{Gp)ShJBE<))(Bx4pc2lo|1e$dVJVkYS1E2W6Ik z7EIx#S(gKZ^amdMM(eA`+WV5N%gg27XN$7^=Z1Q?5U}rVsts%OGp}KTp{FsZ_3u!;^an(E?6SmO8 zbPZVceUg}j2fy8XY%3xYD>)jwKa?5Ch{qKWjrE?UH`fL~c0CVyiNq~H>i8s_*I#DnW?~-yY zxQ5%Z^IZ5-D1`XVa%&09I=r?Mfi5zmzWwJ}nOIJ-xmH5TK)@OcZg%H$ zsn<$_!P28L9E*yc=ZVrMlAQNwfR8(O98!W0btU$qDu_`gmfhkF9~b}8OLE04PNNy) zk?ua|AKf{D!M?Y3PCts8D5%YrZO|8^qx+qlrofxksq)*YpYliN#*PA`&dI~tEU)?; z4J}VRk#?BVkDi)K%jp4h1_B~eWTElp1x`i5_+zzVhg#PM zCdjZA49**JW*@%Fjs4p?;jS7>2?W@cqcAP)@j{A5z2RT{sPT5Ksz&Ro-BPyci%YA{B*#;cm}-o}%)wmXj#0|! z?NYu8D*yvO!(Y02l;z%%VvQ^s2E~cUc}=FhneCvoE2GzTlKHir?DM)BhKhtgY@uR4 zF4d`LOxM{>?=3i!ZX1aV#gWejN6j+*al)23@LA%>28kCam5LLYy5(REgk{$D6qf<7 zWOizsbacmTChVNXjb4ly@m1}Q6Sxw#$`cmPLpezyGR2sTB^MUGAGr8K>+oll)y5>| zJ6pah*8g#qrai)C@^wgiA3e08X4*u;@FXiov+!jgkDu+m@!O__A5Kv&jnu{O&1V@f z)~W{H4Uue3D|ZiAHs6r^tkZTZZA%EzviLakw`_XKq$s3iSVo>sycsmlcUaDP2Wq+r zwzxG3jby)v>(*|SzlyRp}O4_ypKCziKOfRszMbnyi?nkyKYtm}5ThObbk(wgb z^FB8eM=e-fsfit%10_j{yrGrvMC|5&du0#&0}ahoe0zO^6Fqp!u~Yz_Y0P#iYgi>K zqI;$bi;G#!-#h&oeNd*$l_zv55z$(($xlCW9iDVcg3l;j88yM>#>3E0c<{?r`K1K zv`5hTIkaS?L`jHF8n-OEjJi~X>CHqYSM;mhmkt(lirrER+8 zjDA6*f=c&QJuN59S8$Gjhi zm|vT+K@j)LQ!3l2Zf&9!rSEF`Z)(>^tLAfE8s|135kyz>?9myaaanQm&vQ5GPFcpH z@YsRWpxi66c4Z&8ZNgChzWl@z5$^+ger#HHZ4xWqjT@-o(nH15i#n)DN^2}yG2V2S5A{)Nfn~p0{!gPPK{DFp^Ig^F-SV9y z4|n<(ni7ykJ8X8l1cDIy(nh{pZ@5Pgh%K|DzhiT+4%xIMOItli{V+&`PvNbFVW&)4 z?gX+sHs=I1Ag(!1+RJ|=E4`2kUTtfaWfzf4F+K_Z%V!G-wY=tUS0iQ&tsJ_ier}R~ zT#9XJqQeTkI2yS{0VgaXH56>_BR-kv1KRk#;$z#dJKc^2dB2O-70{&np|^lE#jj?O zIJ}kBuTt~Pnb_Q%JY`N`%H$^|*t9!VlmIN*K8rQBWw@y#_z)RzffKiiz;cGULQZX` zwBdW8c0Qc|}Z2A%lbB3hd=1yHd7;xT`hyn-7~s{OP(Ig?sZn zdNLZ53$5oyzFv5c`?ma#MQt@h zp40Iy_NrfBkgwB921}LY#ug?`-0U{F_!TsGc;n;RALCftt{84pLo@NkVp-?4ss(Uy zS=q+&)~0)d+0~rnB;DTivphHK8f-xkC!iLYGKl9~sRGfd;fHrO0WwCgqa{ z8opf@toy*Z8}&U&@qKqRj{S^uFN?7#G^5Iu$l-e+vgNy;em%)DiHPZ0o|xR}55W?O zlc)wunNF6nqYY73=?YHI$*CAEpz=(WM-5|d-yK7_dA;2Puvz9todOqcX-@$!y4i{GFMh?~85`5F#`W1S z{TxEPzCFZJie`QHrG_1Tu8oA2UFK~*9Ye}OhUW!yjrOz`(8)c1IJ#GQVr5D)w;Mku zUzf~FtNW?qs*NSp-Ejxam<`?ei-8f@x7-BT{m(o3#gZD*px-;+x`zHHQ<@#AS*HI+ zm_SwowCtdyg6J!vA9p3Ar*7;T{3M}e~;PhmS+^~&F^!y$2goTrq;S8 zJm)_go9sOTa++0&un|++K@noY%37nr+-zRnt~PJbE><&(cj4G;f4kXiEaiNwxo{@C z;j?qs0bWJpMak~!OQutdeLi{0VmG#;|GOH!)Ie?G2@`Br7M$kcRUAIUaU>f*c5Q_F zV{o9_p}XtSp?h<>g3I4)bLUcU%KHGSV{`8K(6pW^R(9g0!(=*f!*?^MyBRR-m71Tn z-I$r-AR7@l3E?K_xjWqD1F_I?r{9lWaJlC%9n2>gT`z_o_RTKcB*=v5a}%!qv1S){ zrPoLpl-L-7$|&=R$3|>hYKCR1l%%glDKWpCK(nKqPZ3kV>a(Mb;KN-%?4+yIHwh&f zAYcG&sUD#P0yqpy%2WHQ*=Mk`WNV*+DUl&3@r+H1e|NZl8rv?Zv2^k1n9kK+tN5Tz zJ-?a@p$w!m1g@e2-(HADZTZ$X#fU5}F4k^!q?r6`OpFX|9HIb30Pm05m}?|B}fa3>jPO#DWwBF6`QA|#TozaeD&{`_*(ikL|*QcK@5ww2nQ zl$!KT2+Ka2iX+9pCcSLQzu&;5UlBp}Fp#QZ|b5n5Tu7&>{BmX9C2wMfS?i zdv9EW0=>9Qersjkw-H0citiV&$2#;`hH(0_T=fcKt^4`SxWc}_6DjS=j`wFGxSEsD{V-0n zNnL+Uty~Voyv}naz+^w*mybZCBxTn^bq^kUfVz|IsMI2*twY)i{wb`Y@CTfkDPY;m@Kms4= z?MRx7G{(vL=y^ceofN@1^zRxJ0~yl--$wAW<~rpO?Y7_kIq63wMc5{z2BHR?s*A8` zun14cH_c`~>|rt=ZajpOTtaS)V-+SZ#?%@8z_ATi?v(+Dk60JczP4MLHDdq-#w0{w z&UnPTZT7sU5mUdQdxZ0BW|qOEuBGcAiz2>CICW>%EuRFG_}9Ijo&_xRVErshRIfNeZ=3)c$5 z@j7kq=k}9^4)rmIT(++SpjNgsRX)BvcgDCoIz9`jR^G>I%{A+ajIQ@3qvGh! zwt6ZRh&r`GkzF>>n4R+uA&IM49GzbI=v`-wYn*;tL>->_dErGY%67H}yTwGGK4eyt zOeXlxCck+(_^oPaU4E*3a;7=R8Ir!O)c*>4W1EmF>BQf#HQM6NGh<=3%!Kct=SIe6s?N>K7%bxyTe%YuhI7!DX7tbo#!!D4I^Uz}&$AlxopB4b-7VR)FM23A z&8{djGM-~2sQm(1XXZ7erbOw^(`h&@qpE$;*hEvZSpO*>Cp}B^R+fYLCD@NWOOmZv z?;DO_ciuvS4?dyAcmOVgIU)B6KecOG)GxmXzivx`E+r-^ilIVFyVwSu=txz@GW3pt zfWO>Bm_Mttyr)R@nRL)P$eztdL3U)iIDT0pDHF^I28`daW1)Pjl{|{THECr2J{3jx z$rnyztqTHvs!elfYH|TXvr?HAS5ADmCo@t&?7|$4pFp@oZL4sm9sneNb_d!}nQMq4sEI zsux6&)?=Jr5Oz5%IEAB7ATD!=i%-R@6qYZfzvk>8>f%8<=Er_~UQj8nt}P+cgHsA| z!mFM(_kXZou8@E_{fMMh9xBw5|4Dwt1Ngj=58Uf-+sMUxTS|I06kYzK?BG3N=we}v z)^~T=9GadytU{=Jflmg^ntXlUt$aCW@cr;aG#i+)Y@^TGt-=j$@nG zBz!j2^}X*Q;j-;2b(~h7w06N(g+f$cT%JOpC&iSu6?2rkc*`kS3v`=sSvalVamKR| z1v}0mha20e`D@Ym1eX4%;%Pw^RT|1u?qH@>$RUqjgG8NU#;47JfS*pEev(>TCTC&x zTiXt3j(jzy!ldeLyH~Vuoep(pEzhN4&RR%v5;xa({;Y77XnTNTZZXkdJ|bX?w%4&HER)DI`$p`VLb*z8{*BKag zrd=xvkotwS1iBHTelYEqmqag1VxxqNIas48E<2(3e=AgkV-hvN-FDjRsbPp5{#Pnq?de}qAVlX_3; zXFCftUmKK#3EI%32zNWm z1#4H%aDvBC5B+|;GcKOS$z+xv%c1|ORX4)HH7HpjSs4VYCMy8NWp+uV7RDLe`;~Np zFdGXyaOJzCjA+lx^a`k7GjD2LP?O+Xpl(j=0g}aeXAxcs)UzM=seCYN*L9lj=nMFa z5Di(NYZGpTY6ZFiMLFR(`5V6;_K2`zR37NxiOa!PY~-r=Nz&?jCK>8!u6)$6XGWN5 z+t0w_&U_eL7Y!zPZ`aB)j$KZLa zp!(dRd=QAHa8_vkGWCzV4)q3H`2uC~16zxUE;S$$QsUw36B-l`=RsoNWznA}^6TGV`=vQ3Ewy!PT0UCZAdG8f|^ zu4w#`==QZP++VG2f-%qU?%-2Cwum%`xE?TQAO(t$xufC{f95XG-;G_Dyo1V$v47#^7+5kj1D; zpsv+hIzvjQl0ltlRQ{lk>(0bfVsBM{o_I>qxw@c3q20*nZ>5RSdaa(xJKDIGo|sIM{s4n^|oCIvOG4r&55m$cw$`L`tTnWn+yr;Bx!% zIvD+o$zwF+Yt}k{_}P1Bp$z^&fw~dNCDVDE;v?QcI0dx7PFjcndfZJfMJR-@@&_k& zBSgrb*NT-O6gOYKJ>wP|ctleG3bjL8+`dEj^1wrEtK`uZ1PHDrZO8E#_U4~LNBf7u3 z(m}>|kidCg^<7I!#vrIr@@nYskKKAO7^xBV)h1p8yp5nfgp9nU!g#=u5An+lwpDgq zLo3PG48h7}JO-yh)9sA4FrR=9+!Yf2qL+(z0nG!Zmzs7@{-Eq%$V$e^H{3kj11#3# z{j!1~j>#>e*ZKuW51$V<%cV)v6_qT!vI#JEod!|(!caB#0j~8CJf^PQS?kDplyC>$~4pser>okZUMh^eE?TBZdm^2e~J&|!wl_kO6s(GfxE8o zamI0@EE0m#nu)&go1)$tGA!H;HP1de<{5cR>z55ERJN*D{X5@5G`tNeRg0<#3&P@Ldj?HL~1Q zdhh)ZAEFS4T#ielRO650Btotw16A0Vn-T^R`DAh%%T)BR3U9_k-IJO{jg07gp=%U! zMrJqe`?C`Q@8}Qs*y@8Wt#lbk-H#kPCV)FxI{EZAG@33!cr1Eynev{&i;Jb!LZ!-Y zs_Vp}EFbHgdDdhVG5Seehfy_6-gjR6cH6>HE9N9Qe(F_STMqG))0`5eptlZ8F^EE2 zt{d#E)yy-LDl5Rb`=VZzm$$^7(~NvvC8NQ%*pIXSE=_*SVp; z2g1#DQ=EB^vQr>>;LH{mM9Vd}2TaK`O~E(Ohxdj8PRkuru_vpA+&Fiu@n_T6to zq2P;|&NmUzK4Rj>k0m88A9mPoa%3*u=iK~+^+R4j1TixMx8%IJri=K@EyFuxfgdIH zZ9LHu9&HLxV;^>}qMKLlw2D7GA`r2Wjo@YqSGWJHSeg6<&L%b+Ks+Q-oEx40vIs{D z{aJQm?<@I&1}L0{&6yFcrMDEvjHu1(sLa$xcc#@3W$^+eUQia7qUM#=Ec?Fw;*c7R zVC7gzq>eh6wzLa4ADEE4WiaC`Tldco&N@XIwioL8vR-*s;6oR_A)=L6+~WOc5`X5O z&~ZGi-nuE6=n~iMO!d%fqfPHB;j%FL6r@e zL^33qzLp7axfnvZSUwlpT9`5!j2z=Z)gwe?tfxL^2}s~c4e%%G#viTBosTkj`9eMd z4`5She?wj>=ekVLRiWs$W<5@SR6A4sw)>r&+@~Jjj(e8d(L>yd)%tQ#9nC2f3OdV( zwsz5nXQo=Gz1f@ml7eB(F?7@Mt%zHgGs1I=_OE=Wfd04Wt~taL1R84k&?k)hOBgA% z*zCiSSP;3Q9cVkN1}8e26D9)+ZRXIg{wwx3DC^>p>j293`|Su$QPO`+l%4T7MK2D?rLx%rR(m!;1j%pIr%Pf#fSqEpJ%;p}jie@Mx!YFY{WE+U+fGpP&)=*+M2!M>eS z49(gAoIyn(7eu4!MX&O$_L4)rl7+@G0Dh*_w8C-9Bau!rw%!F=5Q5jxvY_;M98YP8;j)A(S>S7;Ph z+^HRwbVRd3BPBO+vTY3sbM*)+CV0PgeMtR{So}*dOznY;wDSv_xHZMu7A?; zGu)FZ3R^uNg4(ctSka_uoJR3tm245I_N#bo4aIyPobUor9`44kA-Gw{IV4!8uoWe^ zeAcB)z^m@)B8K|)rfmlVVZC#I5#FB+2K)`R!J3Q5z`DKd`cNsQ&SDH`ZjtFRI+24Ml(FlJ@GewP^ zLM)3dCtzEC)MTq8U~v1fXuU9x+-RYl_O?HPCR=c@9KN*eTWzFH>8j3N$gPCyyS)(S zj@*`+If{{=N_;wwn_oek*{v5ld|k0rjZYs%kdMalQ+NkETzR=Fg>SH;S$=dc=%6KnzauI~JX z;XVRYTJ>+9KFuYkT3!41L6~kpb;`SQm~ZjO93?>n-0f*PtC)sC$I4kpI=w;z9)RZ3 z6JtIIX26N?c$X7`5wS?LVw)&fs3hlFN@5}{oG!%6)1&|QQRU_nHW16-OOQ8wc5YsR z*U~lV=SjeOCl5^Y3yTNS?=G>l88|f=ZQee3%YnYn0RWRs3=Z1r2T@zh&nQ|GzxP}| z@iAL8Od%yzBda^{c_8x=DiPyRe};V#_5LE(b>G}YztrzT<7#P>yzh^Yq-^&@W3xYu z@Gkf;gL53>o!5vT!R%(qwqGr^FdRX-me3tM371i7=njlwAZ%7}ukFuV06i>r?6P)0 zSR#YY`v-0aYv{YpcjE~`z2?8)w!@kGK%T$L%)i+LoLbS)*0Dlsk9+KDTvmN-Z zQh`W(HH6QrzX3Se@a7NK$1z@saruJ=w763LYP9>1m5e(0dBtxfzGff{RlAHq=d&z3 z#r_V<_-ST;CA@-~zBjutWd?oN3qvnf&gS9cVqOUw5%aY8l@$NE@6#tJm>`zs=t%Js zz?Pc8iIjdDe%#-g)Zkw}$m0by@H*RudBTvXvaA+#805y#?z)FbUJfH< zw(1GzL)6V?*h7Y?dYmit6y|w&4q~o)V845e0t?i40Za-?fph@5;y4C=R$(BQ zWf|T|2*CfNsNK`~<1AydMFQU>leH7Kn;8i64i=iuC3PGk_M>WwrXsh6LXeAp8oNhc zF_khDV{*>F#IfyXnFSBMGagj;udfeoorG7j<|yIv?)7)AQY|{3-zI8Qbd`FSzoVNZ zewl^@dos`zrHYKGZ-|yY$8DjjAa&`B7Wsa)c9^mKQ`BR<@QZ;u-$b2_>AlXcy?~T2 zu;El3&M z$gxRtyR?=2Wm6l1nk2Tk>%q=n!h^@r`s^rSk>H4TL~bQ3gUI$*G)omDhszVor(PmW zIsE$Sb&YX6O_|`wgK55tg&P``RyM`6GlY0M(}Yd2+Q(iJdZQ^70ZZQt#9Skv7|x zH;vn={*$fCC(!h>ug9PH<1UN*@fj1V<@gv)(rU*sXwJD>CiiV8D#2SMmCpe3o)$ez zH_5x=5mS-$(ezGjCV8;L;t;G7O2lZ_A$egGmR2}m+$Ohb=Sizi))Fdjo-*8!LEQry zXY?QH?Jtc93D^9DbajlZmq@ilcTYRDp$LB?dq||ccog*w0f?w&*5UFe%sn9ZAzgTV z_)(o~Y0#qQzyu*Y6NqRisqmt5kwC~+Z&|2g*T&X@Mr_;N`W+>`QacL9Ax-bcqhWSG zty#-%M@ALGy#UqZVmt-|7nXeOL_>{J*M{D&(~o^WpeW=Jg~bdq~D<)azK; zT6a);WDccqK{DpRt&b?@;Dzzq8~Rh8aEY`$bU{7t zeCO{&C}u1LP-r+ZkaCvJw$R1J*qJ5^b5`ptNDhM)VQP-rrSgT3AGRknH zOIc6gLoSJOwozr+CDdTZ;Lp-wx9PVkx@*}dq7cERik8-=m93F<5*JEdcaT!Pa5t#a zzCjIm34XMd;{@cSzxe0hi>n+k_7*ryL+ld8T=sMdMYWXyDSweUP1CjSe|hhRs+*~1 z)M2$p1E(KerA-f8nFLN3X)%}*h&(!1hl9M+grvgeK`AHSUA3w)p8Kv}Z%`uy5JpT} z#~|)HO@AQhn=Z8unD2N;ugv)OyAXID_)>7p>S6hDdxl^%=_JM1$Pj0}(_TP!I+U5Z zZwiwaM>#{I^M0s(oOtwQNp%lwGw_v-8})6`V)TX$13Sx%Q{-;EOI+!zkKU4z6hnT0 zVu~|+0+LkoAhZ_W*v3%^XV%~gi|NLQ+M>??0k}X%zn)msUFPGlxxM)H+S=lA;fKDG zERRKBRlMs44%L7xXoEJWO>)<*jLZ6j!}V2Gxu0RBKL3G^U(!v+)^3n-gIA_lEt>3= zOOBi^hhhM2Iv)3=E_1PFTMM8UnTGx87$`vMk`0}0-%}$Kmv@pWtJ~wcC}OY;a3OQn zx5X@{R{GfUoONw6(1R(hj+J!Xw&wBzW>-DCw6m;5tgut6u^7ExHf(P$PcQBQOzo6k z3%lvq=NdyEm5PdcF&>Scu(S(i-8Lfxi&sU7PcveaepD3H?Z6t|psM6JnNQ=X!xFeX zu!~)X=v0ro_D2L(sKft8Rpvg;3FY|HdQvMqZY`Va!{1&R+-aM9`p_kEWbaMLXD8W- zV?%YdgDubd8F&=&`K29Y8y=-nFDXWNBt?z>=Z1)o{~gV6-gJ^xTXBq7$t_8uB5Std z8iUTHHXO3S{V>s-~N@82TdIcb6Q{jYl84`0i+*rSE?z8o`- z9-ZG=Ht#@x#eWczF2Uo69F$qJjpwZYBM$uAzPpJoAFu^8mQn; zcNehnhnx-Do9cBjbyXXAdQk^aJNlxX{@FgSokbg4$`cDZ$ZN3M0e{+VMxj2l6x1?i z_KG6vLboAmZL^r6@05P``xiQ-*`MNyI^wI0>2*17Ho{A5ETOGHXpsMaQ^BagOmIT)VZYJhP~+ytuRt zKF2T&p>ovdf2UMbe3g{-hP#@wDzhg-`cbM5tzHA=yy+BM?x8S_Q~EI8sPWvAr*8h| zGTLfA`G)?Y-qcoXX)3cdf2M1`>n&dk+u)JwR(5UB41U*aZ6?!KTfSGvGX2}-n7~&7*|t0?i((n z+%It2zUiBt>bp-!%*TGIp-eW4+T(Dau`*kIYbV!8Qvlw$M$(vp#tbxOpfLmG4BT_? zJ@TnfeOmbVCLf1qzj?|jC(FI}+$|kC;5kA2OfVzhEfxpOBB z#MMdj=FR1%8?KX2T<{U-^3#AO_<~?#LgGMF3=wILd&%i5 zYvkPryDx17`V;yd=uR3KZ9^>R7B!pRey0_hp*=X}*@urseLQ^A6 z%?b3P7^uK?>Y(-VpD*{-FR4A-;+b1kXhB>%0OdSHy}5xVcL)@;qCNJDYWaL*Yj_W% z6n09PW6BdX81B`ibdUCcfJeRZr0%ao^LpxMc41$N7kE4_d}|zUKZ6->@&_06lItgq z#G|y$@B%1n^l1ZD$`!{=(|iYRguD@Mp*&N85HYxaL}p=1^_rT2BFI9<>zNicI6uRy5(naG_=h%Mh<)H4H<(8KR%F`d7AWhI7 zYA@K>MlOA9v<}>H<_b@pQwOfa%)6Po)}x+7ex0g!Y|`P2!|1rHWk!@VXGZC%<0q8R~bn8s18NTc446Pii^H1Q~Ed(tsz(a zIB*DL_Teq4+p!ilkxrv|KL-80-a_CUYpsL7SDv&$AK?#lMr))S`+m$uTZA?2czyAB z|7vnCd2x9=Id8;T`T3+iL@WRGu30j=)mE7Yf0sNw1OqiKv>)U3b^vC${l^*e@W?gS z78MM3wUST#b-4WG?elcGfWWn~u7%34K5e0#Ja7XBJDV$i>2XWt(RrO@^2&DF1`hJQ zbND*>+VM+M`Zr==^qY?lkQcGO86h{2?u?a(itx=0S}fk)o5JS*VXgj{(tk%3Ui zs8(Z>tEoB!<-})3u}h=MLSGTjX{aUYJ4_Nw$el5^5RU4;Lq2`@YPsRLS{?B0(;j_2 zs<&gH`700imzS1f9YVB`=lucQcE}AFc64yxTI|M{+k zDtl@{d-=iBy>-BppHDolhxgek*Nj^tZChC1bm!!*7{qKZ_kVZ}LKc|lfE=%@o1X6> zznjtp*VSGfwA1VAxK;Ab5t~#gW&lQ7$`oEb&*8nAnk30}hu4PXH)z0%v_@jXP8+SQ zC?B6vJ($3oPMc6S2kxe1Rn(}MPE2&?^7IR+6?qJ>3~_YaLB#b8DYIxTxI<9iG=YSVt)lY*c6nZ`AntAFslTphhF`_g*3{~r(PAlh$#bGv-t{pZWp zx;nY^tC!3F`uxRm<{76;?_RagV!@U>`XOW=EiNU2JM*Wyy02o}n2=hTFj)HHM}qlK ziv5CGy&1YaNFY#(c;Q(oH;R)^Y3a32C{dZ?RCSZ^<4U!8GsJ&}OAUMj?G&#%?#z`O z-fzRaTOlTbLg1{ zx29yZNtKPs^VI@DbSBIpwQ%&98?+#r(HBZ}5Jj_FhS`K3E4scq+e&UL*^8IH= zWsIe)}b%wXD}FT6;0>PO`1e}7tj{?cH%Yg#`HSUshKQT$TQaw{GWy?aI023^30x?<}{!+Fw3% z*aCU)@a3`#Uo?L|y`TIB18Yb2+Mt+-_xcxx>LA)zkDiP50N3amgukBJADH?ojyPvI zW9pJeM@u(6y7i+|r%7)NAU%V@x2whvm!GyCEEgUzU*qcGi3ECH^;-dDdY%S_`&6%? zcz|5y6_L;qg>Kd1uXGfAuIMpx5^4d&ud{UFsVERh4)Xo+ zL*?y5*T}~Y=HUMx%zo-3-=8o<2aUgS?7}>1+UJa=Z#^|s=3&s_rngU*F_>*TXMGzC zYz~vJJ~~W(e%1`E4~4ZNF$;}@Xjh#y7qcAKVFp!GJR;m*uAMMYj_I=j$Hc!_mhwx` zY=2RrIvN!HLgYR52h2B0MYF+&#Axt^)bd@`pSePH|7FS>TZ5T=TvzRv=gg9Uopwko%#2!%8GIK#I0X6GBiEcZUj}vFju~Zb z<%%Z;>e2u=zhjPGmkuu-_R7cTZV_0nL5#=jq+$)|YFgZ^vC2 zs9vpje}&o7d_?uP@0lrsyX?>(hAw@0u>4?rU&!znw|ca8R~{!+dP&Hf&-GEKi9waC z6_<|*qTdSO(f>re(q#n`4(hOt)iluN{hxX9&ln6@25dei&TqCpY6U(1Utzl?k7mA)or&K>RRKFF!qdkq(3s zAJ;m&>WMybWZycu1naMLXtfXPskW1EKGs+6c%_?s?kF4sjkx~}>nAL3tAl7)ow`iU z7`jA9}8fzLelttcUhd z44S=f%qBVI)BrA>mB;3_m5-0*vIx*ce#tk-`EmLXEd9Yi+@y4X zQC@4X8|@^usxmdbs44m@RmZhq_C}%Q>WLZx$UR=dfgQ-Kf5mQ+8;Iicf*t?+JwV_! zs#OZa!+JEzJ)XzK@A1qHeqqK8i#>a|fO2DL%s^uX8Z*$Cfjk54+O?CpbLU9cu2_hb z)0j}sKmUBW>8C%FXC_QYA0)|!ALuAAzcLv&BTeMYw~ymzkhE*xPJVjh^>XDEU)RAn ze$H4Q9y?}dANbtohXLK=RtbH#N|c8@q@Dd)$H;nKM32QD?9Dp>^ObiN2FHIa0}OAU8Uzadqw2vfr>``sXei&Dz`bVXalrRrN597pD9xSeK5+? zj{6G2yY_N>N`$c|PMfrzB{kgo)wptmmGU z@NXO(>H#Hng(@0?Lgi1h`bj^$UGlP{X6P41)|`bwwWV9y$gidjk}r;(Bdzi1EG>!u z3*e9GjWui_7~W-zTz%53@}7H-mq!-%k`Eua05htZU}i`=dGClN`dAi`jO?~W?tS+& zczhJuDIhoq2oi2@QRoD%YLKDzw0eEnE{Ij2`^&=@=hcmMmI*B2dx z-USCQmOo-}?$y;DKG{PwlJnB}$r?F0htAb}(D8dR~-$`i)jR0`Zx)p2qp z>PG4YoALon9*dnUUKYmiL9k!E=HWH0Vb{6N2C7!pwZsb~x@ulNJ9aT*o5QHAe~w; zR9kKkD9IsZj)Nc8b-R4&=;bNSC&#V??$+|h*Sg7v4#EsUlv!&zXtYZQ(U>m|%KYLT z^Dt}IKEEil64Y5G4j}1ds%WQ*6GdH8hu1itGZWucQp7cBw8Gk zoENn!$A<2Pb;Ys7&J296pFX5U=4@!CFA}=uj3rp-ZYz}Vmh+z5#`n?j8< z`dIk6)(rQ?2Db(?INM<`xvAWG-g2w~jIkZ49oBn?Tye@8`QqPt$>bHSMKcwOxk0$AfgSNp>TlvJ{8(k{$IQZiSZo}Hc><6HS15~EwGNnmtH9YnzC{DK2 zKf!?KUR+KEg?GwfTi6@Ez-iR$p+r49XJD6j6(+poSW>;Q*s68nBCZNb09_Rd&DdG- zWdnRn>A$ON-nLujt=cMG+qaPZ-R#$p|E}1LaT+twn1RL&yx9!2Y}rz3Yinim=FRfV z_?KkGisiC#<3<@j{uvoEWU%$}xl?=!@ZVWyo*_TH=0|em(I?0S7korcJ^5{N_~D02 z&)S}_fd+eNE!as6_bK#BWw}p}Bu+z;+iA7k3{h+!H~gTZnN#emZ*yk*A@eG&4_S&A z@mZVUF;s41H5xa?i}?I(Lp_0oZ_>molku(x;*q5-9_QSH`>h>&>hVP>q+#bY@Hpi< zXn&hhkHIthh};N%(R%xEzT)lmGOT+gCih!~tAJ)zp%-xjN7g=K$DbXCpplQ)IUef5 z{Wzg_W`Yh)MBEew&)QHnJv9lLSCQB+=LHVljPVypE?jH35vKfur~AqR+=S=lx9^#b zHPDLth0pwbxb(o|Ka4#WUqJcgYXsKW`Q{1pEHYi0JKZVH!ynYD6w?K<0cLC-cjq))qTG7Ez-Oixoh-bjCc zoY4=DI5o#x57)?Hn7zhzkc#%ZGbQeb-&(kvsH#V^3;b+HsknU{8)p;X#Ct+C|8$Tl z+*|M^=U}Q5sS4Jgu)1D8JbJNwaMTjK{jsY|Ueyun!u61!;8D%rVsP!oag(J#1}fu* zo5qY3Gf0-;u{AzY%XQZD=%Ui=eQUy<=_CBi(59QHub9J1mS3 zE@*z73C0sgWtK#ZRdBxhjmHo0JtLgi`u&q<%P9lVfxG#PwfHiJH+k}U2^qDyM&s?1 z`(U~#q+5PQCh15^Rr z)eGX+L591i!(R0TYq8u+2W$!4@rJkLLvupTaD+R2#YTH;FcX=dCs@kKyaASseIPIxq_=f8O8HUuf`h**3JXe>3QLGq39;>9JvVt(0m-dy8P^g zZV=MGvd(13Vxa4empjV^f9@;ijM$8PZO2QXc3_4y4dQwzY~wm7Zmwh2ufuLU-prH1 zba+l2(rde3i(LQgXbxz4-AD}Rg*`p9oOQq!x#SC)}0sM{IA@wNjVapQUm+;M_c9logW^C;pn zuUZl{RtM73#Tvo?h4hgrtL6Ip=16oO;xp>`N2nq z`!Q-b7=-=2(iD$7kSXu`V2XJB_yH1)4>?rS12?)yiW2rN=S`0yRvtdI9+If14fjiS z!J|U;g3s{4U_?(YLM$?G%|1D>H}Cqvs2|#K2UL0AHwJQ%A-rcn;ViTdjKvq%{-Td{ znv_A=Ui{LI0VPrc9PMTvu(&ncvK^tV$H|mo?lG0ezJ5A0@9GRTR&V*B+`UjtkNd%I z`8W5)@d)1XLl0KqXFoWsQ62`2!h#WD4CsWnZlc|*uvl;(d=XMS@bk-e&cooE0+hea z>>@v#*jw7-SJAL#%)y;^=$9()A68qZ>qi}0yIE&0P)23T0cSCAjUBVj^RP#AMLjzP zW@PzFhfvKR^SniVEkDREa9Vzx>hX-MBP|BSrR1RvpKM-1-46unPpQ{8WDi;4m#7)9 zB0k$sy+p~OIKK;k+&wugM@nUPp~43Bu7 zG_7Q9UM%A)>TJNE2i8=gME^T3z7=m52EW!}c397LmNyQ3@tmPzoMC*iz&D;8B~w>- zz!&@7$nh@BvM8SNpgteVW#KVQ_MUG(PQ_x^M-A33v>%x{`z$YOajww!Mywx2C)(dC zr7`|H?t*%I+=;?Y4h#GtD(Y=7u2=eI3EXjBflz@R((18O=d4byK5@Vsq9LkCh5D_dHfC%eS;dKa>xond(x!B!cC>**%n2Zg%K^ zS`B8Y)yXgLnCBny_Tf9H_QIPh`)MC{!6BSUxXO}%W-}LTMqncpUj0#wCl@t&|KL(f zs5(A3{WBK6ETUtp?ee|@*WkL?BY&OMMShF{H4a!GfTaq)e9SUA1he9*f~qw(0Y>>~ zNl6%*rkgr)x7CC}znu8OCIx87CuDK`B z{kK{8z8nYG-07AB^C?b6T-tLX`s*gtVok127?`Zb45!^XNJnGNOk`r>SgAvm#GX-V zjq0&OZh7Y-`NL~nd2YVe2mW;a_IU&mNpFUpu$_E7a?ucy`u0x%r_5n8sKm zf4=l6>DUH06b+QSo>?M)ettO?l&+CSzIA*98gE_n*GVhn_9qspgRz6LenaQ-ul8@< z^Z$94PflBlLBsWO?vcIafL?sVRAXt(!2hfa+9?b9Jo342%V(nXHIVA`F(zD;NTPMkPNzVp4S<+Go=NDde_Oiq5=@vaqp zui51s6?iinTm3{S#Uf+%xmZPN$qzPY0FzSe7nGh$(bQGQl)|n;!{VthiyKM`0*vOs zkQMQqSD%4P4SYj8?sB6$Di6^C7)WDzc?ky6deLiTsh4|R-HV$I{CG0N>N?;%@=3#L zrIL6LUynt`^8gf*%dflZ2|L3e}A_w*2Ivz>tFhiGm*}m%eM#ao! zw7cSXpk8OFxOaD~r0|G6{ZC18Tr@%`1yhNG^(kecZ21_DM`u_bp4U-6cF5{X1)b^?=eNfNNM5Ev=iTB zIE!E}W@s(czb(63$U_Ty$}e6SDp!m@P=0pCONuicXm5vp*#w4}>WwuQj#_ zvk;Y!7u~o-zic7v%A5NLQ=;C+@Dul?5x1cmYR)8EfX5vN;N0QBmHNY$*Amw;`_s%m zc(e&K^|U?KY~l?)(jmw%%VNw-8`FJbmWCRDFSrUioVbHk*;Slqd- z{MXqpW7a?&e4qw^PrvI}vj96BGueoXGtZ8~I&O4mYV~+aC>Ot1iPBM+8MPaOtiO5J zOL`nsSh)#2bi(3!E>p_2wtC~|YtdjFQOE5KGg}_$QVEBZG|q@B1G7#z18>m6IR}q> z_!?xN!{eOe@hIjkFZGuzPME7F4j;+w+hK=r7NLTp3qLWIY_|LGAy}KMp)v?>0cH7* zaU4wM?93dmI?{S>IGA=N{wSS>x4YixE((j*q7Lt5oY~+FhcI&l(S4xWIzH4+C(2o6%XmiAO=l4cUZ8fg!~IuVQ^Q&QeN78s+si z^)eEJdsm&ZOv6lD)mr}R}7N?1IySTuW7HX;ly5sY1 zcag8~k=YH#nrP22=KTX4XE?XxTI7qRSfA8s&-js7ui)U?8N=#)owN<|*xa@U8`VwE|1RxX$VX1>FVA3L?5&hNm>%%pWc)hRK-sx_U#e?i zuViXK>#d78vo6XbuNM0u`@f0KShhv}@$xEJxv}BtssA^%wXrRY8E7a2lP6D>wr$(W zhd%TnY1sN8+)2I23c#L5(lZR_M=!dn@#`J9>zZ}0CgIf4_HrE~#>o?4* zmmfZ~59^{Nt5jz@7Ef_$AjHxNi#4`iz06)+BhyycFSM=#of?_2psB3hf=9jjaW)C9 zC{^OTOVr5fO+f=#Ak>)^w(i{GquQONwudIWP%fY@iUpuqvF|>( zUdl23x5!;ny2;Yb7$go~u;ITMc${Oxk`8iQf6j{Hzj6tQQrf73OR;|tW`}UV=&!SC zMd!0o!T)w+T{W&7)fsP+9E1V4@U1?uAs_ozB_LVd5J_T=2YwH*UkL-2EO!lTSth24rM8qSy zZjwnWyJAu27Ut8nA6rzbvzLZ+#sG}|BU%i2a)ueG;sB-3)mvL(y)+Kgh-}!|QXa*6 zX{)z!O_~~=A;m@b-`0PPEZR&gg{}E00Go(u57p^q*5I8}FLRn*+ur z*lJ28%BjFzGji7S&6Hc^FMQ-+x$V_K#xtlz)d9;@m`OIW>t>ar0<&ZA3>yj`y`(X< z`DVhdn9cOqLe5lTJj&6>lFXMy!$t=XKNd%QQEr8-c-}iOW0Uda+{|c8@LSs_T6X_JMnqTcvJC;gNHeCl~Pv zmB@6B=JLKdJv85}e{n@ed2T5lfddvo;kuBQ9J7#Bc;lp9Q^aZTgEr7CTI+vq=&O># zp~s+a)M_cH?WvtwlQ&MOR)cqgf|@M9hO{kIU_2#8N8^#cWBP59yI$*wzQ2{#F$Q(; zZ~EGH==(c*-%oQ&3!L1Kzma%5=PJB;b1K$Cqn7DnfpV;!#xK7kyW8W{eKBalqHp1( zy~Yww%uaBb+;NEe3ARf_idcCtcUAHD*Zj3-_b*JLya;>a@Uft`ewpN&eKjg(_=If zXk4!$(KtCZp~a^Y-Gi^+Pwggu!JyZYN5iB;28H@8N~z}z^~eM*D^jVts4m1| zlPFbl-^Qv-4HI>t7jdaag^U`f?qb)N8kw}TjXbx6KQdar_=xDGC)jw&!VNs9?FkFM zk#pmAULWS0l#gnzs%s`wRssKr zwZY3PTFP1A)r2bC3#uGzC=KdyNl++dNmK!VcOV5dEvtq%bC*$*8BVk3zWQ}q@D`6g zsFA2~6W2 z7naNWjxA4pr>`fcuaT`g?AiF-_vD3}6aya#&Esy{xmTtw-7FnjH?lwq~)WN^>6rMO|V0pI*y1vWlL z!&zT_yJ8we17+Fz?J{-oX6fFcrHt;^0dLB&UvxrsCI-E>?ARlR4(y~e_Ch{qMV)NK zgY#qhca)Z}S-okeELyuw24Ic44w%t54UfRC*sw!})V7t8ecFeaOv2!244}>{nkW&6+jS7ZC6R%?frB6GEq~(w{#3_ZRgY zR@HXw$JwLO+J2vOK9g&$y5~lB3R)>o@etqDEAga&j#T5OcoDzAgJ4;_i0?8DlGjK3 zO?KV8Tgb=m*eMs?y-UtHsHq$`m^XFx^4$D-xogs18Qr@^PCFnnKpPe!=7%2J5b>NU zvHkPL?3G7nVKMhVHI*3I9*cO@u@A1Q4D@~@^03TVYJTK_^!lPHp=kCMXpY9!s zM=e%k@!_o)@Trk0SWNiNDLwR&j*E`wFUgLJMqX5waD;|asJF!IsLvd>SbjQbpj`Us zNX!&kjbEU*>cZ(9T${S4gIs>>ENyzU&BAB{EsD2Cl~m$5l*B$L|8wd&x%P$On6|&WPU*iYnwdd2ylKis z-aJ+%9y<)cJ4`KZXDvzj3$~iNBmIe(4l3*uZ{Dl735ZqUICEXHbB3+ZMg6zq5v5Zw zSjgF9i#OrC#rkL)ceK!%X*?U*1030VJqFzd%1tj0k#Tq#!4a4l#u+4+95ow{W{#Gt zo*99+cCN%1@TPLltlqld|F=(=iDsL9VA3?er=_PJj}mT@`{vXFM{j)$cP?h9acwmY zAhW1F@^%$`efohzJ)B6m#buX@bO8ADd$IQNu`A03T1Q!d6^ z6SrdKT^sr3D+8n!?R&>ipFITMX$B0yD4w8bZ?rpbHcZ`)IOz|6TKszfzaNBSJNzj? z`E-Dh*%!xIc$<80OQf_!*fNa^-6zkG$#(-)wT@Q@| zly|**3QYJG2(88l@-GD5xPyb}3EF zFSInzK_T)fP|=?yJ8xL@V`WNLWzB*w$Bzs9!XuZ;=l*ekT=16>a>1c1^lhdbD4Bu* zu{)>q)VKIubkq`S4_uep!&rWPynX0;xqnU<`TApnQttZXO$`EoBAhq^KN zL50ZLVzwLK1p3h2&UiV;ZaK2g7Ja+tJu_?x9{MNdXa<*C_{zhBt{b;Nb$FKleRfwoR^3Lva_mxlDFO%d{xGc@)=Ar;f9lKgTk5|EC_XPDAZ097 zG#xOj4~4LEX*F@X;>t1=KfWnYuBN!`ys-R25Z9pxcyw+DX@~XUILmo12D>J&Y^(Hi z%o_X3@yp=Wu75thd;Wpze^;};Q-f`yJK=U^ph>+uJ_K>d^7w}cr)iMlX~FM z%Pv@lZkxWz`;gjga_{uc@{2T$@e54yX2#w2lv8ysaPBBOssEq|E%_M`NKW1rr8D=)@_%(x}uf* z=cLX$%j{qbVk)eF#*mIP6UhO1amCQCyYvm9eDmjsZnR^*pr7|owD&YtQ>&=zA0u9( z)_!L_4v%4O&(!1ybKdQZU|Y7+>zt->-8tUu1fsLz1c`>I0^|iSFyxIC3zLO{$^RwW zqla~on;)Jp_r0`2&N#TIEXTmje?7iXP9BZtY2HFP{oo$L*xTac^dnYcJr|_uak3V{l95&c~0c711)7Hp!f1M*AKV`r_9Y}llRU1J+`M~bD znXQ*UOjs%$TpQW97=WVBKVy*Z=7;7BXU;_#(Wjk!?*n+HIbO8D!Vd6zf1W9iPc7aE zMfsVB_LMK5GYmIjJkogmFb2UczGJG)TWP#g2I@fI!qqnAo%-~5;aT=v)K`D= zS~_=r{nTQ0_q`8{`rp3hT;Ob^jT!hqF#{j`;0I;ev}tnVjW^0KesPO*?ATF$bltV` z@FS1N%$YNzzb>HTlbU_?rI%bJ+qQ3$|G4pIa^-h^pl|RzaP%m7`iTc+K>vQwhmv*= zxX$fT*Xyn7h5`QAMXNmS0xImx4?e2gj}gPcAo^v9G}&jR-{aVZ7gv zIG^m)nQECbE%mbml;RW*)yg{_H__lKS5<7#Lh#y7>{xrr>z>liZJNk`oZmujd1jB? z{qi2U?ODbVX@f5H-^MnPFPzv^Z!~D-jRU0z7Es+m;~0pISKjt}w6Bq0zHg6Q^&sAM z^n6qN5TcdvT`$KE+9y|>u}ip;fN2-_RO0P6e}(^|-HZhr5XOD;#{)L_XZe8#g>J`- z9@!Laocia!33bg8jYl40t*0j|k&i*0cFAwvGgp4_bRXR8*6No>)rY|;gEq;#IX_Q0}nW^>XQvGjVQmb_NF0Kz}~&Wx4wKk#fo7 zhiV(!wb(6Z4_SeMwYl0(1yi*FB0T`4URgy$P-P4O>mtJ$8>(83O|BN|N>QOF7!~$c zD(b*qeB>NF4!Be9m^wiIHoLeV>W%#0c>1e&6c6j~c!S3D*evG_UoOus?IQoc%&zet zehOa*MNaO&R=#!IEFF-$_|Y-?C2=&~bb9T{ujw04(}m}-?7G3Z<9IK`8$LH+(C)@b zL-Y$GkM9*H%#yD@Hd-z0dgt-qBX{C)&DSt^JY`h}9qc@Lz&fmF*DKS*8}0D;)Q?V^ zCf7YTM1Jw|Kz#9SuKn6Lygl@yBj;Ib03!9bv_>w{;q$!e_1}=(qQS%zBdbBRnl7!O zkv>0n3WvNFQ)L5tc6GToI;Xm##}0j09dG&$9j>jl8#9ZR%P(K*C*zlNkrVo_7tUz< z@wl1t^B4Qecb^%G>!n5?E4=vVg>vo)e?mlk(2p1QdC^e|abL}WmmcbqwTIwQ&7Yk) z1B;&9g{s#*)!~Il!_aM^jU7c3qgEOo9l|q1^Xj! zph*KKk0ok!FwKosKjrsz`Z3zUI!~viHSfpD=?)a^{|4ca%U_>2L$1aP25z3zr_}eK zJZPhQ`RJv(Hj?2XdIKGJjf$>|`J-|3<=P4TklWrE#BQc*%bh)Jy67hY`PiYWFypheeB-f!ifth9UUvLaIllkq;#AgN0 zdpxH3OAM5;9NKG}{QMmYF(a3gPR#L{2W-O3zE<+9m%Gc;3wO#d-?b2r^KHV+*p_%p zY8SZ&18Q92;lg89$qP%{VI~>B%i6g|pHbo*DR_|&9lXJ=tEpY&))%|mb%nv1U%Yd% z)MD+g9B-thOySk@YiI*?POT=@>kjc*o`ZUGv(uq$XSmsl^6?43S`Q}jTcH-sb~h!f zqE5whQi$Auj^#s#Y(l7B?NfBH?!(|uJ*d0R61xS1 zW^C)X&mXCIq`!-PHKoCUw9D_BRvJj-8gCp(7D zcG6y1wsfiV?%fNs%KREg{U-ZQ{AoAp`zm2qEV69ba_QWugS2jA3xeAhf2wD8{17Vn zql5588ocf(TUBoWlAR4CkTo2Mhn{|n?KhssXsBLs28M=k744{w#}6z@9Z$)BVF$$4 zzM!f{najcRhlts$_URy6KdjqIzm@s3b~BpCg`IwoLZXy@8)(lyIG?+=1~;VjGN|i5 zd<^+hl*4%J$}JjgvW>mW6dHC@WgR=^ZB4l`ekwd z#_VTn5qKuNwenBw^J>Czo{tFf5j_rc4eW?VI(bL9pTb45Ig70$2HdJk9zb-qRVzF$ z$iMv+&Me_0quuc`g9IqrpS#QO2JV73q*!>1*r8SMc1DM$-mpexh9*Q|Ur@6*JDj(1 z;|c{H_R@{D^UJWH{2DyYIS}VoCk$HI^`<>T(HsP1bGhD}F8uF&#NCQ@*Z4+AdowVv zmxk>_Kc_q3a|<3h2Ie%=klY_67^QJKQzc)h(1G@D>3=DKJ$I7taha%3#F zb(k5*H6!^lg37`d37q@8(ekagEt0b^7{|VGGxWRS2OTZBJIBG7=B&YS?S?_orWiO< zAllys(4dvk8sZxWkf}v&h^qPG`lU{TmlxnPD5pODydr|KxTFMED{F1L#ozP?4d*6rT}-nCniua^6~AEJGA;e8ut_fez{X zHSko@2p;nP7kG6>S`&R;LwNnyQqD^I)0dCb#|k^O#oLE`X4YGY+7dq_j>D`p4hr!t znw|V{L(X{PY%R{};y_wnjvCqpGqsxOx@;GoHB>tYKGylt{0(?ivi<80qR|f*-QR>* z?>gLmJsg3@AZt6fl*chh_Ib<>V_cTMOj@qT@jLGuDgE*IBnuzwD>*KWJ- zrIpgVODp--dq(IZm(&|pJ6vDFz{fpVxJDUgyq$spv`=8h9Lqoq(DmukT0U~qiyDgq zl5eG)fR{dSaE^tKZ1QcW4`9%2HXa4#K-$0_ZPLpeIDkgJu{37je`*H!7<~T$cD?8Y z&P3=XkKULaG8qmgm=F{x4oJ{3ZYx-YfBLyViSm%cuWE3U z#Zv`>ae2h&XJQdAEa2JYm3D>`aZ#_gSNugN#{w zctno&z2es)zaW?H^Lir_YtGEm`+28Dx7Tz&k205`zE(oqI_*PY)*3e=_gZNml0=F> z)1__!IYjXB@}6v-Mo6K&sTjVJICvjO!q*n76z+bd6^+ikg&N@^fs{Sktf0d-yL01jO^PO2n{i(@xM;3;RY?Knemvohz6UfM#3 z$>5$nfw}yyDD_0O7>&$>SS7%e|NV)zZgk9`Y@>EDtl%xY>>VaF7W2M=HVd~#?z|UW z)|wVydTrL(yALj*dS9b|EuCS_oOF_Uiq~%QIhrOuh5Pi=1vEyYzh>-clG0yg$Z`TL zzR#kdm3~nR+)a3F`9c2_3pzDOdOWJXjcD#yIP(G>35@h=3RQFV3l%OAs0!pBlzM1 zoPeLjbk}F_xHQ$S;!p{`rR%8}CZ-v;gHSNP)1Ch+!v~W!W2hE&O1k}cE2k_*r_w)M zjAL1^XzSt>v}+h^A?U@)fUQ+kL1}%5fpdy*AE9xA7pHs!CKW;XgEM0(v_G|r>#adb zSZw;o|59Q5JpJ^_m(Qj{l<}eQ=UmZ>I5WY8-qSeD)r9nMF?HD=6MAW-D8UUpO%?@Q z!1QiVhV7DsmwMzUe?L@a%kLUKOYCGYoUG=Kv?|nFn zelSGaMVF{$Z&5g1bswJ&Cl8yHy+(#oP%1Qw&+hzJk7t>(hTCEpC^w? zHk4@}&>a&FxXrE`Jy^{~o_A2M$?qw^lML?V(e8%BqkWx8fi*1OP5w9wlilhc>r8eE zhnjGN{f}KURr=bq=-XEga*Q-HMFYNM zUT^z_hG8KFhcPI=Gf5)!ZQu3&$>G1tVXFyj=Zms(5p|z^ zCs^`yJ~6HmHCmdL8Al1iRbm^z>M*o{rc34>XUG0@n{ z<4&evZBJ~qvhb|?@hb+QelT1V-AxaE{Q%2!Zz5hT&ykWypR)42|KUBPB3E%*$!vr1 zv?ZL%K|TI!az2z8vkzJ%nBj=-0ngRmzj3}tBbjoj@P$cg%qKY1&m4k;@p-*$j^Iii zNk{Wxh@r=svE;Dyej+8IgW}A=L8OG?Jby1l|2hSgV4Q#o+e}*UTHK*n-mNj23lw}` zpf|4Udl*g-Mmj?n8l6uQClORI^H9rYMqEZD4WBqXdeHvdY*d;^vKqh94O=w8h;4rM-GBEv&7Yc-E6n#R)Au}Cg_(!A9qhWM%Z=+ zl=4XP?$E$q7pp9bVpYahwVglJ;@}MB6Ud{7?`T9r{RD_p_iZVwNPp}vVjRfoWO0UL zt@G;npq_jOy@KO)%7gC$7>u?@>Dy}5cny=O>$!f>5)CWK-({ui#~X|+|2o|5@**e< z6SKid_tj=}$x{6&@{UwcbWwjrba$~QOy4<+Ym56*oh3R}8+(YAdjZk0jjI2tcmE_^ zFU=ToVC*kuIC3B+Y#k^vXJK-r@59rJA4h%Q9IbnM#0Qc!`rzov>BbEf=YqAc1sBn# z|3T5BZIxE{3ZaIUR_oraP2Dwp@ue;GKCbTwH{8dZvZ#NVW^Bc!+Mqi3@=+NnYDW{?7JCO9mm9eGddBvYA-KgysuZo?$gmRxZMNxuxJr$UF40lg02oUJFhKvufuM76pGDP<}8fV4Z-9E5pmjs^M$_r8X0+|nIQ4No6SwJQ$CNptU( z{Drw|9JS}OhbTUza2U*-L%G`$s5vVWZ{R>~ihz+SUUJEsFq?WaY}_~rYDQWabqMJ8 zL2_kX_Y==Fa4mPj`BZAf^#&SRTR+U20VUPIW72LHDQ|aG3?sj@%MZ4S+FC=%kl;w{ zdt-9yhAeeIOx6={JyeQ@D7OvsHb8rDM?xADwZ_6VWRIsa@~avCqkFF9f? z?budpbJp<7?b@v+ZTzMG-6)<7Y|yCl`Mgn&=LTI?c(^QLekt6`>(ej?3DhGp8i^_N z^D6dq(%=rps#N#={zA_L)NWoq=V;Z4FEiQ~Q!=zf<+lnx5fH${Z^qHe<5Q9J)5SQ= zWZ#n#uJFhOXr+k~nTNPvjNp`Q3N6utN4sH1bh2Q(%ftt5&$1;wXMLi}u5*JdYaudt zB3|DUdsS-cQ<8*^zHK!LZrTFQ@}(1YGkdYX|Rf9up$wa9b3<2ZU4N zKkvxl)GalR4hVOhhX-e$2iN;Pt1vtQzh5RZk10S-r$Y8+Rj6ntAU9ta?o~`%Qj@6H zfOC+Eoa0l3V6&I1u?((+evg=evyw*7`<;#*YY#gU!D?&oadBr=!W>5Lq#pJ>Jh774juOKM#V2)HroJ4~I%*NF-kox?huX2-A%xryB6R7y83G5pp-g?d8CT-q z@m$V9Y*c5%duU3v9af)pM1GveWo|TCHH>%F4UA)$`s+X=Bu0Na*8LEb9HYlc7yf`4 zT7N)796}qXuqLv*%jRUHUAIpxOkhna6Tgq(W z$EnNo~<0_E-j#8+f}`K83XE=#-Iqk!m+MprVO=Lp6Odz^GG?1pF>zEiWz|h_R*siNB5n9wZCCB(=!<0ax8ly z{CK@5;y%DA53vvk4aK)b!!J!b>F{h#;Z1iu9FtmK#8p;{_!`W}^0On}tcZ%>y+8(s z2A40^aeEHnniqR7$~V9hKK4FhpQd+6xQ1#yQKy53>b~5GtN~br-3KGDt#*sp(W z>I?}MV}yKUK^--14idEHiC~=lf&Vc7U@Q6H!R+8_63{6WMsg#ioAfCnsr?(f=c1IZ z6s%{HL1SxDf$5-8FmkplQoQLtI#w3lIE%!%N1m`!v5N#dUF}x_GRvpfaGGng9;T| zVX~ZAbx?xAWrK>ch4Y(tTz}|D_cIPqIX)dg-f(~5B_r@ z;y$hS$73JzD;QE;|5=5zrwpb+>SRqR22Cdv&Erf)h;M-22%TY`;P_3%iXdOc$B-@CvuMIe z6KBoYPPR%MAgXHKMOqNz8_}&N07s^cXFw%+Awr{#IFxqi&QG`PZv4R_q8otV{Z0xL zu~f3tif@ypr0SBvCq77zTgie%Rlgq+x?qP8fv4S9i$l+YJim?MgHxH+Edrn>Qe*d8 zT7Mq$rMN>wpQgdx7RWhXRE8MA0v64$67NXmqXJ5nlspYsr>=WtOMWWTp~lNinK$jr z(t^%@meS1_S;k%vykO5iP!g3rjnW^rA{QnJ7**;3MDih<38Va?&qSnQ&%bSBjzWwHd9-Ikfbn zwWCRSgGlx)Pxlk;!(7z2)%XrGmR&vF=p$|e5}mPq!@pi)|52W-U&cYc9R58a9#`D6 z`FZ6WSLs9N_}x8G`G_sCQMbs+xeo2BTr)J_%s4xmkRWye;f*WBW=`^P}GD10D zl-~a$OP7mNN!N}XxdtO*pgMnxhj-ebK_rLQ(38V5&C#}LFUY~S&2<)-sbmckcO|CYot(#zAJ?Pe%vAe_cI{$QPwq#qY^ToO94tx|% zr9Gxk^?)6>v9$1-aruHJBSuXuV$6<7=vd!%2Szkzy(Msd^= zDRMnCR?aFUnT;mwAP_6z6BRN@9)(^SLqSVkiAWNb8uePXwu0f4jSLuYJ$QIc@G3r5 zwi#hwCpnm9aa*slXLW@q+Q$0mhQ3}XZgDEQHi8FUmZG|JWx4?i5RU~~%)OFTKNqYx z`~-BKvyLwF)GfKB+QIt$)Y)AR&=~UuT6Yv{h|Y$4LS$#KU_mb~xrwS71Y=EF`$M8> zY@n_OH$#{tBzjZ*bT$wm2O7ReLi4m(BX9zS`BgzJVI6~2BMCj6`qEkYxr1e*11_@R zy(SK@b~*jL2tE9#lumAIjIl@5Y@~NG>FijUS<10pNShjZWr}cjMHt#~IO$TQ6y+F4 z=9mlx!*Xhj=9{Sfiw;ea>Nw6R9CO4M60E(eL??V=gbHk-0*pHHDXY!b$5gjZ$b{xR zDfHMN6&qD->J3gUIUco%_k3wTxnVb);nlR{Z)7W<`S-gYn`L)Np%m2oBbjWKiSu=)`_xOIyC}E?7pOgF>fH&Oly0D z6J_Du4yMbSB$4(Uej4eeh}Llyxl9&7`njR{oGh6pjU*Tk>la4%76P?SgJjXDI5sHh zpYB&m>?X9f$bUt>W|Z=|>8uY0Q=h8LQEMDy2(r(VJh)K+DnyT;h6vi)@sLtNE-+nf zP-6<}Xt=SnE0oxwoxlh{GeQQ9oybA8XKp-3X0HVm|H*oXSRKR&9U9XwW9bZ$79T!^ zetsEpWGj&H6f_8K$29_!b z%oeKIm1^6EmXPiTwd6}UnCwIq4!r24M_4Cyo+0C2j1ib9_k^8|WwS!9gp)M-TY(>( zuRAt5Cnr_Uy?1{sh^>>78Vq);m)|9*?cCS zd01!rp+}%zP(d&6fQD<`WzRlg69s+oD#W}-PibM)TOC$SUB*wGu_N5T!%IB8=Wdjd zHN>Jsh;;U4xqryp2SMl%a5AMOV>K`pnuq;UwO1^KHcfAXk?u0$s;>%Mn(z0E!Z+og zgEMbBa@Hu=wd(pME(rp{7|)HVdpBa1D;Ch_<;pKy$Z#1t>6fHL9d~|XU#8sU9qFb1Q@Jr26;JiGD5<(i&CEy zG$FCV^!TEX;al&5;2K94#lxVF2$i>#nBX(iOThxwbV5VfJT~Ns0ib!Gmz0z)TwRx8FQGPk4Pvn}YF6ETc zV%_F-_UQlc%O(CmO-Z+i`#Wn=^z*dHkbOt(Be3id?nwPjVxaIRZ1vNRRe>nyS;g11 zH(R^xw9DfjfWoAtE)ADZaKPw^_>Ds;dRZ04!RIl|&Xw&yw<1mL2p+Povh)ZzFELBo zub6EEYe5$xgN?wl@^pKYuXvOmk84oS*OmkGCg?|ux!3lHK@M-yw=rxJZvp^gF=^{q@wZ0g-U`(GaDZ(GXMWB3A z(b|?HYhU!XRS^0SLK)w>YhrRz(fD~N4stSt-{NNJQ`^w=!`Q0E1l|5D!H4e5ncq&u zi^O4Q)7rb|TK|s#hk?&nkMq@D6FiYs{@1P!Ncw2EUp@-k7 zzc%D)7?jte>sfQE$qNVIV0r=EV1YNyd>z`_{-$K#D(b6YMLk5MzEG6-S75Re z1lWQ^I<*m2vGW~)U-%iz#rZA+&*bLbmyu||l2}MZ_(=@HSWN6v4E552Ll`1Uh&_9=;%v*xy>lqWezeM$K7Lw9S! zPsC;sVAQgol#e2bE8{;b#EvrVLa_h}oYYW_lQDqMg8oQx#y`QpgosdTIFkwXD^6+^ z?AjI%CLKzSRyShM6QYyq;aLdZL#C+oWF6{&xQqi{>0vPuobcsRA9CO-qsS~}fctnb zL|{j(f_7_5iM8=Nb`H0eJOWhaEG;huk9?{p+3_ypW}NAb_zE)bl7gevAyZ``R)bG< zn44D3B*;kq-bJZ&Tzr(ITG7741>VfM+1a-+ZBU(S#6BRM1vpipM)(d7!njT4)*@#1 zVkvB57tl*t3 z=APwW*n7d;NIemvB*QQ3^bIfaZDIPG>f1K;e0%&7pdRe13P5vbGKm}K*L^Dor5K6} z?wl#R;U<3uOhRgTVYaO@ZA!ma*VZSAw%eC>N1P?hy9oWy%sn}p{qGf*ub^i$cNEoN z)}JXJDvPDrd)STgGEyVfoN=k8HG4Fl>9wZqT{ar9>M!hD20x7K#t0VS%qbVr(8wE&K8avQrgLJFZB^l3M0X;SjR*XaX zHsw?XiCAp=&vrrk0n*$YT!72Z7#>ZtH>oMl6G_hn)mx%$9ZA<wKb2CsO+`5~I+Y^iWJs`NH$t>;;spf7QVr z23C`~5cidBepM^yY|Wg1g|~TN3CY!%|27t1k!Ib?DyZU(j>VnRD&iPJcY zMX}1KbJepeRZP-4AOC$yL&X3pJiIeWE6hF6#FC7zc3^kVkpQwnzAMe)gS zZwDxNK}X&7Y3qkdi_Gs*(Y@M)Z*?d`PDF@j(a=7D1uP><3bkE7Qv$o7W++pvtf~f` z7*eB8mTf#cx$;A{YQ+b-nil0>w{oJ%r9C7V)p59d3M7Uc8Qwaxrjx3<^|7mEfe4m8 zR^KS_VOz%q@DBp;f7{vF*X?K%)(FEn9nvY!$LY?LfdY-Vi8L*_hq8c85r@w;D?|(@ zkZ0u|N@4ZK-)d^g3od$PWi`8{c~)*9pHcGbnQ!Kv^`e%XGtLQUS1xp`!MulR0nKcI z)p(YddvY`1>zR^(=&6r*l)T1M&&bf(Ua_^ueRJn}yF~53vXVUu7jK%KAlx5oM;O^+ zo!Ax+Oef2$Rp!vphJIUy`m>9dz?M(zwU9r3x@91d67-bl^CXsntp<#5ekUV1_4PVZ z)NomM7*NXXX7untfSc~JKiLhn`7~28cH)e>V-zdc{GF#$38zjSE)>cCD1!KNw5p&# zjX0|Zpk33dN`6+|pQe1rT8a0T);K%~8qI6wZM$9uR%{lXJ%Z79> zw%4_c$YBBDDin5QSvjf@d2Iyu%t1Qf36G>XV3a5^kfnx zVI$2*$LWUR&yLz$$fdaBsjL!QA9m!5+-w9NDK?>M2PKtTpAPwl))e$|Z&zZgj&XD* z90>6KU|1ik$+Bl0h1fZ%fTHqD05g|sCoK+}bqN(sa(6nFRE-Res}1(WWTxzE4JqHM zMN*P3RBw@%IO?HK^Ek(!e_3@GZFrP$V3m^QJ&^#y3C~633k<*|=1&&(Y(8u26>`E$se2z-`A*DC=TLJ4 z2H?he-nMyl(x|_#-{{T{^-8EBn~6=@4O8WMEKg(Ce;1veYAW$n4Rs5rz&cWb!M)+v zb7`Q{8cGW$yYNFppibLueslzZmd~3+J!$}WKFqK%t}!3mh*$*&j4_z14$=yc-a@{2 zao?rEV;6%J9ZgA_oE&siIIZnV<|Mj{nzL{g+Q(XgdW3y8gcP`;eWw|wl?i{{MTH1F zhLXm?8>YE}2o@nu^e!`ukQdd`BC+z^c+_i$7l7%GlkCo?#hh#nXEs zEG zv}Gnzb(-qOQoiL`8Y|kyQykCDL4M?b@z|$#Pz_Y|U^W9N`FNstto}(B%h|aH>cBJC z)a!;8Z|9?)74HHJ{g(n|^nGgxw_HuN$RXnab@%>+4Dw(L1!i5pDE%%Zm@qrK)?Fg8 zG&s3QWU~+>gu99YYf(y+NTD?;KQ$-+^j+kQKDFQmJ5VV^Am$x*+HL`#6^InaxuP1$ zKXX)c2>SEK;Q=m+hY(U0>IuySm+hz_m5S-pq~gnGaXv)3MM`{rY0-%RzIP4%n6TV9 zgahIXFC10mR0NME@6k4ks1!dIx_nffmg2riv_`*4r6sl%W9Q`sm54c{v*)znQQ(tl zS($Q^-BBvk9A*{@q)VgIIdS$-xgfsiEV_tW-Me}5gfPdEOKVJGy~x9QFvSwJL&L=L zWmr^utLM6x@OD9huy0X9q%-cI9pIc#c{HhOdfYCii0hcIlx^-~D8B-4Qm*atA(qz@ zmSp#6?O??WS$e18QV3nt4wr2Qn(Tz=ZU|n?15lcKnDBWI@tu^2GE|PnWc$-4tI{F{ zzf%2i{U6?!PjK;*ZlGgRXC^1{vI>|G*v7@d?C@H4t3UPYb$>y?n~B&XJwH7~TB$|JLIkqv(xSrP2z@FDIT4PCpv6|xBmRc%j(pF`)!s)1T*Yx+Zq|f& zUJ*#e62@L3tl$L|KJV!ai@a7D-Uu!6zStt}jjstYFweIH3LqzKK5XBC<^;lbbd#08 zPKljRTUquLANf#%#yp6kzcUwesgL4hY7(|pb2sYN6G;C9clKxdW4fV=6)817K~>T8 zA414;?CC_q@pH`0uYfmZohWLCheDr;sI#Kwo?)i}(lx-{kCG8ag&W?y==VeR(4|TP zd}JFb5-;nf_)X2J2M5jYA5!UV;?{4-YCD7NrpXC@2Is1NWjTFU)iO1s1_Do8!~;*6o&cxfUT+ijr$v(LrpX`)UU;# zIBEVK)zoRptzTYrh!QeBcIoQB6BI(VhKY?p)tZ}F3AzNECnoh{`!?+%dV?wD7!K=w z{ehC*`|bXJ;jGSNL^dkI`|xckG%d11?hZmxZ;8&6yFsxiyAvGCMZ4!O#9wBCZ%8W| ziCO@^Xk5evyJI}-vdViX1S>>w7PYQYGnh*S>lPGH|@jxjPZW*5VH+U8OoYGTTI0TyRiHmodFK?Z&FO3fy zV`CD3mySqAsPxZP%y|%iI#*Y1{f<0&lf{OMAxS4kC;&W}^n{&#B|{%jBbBI;xIF*@ zScz7;CC;S6f~OkP$~>@gpnkiX5H~^LV2!pS_tX6yU}c8iTqd3Id0VG}b+iT7@rZPi zzv>vJEDnE;%->BHCt1GoB#M<-S(FoCBe_y*SpTf~4KlOWa4zD~~zBfU|AWNn(Y8GKtlLKxvQ>*?*!2xzF2pP(ka&!NZac4@LstE1r3kK<9 zzBb$j=08quv7st_krulCeo4Y$X@ziO@~TByXbK^Dl^c;!-&R?XguFF0W{W)In^Z|> z0;4p{#D|dpf`A)!!8%Lti5!j8M1lUb9M9ng$y&XnuUgQ2maEeYCG=b}g)uP?+6n_R z*Pv|r-&hx!ym6`@?rsNU;wrz}~vJb-Z;`ri-?1<%8w8sz6l(%)m(Ucw|S6f4%zgB8~z38aHA$gU!0gBDDUxNm-+TfH1Q zB9$a)(;FNye4UTBX!{6?Zf)LQ4O7|)GVB%|f872l4=Pd%(ir1!=KBjroy1W+6@lO@ zr9|+);bSN2B2}&uJG=O{>kj=g?>ZY(`dX}H=arPgz2@+p;q|7Vevdzy4k5S#%#-St z6Bq)M=7YB)E(XM{+!3rMdQaamzwUEaoST~0KSd9UJPeL5=2dTm3o~Bn8Vykf5Ji#8 ziOk6HwNlG-Q?xVHWd;^;aS4F&&;^jNAukaj_2+bs0vDiRJK&{ zS@zJrWFj;%o+pnS^(@Z@PN-o=$Smq&0W8P(`LrlOH-dAlj`yyUo!HgU*q9g0ZcSX> zspv8jjBR^T!^r4#@Ea2lb_3t9YWAN*)^Sui09eVu>UcXiXaj~7LXn*O>+qAX6Z$3r z&<+$9BgO=Qt%8wOBss;tuwo%o$!CnJRPHAnXDO*8jPJWAC1ll)vFV1_DiruH%Z}05 zqP?iaQ9d$2U2?1BjX_>DyQKyo4A(VQWFH`u=*1HD6dWTrvZYA5hGKWD%-6{pm+{oe z0yZslr>1|Sh5klgZ;%gahssA)GXyk00Vk8eJUzhlr;2)MgJWY9FF2!1YlBpAAM+SfwC6Gt==ooZG7r6OKYTWp1QEgv+#7_=Fi{9EK?`tTbR zszEu+;n>Qu$AVDVuK*v%Ii`qwZrI8dnE{LQb@GtObq;~HVBVXw=%% z#;Z~70iw+>W7U6{8CZkm)hP(+ra$X7wU+ozI59W487=RzPF8y@Bo70x*z@|{=(^SU z$F6`bj>Yc`h&sB3{P$J=9JmJ%sbP?VGOM)iCFd60E0Hy|BZ|$l_-kH&3T}3=Jv*B%dYikG=DfF&_Tp2`@!JaD06HQ!lLf zuM$9K_?+NP)Su-DKUjWA|0#Vc3_*UY)gcXWkShF{UgPPSU0N>?-qkupf_9DoBhZF{ zskMNIt=5RM_N$nox>|ied3Slznq=8%(mLR>QrgE{WL?kb=Yp=a(* z#m9*nxrdHKTlpc!#tkz9YG|%fDMj{zQX~GjdYR};)XyiGqrG5Umpwg$60s>(q+%jD z@=&HZ5d9@7@KC}YpTQBo8v4=eewj$ zRI+&HS|k?*B8ycmLRi2FzsruuDk_YT^#mq~T(6L~_S4(P=q7$-)kfE+dz&qPpLRi? zNDA~(i`-|GJyrfxrmT03yoLoF%(UntT@SKxNOa+HsyVut$Uk>AikX2h>^nZe=~{eW zSN5*nYrC@ONCN?6=WY6GF%c>IK$W|$AQV1dY>bY`k`RqxHmwePez{};&`RY1B2PXt z(%%YX|+vt1@a2jmjE1_oPlNi%21aZzT8{@lAyE zBkRkt^-H=2!OCz_)nXOXuMsTl>u@NaQt!Cg4G6O#9uli7_II;DM#2hUJa8BB4=W2* zA$Y+4hOs3Db8RDLW4Y_0IM;0RQFw55YN_{53V+Ymh$8O_@4DZbPmv)3)k(7tAp30g z{h`szJ0ALR8wlzrxrECLgM%o9M>AcjSgDYmuR=Y|Z5*JP&RePWh`6CWuod z4Z1gYQ4#yjGyn|S*(ig6$_0i+!ewyzaga4}9N-^%$hSGmPlaO!ltQc=NKFiijU7)F zJunx-#EY9qpShq2P))yKC$&_+;vjhx6G#2x9BAIlfpOrk#cQCY4U{O#TGku344_zu zdC?0D>g*TwqG;tb@lhq6s>Uj_Ibtx1u_*yMbW$f`@IKJ*(?$vll15NY%{kXNj=zgF zRJLJ0`XB^M9N!ejAqW-BAYN-|4u~xUt4>T}QyS*ei|%peq*FdrI?YyZh{*^Nsl&ZO zrmzSvVr5kHzW`+*xC(q{-)at_MtiXBZBzr(SD02Pc|)ttrQ@&-)|`n(IVzbn(dy+~ zqs}gZic8>YWs+fI$uy8hh%8#Yaih&d4C3Qgc> zivJxu)}s8y_bb8gqIXJ{{ef}}I4_$6dL1)E=L%X&MOwqzdNG+bS^~n8oUK-C2%#E1 zn9lr-xnDx4iZ5?W-*mdYgNhg*(LlvG4Gbpgp9)U^V_j7B#Y{F3%(IYjs%+63`?QRD zH1=_v_eWYy{W`jG%#iF-(O^FR)#EB`lrb^|#|s*KX7=pzhaQO-4Bnzulkw{-f5geu z=bX6q#O(PSl=KI)ZLqRKnJ&vc5KEG~y;?BE$7lz-!T@OnXD_lM5B065OiPh8q>xru(GzmS3r*O+lX9a7+i zOcN-7nfWgq#sNV%C@C(0k1V%MpU2;dU2m}&VU6BND=~m5u67uZ_oOR`2H26_QxFCxLipd$@)nn!XXT%O5?oYrQFbY4 z#>UuN_SO5v?Hpy5OAtlW(jjzrW9Lcq{%!^}dybqc5~g3Z*JQI}sSUv*b)oIrUV78Bq=i zttcBOek|!m}7&8II57L2w~VIM9IaXzg-MydmuTC?NNtJU*?pwWj1VaSdc_No2;bGZhJyfxa=1*iTsZ^c z(w?~3(;}{|#jE?y_iq49I^I`vAsRRTt$ySaj82zi8$K#RhKLNt63!kqWxf>ykPc-@ zm?|2>NYo}-NcJykUUN4!sEUl|JZJ+qe;D1vj8Hm_+UFgwg7p&*)wj<;%sj)7y8dGIFJ%Tcl(7PywNqkPiIDSgT4Uv;$s)LzYNqC0Xx2H}VMw zOYa5jZ=Ec8dTjmj^@E#Wsg2XyT{}b#KK!a9# z7!C&wqZH7Ra#;;S+aJrh^OvuPDZ=0LMB{xm#&yir01FtEK-WJFh$CD!2J3UR?!uC_ zn@RTUR2QVuxwP{LlU75&BLw=D#rs0cfGbS-2i!ut}$Ma{Z2de~(_uN7nqAZ4AF$Jd*(F?_9;AFH^k;FYI)bzVpP%|RN z_t;Dii)}0P<_qec(F*&-jMLV0KvEb2kDDzfvQxw2+zy%FC2hbhG?f3?u*!9L-q*c8FSIoerJ0f zHI&~JL(6)F@Nz#D7unYsQTbbJJ7IV^B06tS`z)elm`X{q%?UgfA7K`=Gw!eO3^|%u z>%v)=Q`uG~5>Hm1f|a|~$!B`0aQeUrEh^~OoVP5y+=0dMOck{Rj!0s+F&TrZ?5|cr9-9kZb`DCOhQAvUol;uLQiVFLGP08~2TmAs3E! zu)RSOL-QlZyv%5XxO>(6AZC=Ea7){r0>UFO~?RItbtcdRoC^UK1L z`lI$YLQdR0i*7^q#2j0m(8!KW%wWYH7>(_^h(i?tWe`q%2mTN?pPev)&fH?D$E43(x!<*=LB`7p z`Affwg6=W$ID0$;7u=~+omKtdun-pcV*36z8|OJm7hes9b z|J#EZ@GF|*;TwMab2$F%n^nw;!7c-&^gGR@*Ly$vc`&XC{52o%pt8X-5)w-8RcV^; zjmCp5^^6z)5rFze;7f91r#C5KX3g%4dXa2 zB*VCkGUFd6$q(BrXgwQ2D){$t`*(A2-r-P*RD^$8n1#KAInVnKSNuzomN@9VWG!t3 zu==8&Os4+d*8BhZ@!y}hgGv;II?TPu#yRsJ74mQD(XbA(E&(shc6B`*Kh-bO4dUJDOhfX;D8cW8O#f4!wYcna#zrq8@M z!h#lTugqq!AHMh8fAq|MXwh>FN^~=NOpyJ{{|_e;Q})+Ff9ls zJt>)imv|!_?+xs6JF3eV0^tx2CX^YPUN?vo+T;BYUS^qP{+8uU8I||F`^#kM{~LGy zKLgRvI4m|an=AO%>i&pUB+O+F^tPIvoL4c%D^nyPNyy?0>+Y^`F~5hjDo1({wue7- zg-S?D8g`rxR+>JI2h1!sn%x{WSR6zIL6=F6q1D)S=k3lK_N1Gwh0~`{Su5`0atS>@ z#hhb5>oB1TqZ6+)qLMCVN0RHmFB+jy6!Z$Oe?9tKXnQmTt|sLdc0W0Wjbo;nL;LUJ zYJ&hzP;p*Uew&qi8~M#ToHk#dL@7@|z+t@Wr&jfJPBVqU>l)tpVZnK8VjiaNO@)a0Ndm%AcsHqG zKdk$Yue&vRV7(fITVg@aa9L{>=KJfCr$2mNu6z&kg>cW?8f*=G4aOLGm66b+w{(D- z$DvQn60g1AAvrCw=!w7nYmgPQHhr^d)8^n??;1Bql_I!z4DYRz{xM|;F{?G)^*&4- ztj#-(7#WBEUaR)oAS@rxx8u_knlL|7eL(u+$JT|Rho;YNtb(>}tCF0!k!v|@y@RNU zF=jMG$X0``(v7YAB9{rjh|pm}IgNDit>&3pjScSH@6W%LaSKY;aRZW0)c*)4leT@T z$2s*}FmJ^W`AQW>wD}ETQP0hF>%UGT=yLFz9?j6GC1}LUoajZtV9J}uS?-f|c+`Cp ztyG-lMyt)XGeN~x{pjF!yMu|N^zho2QJv0d-3A(=yG4fC-1bl3wuc|Wr&f=SWA^gC z6M8qGCZ>|Otv5JN_{fI$2_BdvWXx$uk~|eK>2|UUoiFb;cn+-sobYND+Jjt$@2}0$ zD>6`?jKB1C)u4a#Iz??h0ci9KPKB?VbdU9 z!iG#OKdb`&eA7^0xEkv=*O@~$3o&J$&w22kMrGo~Lt_}bnJQ~PApRXY(~^*dS^Yox zQ;D_-#0yUZw&_8wmT#5D2Db!ncjDY1h|!NQV>j%aZM{=Yx_lZ)EL=vqP^ z>=NhnNCE<_ev4a<_!ZbS_U7S`+99<)|EmHemn(Q%XB{?zUIGh z2%7NZ;q3SA$IIe)8jn!c?(nn|3Kltezv?F-J^OkU4>VeW)BTM1-&e0XB8fi!7#pI? z%Dl~-QaT)%vHzJo7irU`*NZTMq%ITD-uFAoinN>iBTH_u9xBme73ZL3{)^fDJ)CtU zsgakg*jKHAw|W(mA3281fouO|{`fQ@CaIn>y*ap(9+%Z(=#5&EE98NVv6v(9$-SQ? zHPFx1#e3-22cL_lJ40tKQ7p(o{`V;$!p$GdmzNoJ$qd{&Bu}OpM89+%cCl%HGPG zW2o|wjXKQvf?R0KD=^gNOY~hGFA@G|;;}O4z-ol(%MUlO=aDhXOYB z)xb?WG~1btwuxQr~v|~HEwzi<9?a}xb zC%^Ce=VZdGQQ~AZnzwCzf)p&1q8)*A&LgF?7du!aC&Q>s0(yisrKJiPUCvLcltZ7D znNSINNpXGePABUCIzuw=Xo-cZ-a2ZdB;l{1mgDfW!rF7(_>?;a#TKrv5fAQFpJCvy z^we>eC{y10xu{8}C5g@dAHKdiEUK>kn$STSMd=g~mF{j7lvKJ?y1P?A8l<~HVCe1` zV35wCySux-gOAVqesBFQ{+Nq_fphk`?|ZL%t+n@#jDu%0HYAm+y$R9pHp_@N#8ZQy z(LYDiTZ~ZX{q7P>lI==t)+tk{6@%YDg7hZm^z`MD&D{?>o;XK7^aL;7-a5#V@dW$} zABQ&-{t_!hjx#u64Qazks)e=Q$}Wl3duaDp$4C>V%;;i;jLFlf9Sx`fNw2h}Y zbehuK?)G6mhWajd1yqyT<9R^BU?9Je6}azym=5^G9+tuXB;Axh{Ro z5@Z60SvEFeTKSlz#14u`@2eYzM?#kquR!}Y?xWp7+#L_aw!~B;b|RYTA~4gl*a^WcFPfcQ-W@SK+bN#AXFX2Uywkg|acY z1-de&Dw9Io-&9Ix#R@S3B#GGuI}saQ?2)IdcC(;dBWUF_MU+s-mD8i3Fmy-L6J+OJ z?Cm`53^LFJozwDj4TmC1tC>!GLuABDEdrR(^`%2ow@F5P6D1???!P1C-;==vF4}=U zBDaH1tOU>V?@rQYvasQzr2FIC+_u`2xTUuhB;$c&iEYaRbm+LvXO;=OEc`szOWl?o zF4EvQR^Mb=4?9Tkfq}{sQq=qEtOnz)4a-wWW~12NG5AEZh0nhNSOgRcv_cqpJ_LGI z*huqo@Eq&ta(Kf~3)p;U!@QvHN=?@9UCxnf>*S#nn~c)nJikd@3L`5I>Yte_TqZs{ zla1mnT!u>gdzyTfpXTJOW|}S7*XlFUU!b62Eg>=AtkJ0u-i3iUQsYPP%$p3P9I1Jq zZ3ELEkPd@DcuoC~4_p1Oxd1~uc%Bcy;}omWnATE`+fj($5gEaf4%V6sL0l`B5#Foo zYj#pu#_u5R9b>1-$FfFjqpowi*mbkfx)`LLh-%|^t4?!z3-W!vAIg>DqWwk6 zO7tgEOqsMwRVbiap=p>rjXxL8;0h`Hnd}{RiNzPEYxKqA02lu1YNOJHu@SuL z4hn)Dv`nf^r9TTJW-FR)o^x>Vbh4bN8;+V%d95CyKLWx;l z-dr}}LQ?nQi0d3oo`^~kePJ_-<+5*UZyDzeqeDw%+h0J(SfkZ|a`ET~o~f*cRCG=T zv2gD+8#+j%;b7#P4zAHCVUxtyMv>O@tgj&NTy-i+>os|ZCgOVT3*I^psu)lkISkq@ zR&44T9(#hSAZcsb6=#z)2@&V~9?m~FTR`a4h6Y3Y^T#@lf%-d*fd{tflE{1H=ARi1 zH~(t~`tmAl+XS_ML{SFH6 zTc|Qs2FDC`l90c9U3%p?l{)&>SygDxlPl4BfVLB3b?!5>%-Os zD`&YXO5sUz)kMQSon_84^z`cb4-VWe&}k6v2F0eTMAQb@3#&_q>77x{bL~8m$=rEW zkDge4Oi|7%p@hy2hW8icJ@zKT149VRG?KhRyIaD?==~rVk1Ka;W4+r&TQ>$v@v&Z8`K_qHo3X_f8Z~m@g1DRU+5?Uly1Y< z&v?Joh~pIO?pya&NnCsYJHdTD+|PLX7=;gL;jGh#nW> z*ZKVTevfpvAap5Q!Bj_vSLoVUwM<#)`sKmK|JAsE1sNt^`8bl9c_hWlgSPGd z_ay`2vJkD4R+N|bc4x+Efd_W^>FKW1|87MsL0oKzJQM(_=H9GE4c3+2-uOR%85!FQNIJpthJ@V>3Hg8F!aApY#S{PSSR*!vfqcxQjw_c_h)vYEyz&PdG}V{vi};4 z{#tOdLkJ}M;DUmEEKlOB|MpA&pl%L6F|?RuX|qigB}*2IxDWjAZusFNwogD?Tie@C zAr(FtOTUbW5h{Fa|7Z3-jKLWzF!1Pv7!5m#=sPBRys7`k#Rwqjq`_}zE*c!vJx2kha>#=c>5hmJHns`&j*5j7CIpxPgj5PHA8*wZ=|F}%21+rxa1c8{tM4x zd*ougK`L?(4lTxXmHDiJ6Ymf}C06gvLNEs$r8T7c!_gD^DHZ1|~IjRf2j$EARS z4c#zb_LOr!hNL^Z|3WTJrNX;cVE>b2JPU?L%eXyKRP?p!=c5KlWw!zEnpoyRArOOr zOZ!v*;{jcka6gm%;qxXa(uhh|YU00>cP7RC_g zXM#1J@t-MSs0lJGhMzP|a~pBA{hhd6REQHw3jndco0NR4VTT8@L|;-l`9q3YE-;oj zpKe*s+Ku~4CbHQj^cSnwvuRy=<7&E=3ZEGD#L&X?Pgh&eFTCe3YCM}Nv0Co*o}+%K zex>+^aIPNUa@|xB(S6_=i-`~?lCgg?s2U$ZuO;wEi<%ex6rXvo+dl+m+6dF&!~~xE z)kvzJ=IxeZI#nNB(?&)Rid!;=l@7;l#YkHmqb|yo+vmwrT~Us$pP6mL+2W82qY=jg zBh3}FVj?`On-wa4{I$MB=vxpP9u|X44L>f=tnW6$YLZPqa`n7BHz_+St8Zo)9~X^U z4TD5H6D7~>$q4xTVVE{9&{%*&W+Pn|!|d&EA~$(~H1~2FXknWY9CBEnId5?3e2G{C zvZ8hVGYcBJfE0?<@S9Gu={s7O4f=zjm#sBc%Wc(7r-cV+)y=$$)!U40JZ+fWJCo&z z_V(_S&9OWe3qCx|1?lNtNVuvam)!dekw|YD;e5SY^FC*OsC?b-{xxV@|8S`z*r-2= zlSj8iu~_5!#RsHiVEkM<1(N$dFxMJr&-DBat{qqLCz^Ckl*+ZgPSkvLo2V7PlR4?m zw0`ubd?yEZMNo)>t~bYvHFu_f(-k50^S67Jr(=qu>Mo>OI5X_cmy#O#OaA;fi_~1- z{hFG#)%Q_~o%&4L66a2G6dZ1^wu>>@zWecMlNomg;}?dOH**BzF}}RnZoV_&Sc}tb zYrem3Mj{oN*6a^n2C7Jk7YH2j&ji}&LNlE&rKM44<)_i$qz6w|ydEW%BMrs#kicWp zvfY4Hx`^&2=sNsWnkK^VT6B}g)odi;hXP+vw@JGnVW))nv6jtU&z<}EVDfP^Y{?hD zo2XO-PJ+BOZIKqG597b%a7Y&oBUxM~y?1m~E2B8ymQB5qYYF@PP}R^z7l))sCAQV5 z##|mo5zr>uTKbbE)2()o4%#x}wa>iqweQMs4F$}!m1a2T!LudrOihlU#F$Ygt=GS6 zYzuWq%V4@NM6u~8`k=DdEreXF51eSxbQ$eEqJ6BR?f(40$RyNmqMyU^A&y4T4vLcT zB#uS7xlf(W`hVyBU>@GY=}}S1*Xf)R*FGH*k;K#CW$lm4Hzju0F@rMmSXZY6SatTB zQp_W+FuPo?!#Vulxp}4w;=39O>zvQqYr5Y1{v1FeR-!jNr=zkB8#lm7Z|My7I}@JJ z`y+2Ce)HXF@$s_f?e+OO6dsfR#xf=S^yYYAJ?HSk8^Jk- zI^RWzzkhi@7xm5Ec{65TVPvE0r8Nmz5|^EE3k4W&fos3vSjoV|L%E5)VG%u88`zB+ zJXNACymQQdx6GfQEFTcJpuaa$I}^$}A56bcb<}j;7( z(UL{v@e2~w^+&^h*3YpV92oECbnre1$5+~Q345^wf`QuTa*iQN;-q&m0p-IfdHUXF zwa>zxdEM>)dR~)ZT9qe}-2wxMTL&i}WHlW!66!t;$p^;qa^&FSIS}N0-nxS6DLgsB7 zIT(j0J!lRLS8yqn!ZSI#nC?%4Qm)^(bh?>QX6xC9(W2>uB;O8Md3v*VJw1OHgEVv` zD5BLe0Kw!Zy4#(Q<>bH76)-pcg40KsEQ)lLD^Y0`mS zm2hDW1kiPg>(NKP&V^}tMfDdHx-ESr3x`4r?F;d|3W2k48N>bJWgJgRDLY(?8;(Aa zh(Tx?W3H-Rxtk-JI%Z+Jm1T$AEt1}mr)f?wAYe<}T|oIey;wF(?TJW2+)jo>m&sl@ zw!u+>U}qH-IGFk~Ub<0yZdZ?bxq6Vy7$dO|E7+DxdEJ}mN1EzVnFr(X7Q9#z*QQ8_ z>o)-&e|eqcSwE+)gpuy?^{vX4On?l;S%P(cG{|xAXTQK8CQ+Ni94XV8U%O$XECUTWS?Zcma-x!ORfkH=QUsNSwd%GayMw76&FM z00I#(eE0Kaw{i(qDxR#MK--x;f|`I%wg?o8qz4WZw~yN7RoiA(6Hz1|c3lt_Ju#Or z5`ufE*AGp~jv|^C&QuQVHl%C})rh^H^W@FC_n=64fVg9n>1!7Pxx|Icn2`*SsN;SyC~O? zLGNI6v;7E~tZ{EXm zxj+OOGcJr|E0G?@DGu?2zW1=77MIs1VQNsk=;RJXgIqZjOf~VpEv|DBRI0M49+=pk zt91A9UQ_xVX^tIuE$f|9308jbh6BBBQTpk9=!Dfd@4utt#rx7$Y_?F8?`074ut$d7 zXm?FX>fjEfRBXbzZxnyXF_#U9x1<1tET2v?SOSaZe>b&h$4s80%mpuQ_x1(cs1Iiq^{1P|qe#*Y=!@l4sKu*L0F z^)H3O6MYqG2OIh5mTV3hro5G99^zYttv-%b0gv^cleiwV%F8ZrJ({~wQ((1yC`3Dq zh&P5*DUYk;ja-QG{3vcyAlVNJ_HZf>4x9PMa%AgRz!${Vuq)q3FA_h!e9?w2>NgS= z{`gI7@ifjIAV)f0__8ii+#l;*>{5l!?Z6hoTfnuUS}^Y-juj6CzLeiUD2%WN~(T^_Qx94!(Zje+Yi&Qp|zXMePWKEHVQ5< zzx||AZbn@=#>REBH$*>5yy8KWXUC`Z(Ae|~h#J6|M6EtXz1LL~<>3qPJyKOYnx#8d z=jpLJs+!3)^>=Rap#rIQ*DECq023Azb$wblyM$2}F5Am~so{>9>SK0@hA8IN)6nGb zoL9|zaoRje>=qqs2fb~lzE$e=8W`oVrfA0s<`|-(o7kNgd&&Z(-k_y5c5{qnG5$Dx zDDMYCQt(^_FA;N-2Og~Wi2k|d+^70*K5-|7!OWjZBW9y5Oa781<%;Ki2P+Mgw#*pM z%QYR%fBBUd74!mKb9N<*YzCi1X4B~SFAJxT?dtNrx<2kUj;X$rg9x`c2m-LT1tM{{ z*R^4gJ($X?Ls>eT9#~snSUIoGV=?c6KQSaypIoR+ghQ7q9%e&_vdvz&7;kij6wh>| zega>nK%W-)xaYHSLNg5tP_5l_)yBtQ#wZ_9gJ_0hF3 zRvjxx>WRnY3Z<+AgDKZ?ZofuOHb)@ycW3PGKhT^$)8nF*t!+0%Pd7e@TP_1|17Z-}H@lS|5+r3i zgDZlX@3H#C3hp-gkorkT>A0LmHO@Cgrih;-8sRaN3sxxyv40y-15UY{NOE6HU;rCj zoQsEH+&bGYCSme4bGhEXl%@<{ubD@djCym7r;`o#iiVIfyL?;1W>R@KNl zbsjEuM8e#D(d4li6&SbAZKGDRd6=!eTDQA+@VQZVJ_X5NS~Hq2uIZ$?oZ~<;m7DV< z2fo^bRO6Yu^|o4Q2-PwbTRodC`fUBlLgml_QhE{k z>}DAH>4~xeQM=GhPBM2}DDkw>pcyVLeMz)_s2MW1Satmu(NbsIUHq%FEz+x#EPmTT zii|;dYTk{~A;DX}5rKpO`l8OT?LU*IF|%hY0nbu3Q1EM1^;x&6QXF6Am&T~rP?BOn z>Qwr8K?%@S5TYNYW*BvK>*1K9W*@hy4v2(JXae`^LW@_D{%UtLPx$Dd%uKq=NRA}W z##oF}j#P3tV`(Q5Op1`J!WyyBpTzkmtt=ZMH7UpS<$#BV9og* zDZhy8(2wbHpI}9Fyk|fN!8A{SFn7>-?0#ct@Hjsh$EQYw@jyIh!`|qR`o`;IDV8N#}Gf-?RQN#tb;;#=%?yNjQ2&a>JK~>_DS3}!=Bv=W2z*gnWUX=&Bfwg z7v~CAHtb{pLNo`&*@WBZz;q-AUHf*8^(KzW{?61mLy)bl)Qt+gVsn{rEsUSRZXZU0 zw#CmAjK{%tvdAky6?@#8 zNU>V4ba|$(buappl3bl^VzZe}wA9Nza2WXq@w_f<77U&P{H83Pvs9l$`R^_~l}+Q| z{gcW0PxGW-e3^AxPhR{XPNzMK#DCE$l*nuRT@p2hM)^xbxz`Z!x{wrnpN)x?Pb$vK@}aGcQqx z()#VEqC`(iuuWs>)Z=2ei|XlI0dmV+U8Wi!yu1{cTbPGS$h5Ef#IPmeYEP31HGXo2 zpLK3hv?vFpmW?5 z$KoG>qte;(z-*Dwr~>B`W2yN51a-Dg?r)Br zv@pbN9vZr^OUCb!xHO*pCNVr_KCY=3?Nk?sU9CQw%Z@tw4p+{=MFJ66!${8_ZDolrL}e?Q(~@k&X!om z)gaRLT0*^0zOz!#{ZYJ|c8tsgghWzLf2vg1vTW_yegMI#{DQ;GD#iTZH&v>owRN-X z=Px;(KOn|!EHSTO(0n-b6`tRzRZES}1l1%WU8}F+F^i-AYTjiT9L3bJSH&_FMzD`z z39-m_cKU6e({|fd$TY-LL*1yb68+};SEuC4Wej}4nfyH-GfP{tA|~cn=JmI`#=61f zWN}Ewm?|V767)^iD47in1E5H`2oc9XBd%D5<;kl$9tU5s%lkFdP9&5h!mNq_G*8;t z%kRa2h+eN1ito&To)(fNN{uwwTav=#=m(|V-VO}2e$HG0hE=Ak;sM9IKuMT)u%=FJ zurU*8RVC}+7OS+EPaM2?gW?A(S%e3T5RB6x>-tNhEQ?AkDO=L^tSS7106Ip~(toPY zr``FKc!hw>f#0_Etvw!$K|~-w5pxVQ4AxyyA~cUzJ#x3Z;oP{SK=jM-zQDk zXKk@{Tl=FmucyW@pi|_GY5raXMk6_^#{JsNHCVX(cn#Ovn)fOt(~;fW^OyY(ozZJ8 zIMguZDKJOS=}?$E%M8%+TjffQ?1vCB>50{xO_@O6<|aKWxg*d=UxK25z%4KC7#fO! z12>W&K3RAokQ_uMZ1nO+pAW?7K`-dDCvDzb?rAQ)I-r$1pMG77eu{38h;7xBU76Q0hY4Y$!X@32`s;K*<~)>@~zw1()C? zKex5AisnKmK-r9^Jj8sKcO#-Nh4*T3y={TPe&SVI^(z|XN>Ba5!nMk=lG4_g-9+J$ zI^r9=HzZG!8IBlIlKr5PjeiB zh(ZPJ$Ixq%G#w}Jm(W!2OEv=D=tXIdE7_WzvYxgF0?-pG`nk?wpJZ@=-gOaGyF2Qw zOJqhqJ;{Fi0zOH6JD8pt&$4|vYr#r07SnYBktZ!su&LH&{b)I*B_Q7SBGhNn_zm8{ zZ}G+KH{cAZXzcD`6A`u8n48@;DrEX4CyHof-;i4iD9FjOcC;096*-gF9%mfIU+rO0 z=@m~ZPd2p(JQM0&WiRGja@$2I6ML!|5-auXQ5DV73BNqieM4$LPBdI>q9jshtt{eW z^X5#+sJ1ctl<~xI(_r)Sp!CSb+RQ2;1f6Aj59FM5Lha!gI<8=1L{x19;zB`WGX!i- z#H+!TBvddeNM;m0jVl+8l2@PHu+F_wOEW)j5ogJMID_c|9>B>gCtjB=1nC!F@v-!P zrCXTVZe1{!yed}fkF1^Ij9{`TV!a^km`mNq)M z?jNV#yS7M+cjx7%beC5jd?z8HngNz2;Bnj2Sis5jk=AGdJM+8F?*JkOrhgB8GY^f1 z1h?);W)i@KV>&(>fiW}Jmpv#wh$QeG6e%sv7)#8xs_0e)e4C<9OkCr9uch2$xD-#N9388aY+2@S7JC2wAvd|sLpZac$ zj=t~}z>NE}=yLjH$}Y$sVpPZoRz35`8aN&R_T64M?3cwLecp-ru^DdmJt1<#^-1ZC z7}bqI_-xWwLP(d_tP@sC9}{m37m*)V%qHp4bmu>JX=`^WwyUx5ZeF+ zlkt9RTk%`1_z3O{mkMGbsu|f}^|~j6_UNMaN}5##w?2P@*K19`#IT8`LOj681cHEq zQGy89ePd&IM430(j^M~hE@x`&gyV#_>&+%VB;p0906W)b3RM^J8t-jQHW~8%C6M<3z+l_2?+snpU>E!nmu?{ysET4aT8jX zlP5OO>J#l&ydU51XP_gwd)#LnbS?u%;>>kP!hRnU#W_ZL8ufBkB0-aITH~2#cP9tW z63EA~=0pfFS02~hr#2qLf%BX0E}lLsBM%S9IwL`NY5u+t`}}8SKkRfozcRG${HNI% z*EXz5MUl90ipv-{*AP?Wk;s|fw}@}1E%%oaM-`1%Tp2yF6ns;CRH0a1wTew@uUPF&l5}?C z)SbKN5x*tUlbZ0_7a&}wb_!DTtdBmX_T5RMaB+wt3?+Vje(T^jbJ}gI$g0ZhOwwqB z$*DU(VG*x$strYd7)e9)Ct2A!;trIV_5@AUHis+wlDS~I1c@{;YbI+|;sBiX;1y8v zC~)Q&dHpYk@@D&w&+@n2ZTIPmX(IE=sW^TVY7Sv-4pmwd$c#N+*P8b8B-eU^hmeSa z0PM7~GDleCKQwc5Ba%f)|S+47^umbyJ$ z4j;nXc3<)jD%@^A15)<_n}yY$Ys#19iLRqru*u_z3csvmK^dx2ELtefh%OZrp?I1sF+pqqOxbJKIpI~)jMbm1n`YJQv zru+m*HMW!8tx(y|fuOuG+% ztDi`Z04RaEC2{x&el32YEl-Ww#ccFc5a{9#F<6^*k!rWP$jt+)L!+7hN=Pv@_?ac* z^vo;_Wd`L7jrpWr?KF>p3wpy>tO8DrzB)h9wv!^M@^<~j&D2AB)i^rx2FZpQnB-&S zqq`k!M#p3$sZVGL!0#Hoq?`bs*>&Gu!Yf1z6Uf7jJw*MV7emP>S)=@FrW%;4Lv$^7 zs<`>wUsLqgSRN~}xbDOOPo={AkwSrsw!EaToI>%a=ee@E5&9%O`zvF+EfwWFyaf&X z3OPyR@Q9S%F0fuikQPtYfWlU1twdlfeRzUAYb#xH{rZou0D)TbUVOdlr&qmU606Or zDcp};Ae#Qlls_+KSYA=5*&-$;N?F-ekrkscwT4xdoYRg#l4^|zz#$m9AJW9LY>Vgq z=$7vo4cDL;t1OWYqWa!lkddRdMw_xImcGUo7a32j&dNsc>+uim-uN0ladF%pCVBU# z#bAYBU{(`_dddl;PZeiU=&=4;W&ND>C;4Tr#Hd!i3BpC`bddLUJ?@!bmoN*ymj2K$ zsN7FW|5EUIY$_xBIBLaJ*4fClCPn?ao+ho;`mb#G*Xu{}5(}=VV%8(f`zx9Su)L1C zzbH4jHqj`T3Vh6B{WJx(ee}8NhazWuviE!J`41%$LeJ$J#tMrHiZN?*x~l`Xtd%Q` zcS{9&uw-97BcrBzgcJYRlKVaU>@%NH9|3BO(m`5k1-j^W#7Me5Vm&c8p}s#-VkXI2 zJd%Hy1eePNRAqk+BF2wP_>uEUjXhSGSy5i6k8w>5!P|Rg?@QG9bRK+cEWLW}+ckA{ zMOwS;pqU9pW4{5C$8Om_#Vy7?Y{VMuxJnWK^|=7wm9%16Dsww=EJArqnEwhMuPUFG zt+9eEkpRk>C*(sV`6}LX1F=~Lp^r>h$50u6%n73EiUC_9k2sdc!qfts&#i9+xz9EnU`#uUH9J0 zS+4Ua?}n6NCjZQYzY`+)iO3Ih zyZ{&%eRxRs?=sMd)zWCSI9&S$9GII%J9hiNr%r6Z`;PXtF&{|w9_fk7uV&kcDVEtp$WgZ{ zXU6m5M)lg$<=xg}+WmTBZF5AdY_!D^mQ?tALi=Zid%=P{ou^FQ<7ZX&S4$#*(gRdj zJcN-oN!!*vNTsFMEBlF?CH!qkv!vvqk4(&E5cT@BJ7U%|r800`)t9x{-XF!y_e#Hh zep-K>5E-#8*48Ojp>@A|YC){Km17>qxk+p5fX>!nUvDF&$eyzBb}93zxY@Q~eb;yR zX(P2N*XsQlUON)Kvm4W+dM2CJ=b{*(?WwB2i~Yki`8(Dsq#miY5V)YuaRD1a{x#@I zPZowhf=eh97@uX!LVNrBJ7hFNYTnh*Ji|5Z^xjWGrC5?Z_2ao)sZp}U@Pej$BXe%q9iET^I7&M@#Dy2|lyFEP8A$eSs(QaN) zR1k$CkN!hXYdXFD*Wg@cR_3$tUft*vs-mft$>pz0!2_^gYzaGzDlY$#6@%7~rV2IG zQ@{4AajOE>?%%i5;wJ>xF?r%Nk5csQXPdMAE_EX+5*K@$eM0uL0{QpPt!i=TQ`Zq@ zsa#Ck;Y^Cf2friZ@Zs>JZrCTkW>!*A0T+df6Ua%*I*goPD5HmVl>5A+6<)4<8XG)e zk#Z84Yp(k>j$N6D+eEGIY6>q$*`w8Nd3sSt)Evpji&mFMpVDa5 zuk3+ME@%;P$0r1*sm5vqN@oLQUpe2)k2vRDZEQ1}27xg%c&?*L30n{xbJlLsLmRcz zOGG)#Rg0DxZun+g<|zJmy#6Cd9?-|OD48#wk^Y~1F++scaEy{Kw6v;xXQJ>)#n?~Z zV`ohbhx`}llwgrbEaQXl)5j6a(&Z)@3O6&o_TfGE*phDEF84;|Mt*ztyJ3Z~nzJs$Iw5H7YX<)ljQHW) zX|lJ2k1&E_`|QTQqZK^PS7QD%ZU6td7b1XClnjjSUXWhqUq7Z;crLZI#o_SFR-Uua zN|`k1$T`12CxLpn+=Ty90~;&6uiV@&`G*nzMxBB_?Tr*uwAgmccn|y7YgjVNHsvF& ziBBx%D%`j9!n52i&rd$%_MI)hBy=TJR8$jfG@(Tl;8e`cZy|+UY?z-yyf1oWgbE#6 z1sp=W?bt3_3Wfip*nKC2kFCUJqMfV0w5m3J{HJ>S2o*RCd>krLAfJt6T#<3ZH_TXH zevPs2l60Zhh;mgxZJ?25@g}LGXYJ$1NuAhJChH7zQwFN8w^0Uys)es!?iFI^cz`E) z_=nK4JA@iaLySVjMjoLp#>&>%70KXAi4CS*ZbxSvj&Hl2<{(%MxNxh!We4DDS zCOs&;|1$aHVfwcbP0P;_LN2}h0`}UpGT2dNnRr|c&1NucmrzxyQ^I~qgZ+dZ{E69x zPJ?Uwsp(JH)Q=a2Gu46SU%$w!r=){Fyxff9RV2&#{sN{z!%iiA zb;F?aF29>KqRqAXto7#d`u*CMnD_WLMDZ~B_1M&DNp&ISWYss5oa{wQ+eo4r%0j7C z*3e0gf3NWqPinx)HR;do_RW>Kb(i%F0DPB*PSFzNU1f^JP=nMzw3FELDV8I)zbH8< z%LeM&O3RR7HzIKM^%Gt6PbRZIOcS*T6#0tMW(oC|(<Ysu;&f2 zPn}@=gA-1DJUyJA-6Ue~PF!s^el4WaY*GU(kmi&2(`Ksxyf;?XH7l`Kt(7vvnnGvN zE7Cl$sTSm_lw#w4L=^EMnC0HLbdp>Po9zL%rbSbfe2N71s`E&GEP4M$RgqSm+d;9q zpFAWV%*1RRCGbvee1NV@qWr^^%(-585}(J9=k}YEYwmh!G=wmNig#OqEX-@?OwD># zvKhYgq$?`&X4$@I^O3J5e*j+0`sD%M^CTj>(a+ZNZs)&> z-A}0+xJ57v?q~PAGSq9;YR+z+gt%!?@uZFPC8Ome>KH1QyM*}&gi*3F~Fjb3M7t=(u zstCZiEnZi!KxqodrjdusP=8F(GvKdUY|hl$=rxyy;A%Ju8k?A`#_5JC=i)*AF^J#A zYP#1Z0j|5f8^AcTS=8~se%f_g>>vWyp1f1HU1Yr)t=N6Jp4!~%cf1lQ-OWDlN&%R} zlDDzbhZR)KC=rn`)QWY#+q=O`{9%rdfRYl)tE5UX^0H>)ufDDj0ltjyhnLNk8k1EF z_1n?UAJa8{jOUdI+IJmjeb#tN4OUzfP;Rm|TDc~!mQmxurA?%`Ob(2aGrkg4uh|W$ z9GHKr_T`vsu_|wptNtvutb5hFhnjKG@;11}HDhVSG~2G=Ju;X}6uh{GQad56+ioX6 zF8zb`)d<-^esl|J{)(YTNa)v&?q|fJl1%mGRew4lHUtE~p&W8B7(DB7e;aD}N!)~c z@tJqHy8YzW&=LY(=L$9JUz8ED+&d*cS-H8~FVkdmr6QIIbr4UYY5cmNUUi;N5w=iC(AT5mjsD5={C7X)Eow|ZWqsAdUe68IP6oSN$DwEjfa-x~lW za<8@lusI^6|2pQ>n*zvA8hm{IJ99wGzN1)AaoOYAy+BNLJqoZSPAs!%zEue2SbCzk za{$4NJ!q7;2W;xkp9oo?;?RUAB!t{2uB|0n#Eg#0T>^gFW&bsuW@dj!u^zGPSEfn8 zqKMC`o3r)RH`nv`4_31Gyz|38Lq*c!(-l{T=cXqzpAR zTzpJ{GLMVxE(M}9S|oX+=&u9!>F#hoRin;fck?FIN1bT3ooWHaY$mM^T<6G4CW~_G zR|IchOOm6r@?+6gnbmXAF!nA!^D4eWAAa*c@Z77)SK#Z{>>EI%2}C2XUIYvSAv@KT zA>$e@k|{vsKCxPbhH+Enb9TjrY`#QVtQg?CeiF#?#OMBY-vjWP-iUN}ci+gU?p^_I zTkB;#u%_E-w7^x#?MUjK+#DbtgO=B-@kP*}`;F%s$`ytYX|8%9Kb}UDtiQ#r`V~nf zdX?_h?D$o##?|p2aEm5FVwdRwz4*emU=BDt`R+r-Nmo0f4U?NQs0!V#oxhZGx+xL=soEj_aQSJ`)QVh7+ACcVM}REP^%BK8i_-6U z1y~ueG9t|%?S6ap%e;PfO-&f`3PETEUrsUfm$mH6YO2?uo%mi4;|IHp`@f_((lEq24jxPkLqH1P5>oy7pIG11_P7#L?)g= z+q)i?L`5>oe}5@myIVC)fFAckG@{hdp-^!yvPM5b>a z)oh9hfR!h?H-O#T-n>=|WwGDjtzmBn=MiVXTGMtA9ehySjFfWG113wb#FB&26>A@erAv@UF0}GS@x#9vu#*{3~(tknP8Xj z*cR7?c|<5YIBM*rMg*wlYu>*?_R3BV*I2HN?3?sCB&BdhRCaX_kDM4I zQLv~AGxW1}s_Sv?eBf#P#r|aEnbu?xVnyD+mhEnig$+ey`%KhbwtAST2774Ah z-CX)k{X}fk0E-Q8`c0lUU0`b`UCjc*`Ui z=P_kF;W^u*BKD>A#WU2_2AvKRTxm?Eacsra)zu>DB3`#)4Q=go!0dZ(*`8zJ?-FQ# zNy7)MAK7C#p9W4!T2l+mOZ>^GxTo30$4|26aL|gxU2V+I-^I0tYpWVst(Ku%cJxay zjaaka{i9SQEbvc|FkvY*m^SIBCG9;ac}_W0#?1Xm#Xm=#%g1e-y}@>H;R-*0#+bDG zfKwFlOnB5RU-01iIY#)d|1LQN(heCXfhxvJM#ER;akZ9#;tN=EAcndK*hotWf4f(d|L z=nxGKy-5j7UcGQUui?#YtZd4S-P?Y7_daB>QeEiH-V9T0RPEM}N90ge zKcI1Zr>}xM%A@{yFOVQe1DL1ArCxkU#gcpT>5@qJ9a0FRa9O$rq!H&RXncU!6~N>v0knc^}6&p;NH`J<{bJltiq?p^WuoeHs?V zd7%8#j}RHJs~5!xEuz=1?YkQYKbV`UeI2-4ngY-uI z2gG@B%p2tIaBpYpu|oqhPZYkaKh&2c_*A^D1Z*#dEB&Y&!DZX z9jlID9)O&RW-s29Z!G=?@@^kRi%+TOPAKcgL|hJc_Z(q}oB25Nm}sJ+Tv9kHx=dVs3O17zrXKWFAH(i95rmmel-d@-H9ArQAeEoA-}kZ|L3x zZor@!P4~^8+Ze~^1KG1MQGK?~R6M7KjwOCx?xjBu(2o|`@sMjlA#foChlJIN7iI)H zg)8S_Hbhkprd@wR6gqD2oEvrrj+V_x>!?3~b9G2l99Y^4w6KW*V9Je2Ko=54x!o-uundwZ1^$j1&Avf*3In4r4#&T%tE% z0=9^Fzt&^q&du)oCWhre8adE7EVT1D`zHO@d7OYe&ze(1f9z*vV!TypO&z;SkybOL z-1XyaZ-Y_J?F}MVsZcq$0^83#%+o@SJ*7F)UzyA7`DyfzyE3N+aTiA4L` zZomq?gbG#K8 z2(HCh%>K6foG5C~4d1q~tn2B|?-XS}+s@8)nmzS+iOc=0c4~y~Y(AHjpe4AAU)ReH zaq*T7ge-oYM6Qn>i(tVoJ?vK&a}!;C%Uk;7ooPHXUbOMXbXO5*1D?ZFy`Tn5Gz}u= zbU(5=VyTOWjiOtmhIff)BbDD;7$cT8i+tOClNIMu*rb z9_VVC(jdFZCQ_X<50@sgmdMSp`=nhUnEUn&InlB~-hZlQ%LUQtc;Kx=>t6KiAtfAS znk__^-oN5kOcX@lA|h2|$yioVZPRe^_wrQbLKNlZaYNhcy;yb)gu<0tyEb_4d>u}Eruy9CK3YXT!xey)$$26f&44$emCumPNw#OsYvn5Elu`Znw`mFvpczt@6lI0R zHMTeE;}&a8iST5HA&-vs#S2`Ln=l6Iju|YaWAMF;Afjx4tp2Yi_JC^tk(*q2Gx@+yTrkL99yZiAJ6MCyRHF~p% z<=1j1dTZGAu=<-EFQEo95%topHHY>kp6%^CFe|=HvW$T5PY)a^ZygNd&7wu_pi=~+ zOaowhaLU2WjL=u)-Gtu1mu)r0V;gEIc`e7IOr#h&x;id~8LC)FU0A+6N4QJfv9`V^ zV#m_|LbL?fFLS(@R=AypBff?U6lv`xgmRa5BF`J?mbL`hcTItZ7g)iV4YEnpt){Tg zK4&ZUDd)o;*B5Qc^*kam0`fDB36(F0=^m<#n8E*cL;8lRCxdTD;)K@lq zC>ehOC#Y3ig{ze$vMu6gmeq$!#7R3Lp)}-vV*@M8iavVCAv%}K1>Y&<2&nLz@5}BV ztKYmCnU89$D*F~)P7vA#y1vz3se zz;F-Z@xS_hAV{WTMx$u03h;~GI#pV>`@cLZQtdxjjJ;+PGs9iVK_yMz!l=a&k z`f&Fce+!BGfpN|G$h%|C&lWnn8r|UWB<1~f&)8oU5kQ;hB@4Jm#6)TCzw57Gsat~U znkOA-JlvDftw)tBIq!wiz?bi6+6$jm{m!jcgj7{6*PPjQkB8U8?UXjOoxwEi8f|gm7KZ47*t6P)9<>gmgJ|lxZnT(N6s-?I@wIR%$ zXehELe~(IlfG4NLoc24#lF~@53e#^fgP5vNyksk3ZUc_Vq{yeSyvx$1HZiQ8(-YFV z=9-|Hk)^%j592}g5E3kdb_Ve=^{Jiv7`^7;ADATKb7_A(`+7dP>3pe?pXtEuOI+_v zyg4XX!6o(4Q^RWI>4WIM{h=rb2^fDZR?>NTUUHJPcK6?>GnTYnm}Jv~kFrgQE39Pt zAZz7C5$dyzE}+VlXbbuviqh{iV?k>TS_Wx>2s*YtUweSU89i=R-FE?UfD81%udlfz zRwE91pS*yB5p*MV&oN(B-Mo&s} z1%oW1<+Kku%I(SWp&*LcxG~l#rr_!gU9|l6Zkj=-A~9+}FYs`koYI{l_!~$vbzFmX zKyutmg4%lQWI-zVqMfkYu+umhVot(gGQr7`*h9X9T;0kbu^*zkuVTPyKQlDN4RMBb zFzlj49SSBet0P36GdN_=1twCuOKAHPs9xWughbCt)x{=&LMGU9J%vULyM5v9X;q%S zV>`5(#*8e<5(P3Pr@TPoUqB8+Js_odaA^Kz-~n{~ZYYJu7y&7l6E#=81<~DuspI=2 zhskD{c?kjR`C7XAF3rT^;j&^-ZV*0~)5&pZ(ff(2r)+iYyIXz1KSdSn z8b?)~qoo75%cV%!DO9Q@<5y3R6vb9d^{IpUe{lA24cR+ATdg zm>p(26n(5Fg^h$A;%m9lWG|hE4}a65kbR!fHbyU6W@^?J`@Pwmbk()uDeI?eq3cI4 z7Vf2%9L~QjP9Lh_MdmOD5T>aTxD?7QlFPG@&~E!0TeHw%t#~GR3dZh<_q@UI(Oy2+2-$JLXwF3J`AN zo!{a4tKr+W>N}#Z{I2DVxVwKK24*Ls;n@hS%@WjA(oxXCLQ_z01IZpjKZB!E!=*!SM#yveM~PH#_6S7@8f_?s{LFXFHCNU3PoZv|0XVa~zR z#U@%BTCUC^i=Sean<{N(Yf6_vZJk)a`sR0%urc*|&ni|7`2yD5-^b|!mO+QcD#g{|7Y@4MS* zkH}if)5^{ql<%;?B{G~~EQeN6+t;~vI*g^OcewqEJjN8A;$`pmdy+89H{P4x&E8N`v53Aq#LnPAt%LV zQm?Wl#sLiT(s9R^_bQ1|$Jtg0eVS3+!2FFyNSZ+M`I;Zyh9pFKg`nUS$8%rbh%RT_ic!(D?ikqZRr4SajJf388*u z=lo`W*y3l}$gDI<3W;7p1;nY3SsSB48RZg-)Nty%xBdd?>k-Cj^mG-o-9O6`m)2~^ z=9to2r8v}6J?l|(V^raPQH@1i7Cjz*cv5^tze z27}CIRVG+Is+^2Z(ChQ$xwZUgkYw0BSQbu*B6*;8KJ1XD9^CYQa`&T- z>);S@W}E}NPAI0D*uSWE&hY(PAXAb;4zeYCQc83`aT*L&Y$VRtPUgtZvoldmfqq4- za_b~v`w0Ig%xL#$lSjtL)|gjZ<$QCX>`Hi2&0?xb_^2RRmYVOh?d48L8>;73?ro(+ zH?v?*L{Sv zoJN{gjCRI-_Z;(SG`fl$kEd9o%GbQkhuu!Qg|GQ#c2w2`$WKlKa+1KrIf@5`((w&1pkLuToLRXPr? z>1MlseOXMgCP?e(|! z*=l>SmuULTv_A*`c>YA_rQ%@X6HRx5eDaa{UO}}(%(mP)D*^u@CU!dte`NO|R2bau zL<9ALE*`@kr8U!1xX$!}PHjHbpfZHuWQ?xNog$~T=ts&o^t6(s z)AFm)N*GfBhx|?=SLL*AiFsIeNk})`Hh72>!2g7le3}e|L%bSA4!nwl7(xb)j!D1w z-p8ZSbkZ64za%i~3n?jo4eg3_+C0wGp$+zmc8o-CiZ5CJa^^nAa^ylX_rK=7N?~a7 zAy@#E^p%vKo^!?$-HP%1B?n==C54xJvW_BA0_Vl(&B%QWf;>5M4kb#arNpE z;_dVfcL`J72-Pm3dqPn*c8uU#1IRM-O6 z{sn)7bkmCCXex79_N(PHkBOq@$iAJA9EJYmVuZ!Pnf}YC!Y_+^!QIv|3QYwa64sg; z?Hfsbc6a;R7*hb3saz>jng%8Nhj6jC8bmfy zQ=_r$Mf0@3l5_UA3hacmz~j;DPc64JpQjBLO9 zj6d;BQMQ)hSxDC0@_nNLDX$0^`4>5*XvwLxY#>mfxE2q`bXWjK>9lX;eX&`ZBn zxE9TghN=+_h2AqzFzIZ6Hfu#rw<`NJO~$2A@0>WM#=+bk{UwIshoF^I(*!@&pmj=K zMOB$|3~CKkILEYwsvmw^sLUs%v{ctnSq0V-~5F z*9w*Iubk8B*6>`;B^*m8Df_O@9`&HJ7%cg~CaX}|cJfy*;y>+<-PE5&h7V`2YCGnd zZD@uoD{{Y3<|Sip4(){eo)(+lQ4!)24XSBp}a+ULzi*~VpT6xMDqPND@ zX)DkMNiB08?I)mM&UknkNJD|X`kkzoTWap9T}wYu*s79AA2Su{@r$VmD6w6?%(Z-3AS(DjnR<$ZV>(W0Ar4yeBsvI7| zM0BJBc&*^dvuMEFluM+3FW#QBD}2lM#dg=X^%_RpKrjk-r>p2wZJWDhiygrywh`rw z(88DM6FXOh)|t6k|KD@ps-k9WtXWG}m0NaQ8Iyy?SE)ECboLlCON#VrD-bmX?m8vK zi)gFer?(ooxR?bT{^$Pwh6IUAVdsl%6=_~C_%N{z6I3dlhD`Rem}f?_YnIorkB;Fm zn^^#DJo%mUrxcsq4958b=aZ>kQyy2VXtOtnN3v8aWQF+JqHLtc!XGnGTq|FeYV|#L z&=<9~%ec(@94MZb`tJjHP0r=+ZuX;g)^nMXk8-of(o z%+F9?dFDxT7ON<@FD2&_i2S$+JDQRj8R|2YpN=j*tCr2!MX+yWe^U+pxG;ARm7K|S zg`e8H|L7;zjzu>9(d%Pan}xY`^oU|&@s3_9*vKR&DTM*)Gw^Je|r|-rBt-2mZEPLyUqMN#wY<8q0580a@O7+xDYbtGY&NjFyXt{QG>&jMgHt zasNMr_+qS2+hI1T^B6SQDP7{QoKgTm1gw*|M3r4PFT*uK zN9K{utEuye`EYo#dD)O^(eJT++H$~c`tQ__VUmO)K;gIm`W(Xm5{Bl4mL+Ea{+HW@&8y8pJ&+pHEwkyFhJltu=rDE%57-5lk zIBK28x$)50*ea;-7WncVfTw9y_iA#HJhQ8FWT0iWTjInQ_VY2NL{}UQq$0V9eE0>J z;!^os_6A?Epzag9JC%f}zW|tP;@#!G^gV)ogTuQ1D?Sz$jceiBbs(0Qfmi_3o4ai+ zlS^Lc<JKZ|LfAvF0J|Q_RL!hd&a$wUrligexLQtboC+WO0=RtQE{!XSsr@hsHFTJ#eG_7 z=KG&R3+gh&swkoc1s;F?&8tCuVk(`tC_aL&ttYVWt5>B1Cvt)E!`zp{1jvbMAq`9Rz$G(|P0Hn%C$j zgkOAuP5fvmD8%U*xSnm1x6Q5rx+7)ts$16M9#2=KO1Tj72;gvI>Y8U|3^DD&Ba{~` zG8_#n%nle_#moJcW#2Z0VNh09cG@PQn3|S00vLX10UKF-fLRA@;1E%I)M81f2qGG) z?uMyO?&JH6_6AkuIHS^T8QPks?l9AzigK2EH}yt}p|DfX32tkmZ+1#>OWN+IA~=@> z6o;tU&4u{vt2!?F;{x;AT@{lqukk>_tNB9W)Mnp_yKGocb_{ltZKE{Rnvs(f{f7KF$zwVv-Z|zK{qmIQTsFwu3ll%Ck3uD8rbv$O?fw;J5f%6MOGUy-~Z<+-_Xm_*>3%w_1<_?tTS6Ofx^#xSS?qawLav+N@1|sJXS~HAVAP1QbB-(HF#x`YWv-d7z_i3 zy&b?E9aX`Em|qHTZS_}nzZotACcJ!1HS-w1Jc^52%IKHLlh4_zQ6*lxX7q~X2!;fxl+0OwRhK_DZ=38 zCl8J)pYGdXc*Z`tB8@4!Pr^kFo%UX=s^-fg&~vGX&&S$bo@fTkF2??~AtQilZ1JVl33*(DK2S}+b)=uiHVBQd^^MeL6=%Z&o#)-?&>yrSa4Xcm{@k2*(YKm zt8kLHexuOENWQlel*q^KIp?9>m@3gr)llQ2^lRNWz0>jIjetO)mA1-1+JOQ@AmBM) z#1LqijUnPw6?!}qmu75822^xR@?GG$b45&FRnL#Ju}mM`2z*|4=yGMYK&OwFZ%IWZ z;4Fn~b9Ht7@xe(x__&ZM0#A2_6^pwxE|ppD@LHw#9RR|uBF0i~sba@Lz5wx}-kVj% zi?yt=&FO#lYOQ-FUHZ|q?&tll8QN_djC}^E2%pQ??*X(38RMGe17O&3-2RcnYKcw~ zO(+1MCJ5ohLGLubecY9a227s=S6zlZP}1te0QxW%`RTBtdWCDneciXP{IukK203t_ z#2%TX>A5*w0tj*;1`kJ~NL%suUTN=jIkm3=KCinSpmPhmfA{eojlBB~9C_+4#|0jG z**(q-fj!qd2i!mTHSQzMHg+wrIqD}rbe+CvFjU2y1_>HHt;k{ zE)D#66~K!xMP$1hqRAaBRViHAr}V(IJ*Wm0!5_W@ud`N<_9S$skOglQp&UAM3(7XS zQG@E*+CH{a7p6qcH4*FJ0?~k<-^=FFh!K!a0iYT(e{VPV}hlKSZB9gCdugCRa-l%WRd2KIH@+&d?vyV;sKtBT}!1m#IlD>D_`PgEJ zv+6FhI7-H7Vtv|9|nlsv(=5S0{Fb%{hn>(Fn4-0${@$j$Z`^qbb zvXL}tyj9D9m)06KgJuh#8mub8_QTezYJU5I>ntI1 z0e|Bz1r>i6M|(H3%F+4G(W1)m@B~25I4`jA42#-6pSpMHNY5UOEVns9FF}%znzGAs zuJ7t+MqUPTNR33L9aE;ge5x5)`!yk{^P5f3;Mr+2)_IdGc#C?fpwZ-CPwNsjRE6ez z8*N zN~~vML#C}Dp`XuzR9kB zU~~|p>HCdv9a=mojzLyQ3>&M70#cyTyVJ36TdU)C;Sc`tKg1b$BAzzBFtttCQ z-LN%<2KH;jPCK-<7PvM@Y)g6eB{dUMPSGDM6e$^$`Y zbhM4qNk4#w*J=*fWv66ZI?>oe>teQGoP#AGeIKr|WniPjO-NeEsFhFS>9E8gga3#! zg+$Sy?W42d^$|RvhtwkffHD~`jRP?skS3x!Lr5wwcpSVMkM$lINiM#N)-=p`T_59< zT?l$gW%&_aX)mP2UIN(e&#z>Y@ldn^rTYLn?RcQC3cx#!2RyXC+6JhYvEG~=9{mL@xHc$vgV{gIIWTQDNRByN z!Q~T9zlRSr{Ak-F;NZ$aJ>afzBru^1+|xoiLrUA9?~hElQP`phFaI*ydg}6%CA7Np z!A4?n%9tg8hJFa=4P^}txY$vxsa)UtvRjYp7jGX;T#(@%n(n)n4rkq$l#<-vk288p zBt}w?rh+}zGU3=BRRF30zCM`CJKkeM>XrrWSMx19*PN3bpY^==$blT^&E ziMS}a*ED=6nkj@F(W~Q?cFspTBC)cPQQ9ppRz8f_|M7gh5rYaBk@-x-HB3@%9>WHy zppS3|(-a9-6P?=c3;Nq0(D|AjM%^1TG+3h_)2iEp$t(nBV3$f{YCuYlYii=D|(}#m6Zhg*I zP)Xxvas@dh^DY;!CoOnGoEYHFIdF?xOVLgYP!p5!j(7o1msE4Wg64eeMV0 ze?E+U7_lI7OSvN5tF@epGg5eT(&32-h)Mv%%`^PWtTLwhr#KOWvbx_356J0&Waoh9 z#(`Q1UWxiQaXuss;W`AINKXjDO3x*d(dOcQz^34w_hGUr&D>u5tZoRIp$puy3V!Lk zH0!m2M^Cm~)4;g+eBRjS2(#^pffUQs5{E>z9pZ70^JoX14=ejt6XmXMOuO(4kfB54 z;ZqN4SrTiIkcYsawhnrTN9geka!)6cfYH*|U(3$t{y31t&?nz3=qYxsfE6(nZQ-u2 z7q2fWUK}b{P8EiU#EyzhRs@)!)!(;}fK5tEl%1Ve#7do53bOl9Fk!I52;RD4+2Fv_ z$`}ul%H-8d!J1OfgKUMC=|b$|WsuhVXAs^xZo2(=Kwn|(L)y^~=%aWhks#P1$FUPU zm5BBiBs)!s+ca+Rg8f#;$BHWC(lP{Qf6)tgD0QE>Hf#N@QZ70Sn?!kg*%TWSuvSmd zsok~knC5a^;0FG?^8|_(nfUVVkHT@~nb9&2G&k*kUtB6hc$45FC_?urB)os;7ss8i_3^Bwe{1_XK(+BAxCmoG{ zrOt!o#8UEXX&U{L({in#`-rIZN6@b;FJid>F$ zYrwNv=d?^>w7uQAx*!OddBr46-^+@8tkp@xFuV&lC?{;ty9DWLhkR zNIbSU)`%J)OfzOzND!#r?HJzc8iy`O>%}@Fx`vI^k2w&@6}n7#p6gTNyh86}ApUK4 z;5Pw05H|W^1=Q!*Pu@RZ`{T8yH#Ma?XhtWfCLduJDIEo1dv^79BN(v999^hd_;<)n zBa}Ktv?DO~T36kcTgyk^HuAtCgj~dQ27O5EPco^68fhO^cocVjQ$K`YbSGx#BMBoa&8) zZNz!=ac^g6+HmhfY#}@asu5>F{AKNEGcNTA+0dSfkgtmtHu1Yomn#f$%a$nLMH12xw8bD zWH{dr&mgckk2WG1x(5g)k)y|Xs);V@frv%il4Co2Y{3rx{I^N5g( zkXk_`3C+vY>QuE1om*5j{2N7j`W&%C*4Q2Un| zoBHHyxI6eQViGQK;-!Il)%xFefPkpe04*AwymR;mDM5jyZTctX!{_P zSXzr;pL;OgjsP@fj7?62)4JHR3hB0Yh#N=*KW?J(sCHYhsu>th-21#FcBGT z=E87qG-#^jrUh>7pDiLW z4q_)<>#M|w^^iO#=})>+&CeE;eSd!R9}QB(eS*d-LJlD^h0GxLOuq@+0@7w|c_0RA zh0Gn9!Z&DwlLpiu)vZD=ncfAmB0@K{f2T zO!@^K?i`cJPiW}l_60%oDvZiTirc0T=n(vz4r$3{7nz#`)IH=g$L0z@{g3?^qr(F_8T!-^y^4? zhnf*`*Gl}qD8FiMdv6XiRDX&j2s&AG^MSWL2OQ>)0mvCFJzV!vz5hVlPy%7@OHxQv z#u}Bb;6_nx-gJYlgGsenzE}FMTn#`9+X4tm8w7acK(hgz5C}BsJ_+%j08A9jZ+yZX zA35PGZ2vxK$E!nwjsS;hG7=mfntR9tHQF&cdhM2GzVl}O;s7^NGsYSk4I+Fl+G*jQ zn3=?j5ftqQdvUu$V=<->QDxB@Xp{>yTy8AaFQc38gr7^mV6<`Fx5;WkQhGIdO-@QX zkN4bQd4orAwCmXX4IZW)=`OLg^y#k;CSW_9d6isN(3%j)&#CY|r<#4oiCS!QLHu%Q zvoZ&_dFcqKPWC!JBHu0eHk9anm{V?mtjVEZjkxx(D;t}}rOLV(RlcxeFlnVW?c;Ll z9*+grU&s(kUwB)1^0A5gifZskb5iH*E!kzUxt%SF|M*;(2s=7RfF=?qBGUXcdq`wv z>0caH6#NPR>R`@$CwjuBZh~%Z6eU#@IszyPuR?VsC>{!F5$|oTjEUylU#zS#V#o*- z3Lfs8_U^s^5I>IyAy+sf&B8YILbHngwm%U<1;QtoTQ{IUHeVV2eaZh*`v!7>s?dX$ zk@EDAfDL%+`nY~V3X`{?L8;=ew}pT&Iztm>TDB4CnQi0#ANbDS*13-yP(lm-Y9^F8 znScMxA&ERg%?0i35Y5cT zGO3^u!N^$(o1d46{(1iYeF=<-6A`GBN2mCf1D5+20c1%6W!mit#BcT5oTh6G07M8g zSBFE)G4Fn^f9FVS??@C-6DfB-Uda3N@7)6l&|DiH^T0+LOso2`rnz}fc+olu2l#ET zcsb&(59|8LYXzQoIM6ush;YwPheS5Ex9X^M7UucEH#N7WV`-boVA`gJlbJuop;d~Qd0Z_(( zuc@IU?{+8J(R8uvEH4EO87~4wz@9F5b?Y7hkok`)DrT)eY5SK2P~7@^#`AIyfpYn5 zvscf{4O|1{@cA+K0bTGjI$ugk$^{UQ9K4(m{ymOD0Hj~k)b!ky1fN5WzfY{i`g{bi z4X@aR*z}R%VQW^R2(oo^E2}cW%TbP?ty`dFnGB2&MX>7x$lVtJVMC1)oMO&kw)d&6Fd)%_&1Y-CwzUC2LH-DUwYp0&vpPWok4Ab~&ylzD)WZ9|qU1fccB^ zQc_{>v$z#hRaH%a9{WUI6ap<<@ zY2o?mwx5<~NdX+{%Q;y>CT$cepoG_CGypDlq5L?xVwe@LTHSeH{^?*+R4N*e42Ta0 zbN~dIx~n-Ifh~B|DDKw=`W?`;+yO$9NkF@P)vTcn_#}*2d|m10!t>&UA2qRn+b2^z z%7s^=HcuQ7#OP1ZqRIJ`NKvA^@j9&GC_a`p_u=L@i$lkG7y!EAc76uB;%R{TJ0c24 zyc5xY#N5(-1(}c21imN7c?E$|!;$*b*a8wAR4qc)t6*(I9WmA&_=cEmFn#g=_xuB7 zP}OXmuizuPDE_f$XmZ#&@X%S+i*woe1}EFNRUR zB6?O90bxwp*9N`9B>QAoUU9^ub<4L!$yfjXbr=R;cT^ zzfBkaVK~Kuaw12MOu(`5WcVj{i7o;mMxQBQk{3bIH~sS`F&)?`A_r&){=|NzV;loO zii9noit5Lu=U(9>^*SJH1$r+OAsR)Fh7=3b%cUJ1Odnfhu2ILjwTdC?9b$ZXBn2@xDS-Mgf(g&<6h>B zcst+j?jQ}iXZKS8kA3jX7Zlf!w&iyS4GoRO`*T_-YTKRMx_$lsVTXl@6u=jui^DaB zXq8Yc1*H51H#T&Ug188OpN!5g>KtYwpT?X53{o8ighL8S%-RlGB>Ye&?K;W=Sfp*4 z?||iD7{l`QD&{PcpWYfFH4WYHnI>qpOyQp9f9WIV#xoJmXavkEcfF*~AT2%D6g5XD zR362xvducg4A{WXOq2)UIj3-1jS!IL!?HTZb3ve=jrbX+I+mq%0wXKwUsc=Lo!<7y zbQx9`bLYeufwGS;EjUEzqto;k;p!F4nivRiE~jF6Y0PBX5AVO;R7`+3`>;C0^$Zv~T%hqAUuBs=x)mX| z2&Uh2LvHbdo4k-0cc@_7Wu39<(+c*9 zGH)p}x3G{_+*FlweuL#c;|WaZBQ^?`W3r983MGi=du2+aipz-A0rt}1g-Lr4qRmd* z^}jm=0RI>n@)>;DoRD7pD(7a43Cy`*u@w59M;XWfe@l+n>VRDF(}(>CF0Yfu`L;AA z2F*%KTUx>$B9>13&O38lrIZ~p#uuO;9CEx+yO$Fkm;@xq zCfBm8*tY?E?}7gaCDorFlu#*46S>L?M)#cpZ=@vK)E zE*{ho1|)pxs{x=@e=8B8EI5inIQs?=^yWFLLfcQ8bWE3%1c^X{FDN`K8_5?^EUX@f6c&~8f`dYgE_J8qCyA5!Y(cIX_=nHE)NTY=7%PVyFr}BdZC8&3T4|=?x@MvheNHSjZM>Yz3^4B1d8Yrf)>q>87T9 z?-woVf)f=LfQWv^h(5w*@&TG~1hHS9gv+XJ`FiS?{Y?-9Lq$(ISstk0alP-BOo3ZP z_dN$YJGtJ4`wIq zo?H%*510`@@dbpgwo`q~@HWEK4op1>Y$&pQUZ2o29N~wv!i^M5o5yF{_U8-}Ysg zh#-9YBsj^kwDxs6sSMW-W+JbHd}eL5;XeQ@ZUle~?E(m%I2ZpS5K9aY7=^^`O2SpI z9X|osRk}XpR-#-#~z#EEy&wlV?ULJ{sNrkJ8HYh9K=gJT<}#hCV>2Dhb;sB>HMc2prNU z$Nfbf`KMqjJN~}Kx=X?#gbx647B{Xz3~5VW;AVKs{~hmZTFEwKIc4&NpU>k^O5&y? z&yeu?;A9;^Xs;})3_h0(ejv_{R<%6^4t|tY6hQ7UWb^G80c%68+ zf@%=;@kD)Bu5a68bVzIEU#n*=G)e7A<#h4%Nllj+KQhH2=zd^LVey@{L!S=w0{6(`N9R*ha4 z3WgjbTe7H!^^%|#QyXUdK`2>3CmbC^D0s6E34ur zMsc4Tf=(BJx^f@OWrcJmP$Kx2LiTN+-2!v#4rbbB4*S$7O~YBOSFjoIOK{!+2P#5c zj=}fv!thGq-WB|}ecyOaX%f|7VvYhYP}p%pscRu3&v6?hSk0;$0zba(P{zD?;IJ&^ zdL$!&+n{OA8AK+JrrFVcQVa79{81;OOx#wMBxjn1ug_1A28Emm2bw5Hl+ijFy{SS z9}Nno+p~|yi8xygELtL?=;^A*b_@m;J|+3Kn5*OgXt3TQ*H;su)e-afuT%RLrWO*z zqX%t?4?eEHJms>~jT4wX0%gvzuC44R34{3Hcs^R#B(rGTAf!gTrvSop(_DZ9D1!^g zoPwrWRAhY(RzI@~qSxPl5xfc|+Z!y3osRB)!P}v!-BqS5Ko-{nf5)Wc@|4>DSm3ck z@&|TE>H>Pa-zyf*KZ3%z*U1-Q9%H2Y3$Ih(x}X$)#BISSHlT<|7CRz* zWFN|hZ^9RWZ_H;5InWd|i3;YP3unbA{_wwMa)3=Q3Eg>rSRtc%>c|c$oV($)!mM(Q zp10mEP;hOO8~y|yzq|fn69NBaQTd%{m}so1aXz>BQ>Zr?tdoc!4;FtscVBN%wg@&f z!e%7Vuo!(_X;5Xv00+V}G$sZ%)}>^qG!$6`szb=<0Cg__M7FZx^#AC3>!_%^^?g{8 z7^H?yff+yy8tH~11SJ%tTLtM57+PR}p%EmM5-?D@Yv`d%P#WnFX^^gWbK?1(^PKOy z)-2cj;mq)~~30l?V)T zhvl+1cLeqcVWko+t8M~LM_shj$7qoPUhES-dK@~rDNp!N5=lW5{)h>NNaYW1v7x5s zOk4N%dvXPmMFv%QZR?ydWPoZMvC&;vc^18ma!GxV%trZBbSN%cHH|Bwfn@R^GvcQJj3e^y^n zmdO0zZaD5FGlRg~Wmok9@>L35sTX4KJ63^IpA(<9Aai>ZaE>>qv}a4Od)FBjw7zL} z@e5CC@oU@k5I$je8ze*Y2Q|71$Mci4 zST7=j$H|XE`=Pid-x_fPdsEtJ@JylJlT!SBu2RmGbt|7zm^$5mfB7t4`MBWP!Kb&d zST5X0to?K5+wnu0{~qc8*EuwleTCG@iT1g2W^LIv)MJ<|y4`MAvcF;d@XHIQ&wpsx zpNc9OLsFL*Y{V1agL7t&q9AOgv#GlDn|pSsG=>_y?nV`&mLu{*FP0%z9BN~tsLXil zU2~gf2mK^cGvvTc<<#-XxV2>Bf9CcX(VPzrd;!SF z9?pe6^bJZYAs|ItxWdyz+oCkU+v%zmm1PwDZ?26md;|Ayl<0&+C7VItKlSnj^fUIXfID$iOxE`qm2?{W0gM-j`GYz+4P`C6rZRX zkvZ34JLc|7^37!gM~zxm-Ju8~i6bo`7a4Dhbe3fgHs<=$A{F7e$ZpGrH}QT}I5Opf z#|x5l%M|ap?W`Zf46bk`6F7eQ%yqH*9k(4P=dohD4mo~3Pd||`{1z$srk*A~bmKV> zG=^?A_s|H+Mq1Mh1f8RYUGf1k z+b=019`QBxGl4|7y3LnfF`@>zG0MhIjE&a?o>Tl`X0wGC_+wQVg>2riK|6BLgP~T^ z9dwTD^slsEy<|<6SY{ObAyrR|OMoH|7cf%to$;O&Xu2XIA4eH`MBaSsS6k?-Yl{qb zU;_T~!?NkK!(oi;z<(v1H0b?BICl8I+}Ka5w5; z1>|0cQOrjj@pI0(<56BitseP8*p^7h$LqX3l$DB?&f%}+MpeXRjl{T+BrS`l(#}nE zY`B9UPjX&6Zu0I-C?iiNi{_zuYy&WyOlH$@rdQ-Akh{LpB}MX@w%4I*7GHz_&C~JV zhr`ps1<~RI3J#IWvOI}{$|Y$nzv=}=P>gUVGQ!zEb|IkWJ@(6lZDAc#HOX@$(>F*u z=V_|qzB~jnntbyc|&EH>eO33uG|r<@(=2zPu&-#9yD7( z63OCs9|zZCWX|BaeJ0iTBN3ajx}k}|{fg=QT1^j+5kn7;Z1pGxDi042wPun2##9GV z&d&QPtb`W_=Y9GqX=$GB8V(2jcTYAo`yaVn7@E+z>axIDasKNS%XziAZ7coyly(y( zft&>xjqZ7VMr!e1&kkgtl0ki%{Ta*`!m6mkbd)7;%@hoQcxS^oVRmRJojDR>fc%c9 z@JC){btqT&0V`2E1@}r4nNk6dOEkYDiZX7<63j~J7e2}3noF`rC_6=X;lR3J3$QEj zt;PhzNmFT`2hG6{#?$o#>`rx~C{5I7#v%CSY0XIDE;zcGLL~_qdYDZ>9=LP&L(}Um z9fTy}7G}+iNwO{LjMISAkts6_wMfx-KV2-G5V!Elp)0BMp;>?2>Sb599V#;R$>|ES zzcz6&|FSvt9qp~DKuB_v*#)Q#j_%|^G?4Gf6T^_HDYBie8aca{)Vthv7nZlQ6ASR} zLRf4wMZe?*VGV9GeZMcn{*%Urh@}!033DP7sb8CP-Hh4hH*XVrEX1_uy2BeNcY%q z;6>{c$tzu6yZ&u>>Hyb zzOGcRvKC9O&v3j2O$F?2yB00WkP}BVODQTO(Bm9|CKbc`Ra&uYw-!WaHZ|9WXJ-dG1Pr$o-F-^ zm=M>m^N+a#Yck#S$S8*Wdq*OAfvI->-lycG+a^1kp}k+Eo3mJa`6uCb|i7!j4N@ zC@nbv8G94<3PMGOGsQ--_f|tI{6&()~tRXk^bQ0T7hPlTHc|K#>wkwIY zY03tzJ74ZUmOH4OKP%opiWPI19Mf2xstGan)|SX!lbjCky4j;|?sy^4Qd156;VZsu zBK9$-?Y*DkU#I=wZ&XRIA_QIXVA14l_eaemXt;$5Kd+6K+j61kcvap)(6ZaGSS6FP z_bA%MqUg<^*yyqH;*Sn_@U6jWi6a>r^@h^V=M%lA;xc&n=LdUFPu2>>hQG>jNZ2GP z4cpiL7WU|5@IL#zvfq+}$Afs8bGF&zz*GIcR+fBJ(t}%aZ7>ab3ziSLlB@RLC+sTu zt&g~nFd!0ejV@*5iof$tfE*eyr_73{ac}OsKFvp|oiW73Nr>Ej&&cDo5oaR%O-6#T zv3}xWx3{bLoAxqeej+5Q;v(awJa2YeH$Us|bv;{;rW7B$TF_?H2%XqIasT`f_v?50TWM3qspnMmHw0a)eGl8Y7}s{HpDBKg zF7aQ>TdZFdQ`cl`pZIZfx70X%Kl>B!?ntN2ll`PttgBKUpbZsV>SG zSQ)1n~a`VFa|2u1Mkl<%$RO$6lbU$|bJKT4_2f zULTPm?+-uEW<~vp$;zMFP3Do9JnLP3qWY;i%AMJw#aO4G&HLV;jN_s2_DRD+>vIYQ z6LHk5YbRQ}rF)NCf+J0S7Us_eS9mOoyBlq`oge<9XKWS|Yp5sUX#FZ)D2%Fommq>{ z^A&qpz-?0B$0Jr&PPt~|8L+XH#;Dh@L1HYrBEY+JL@DC8KKSXA@hqx3-Jx^uDZD5j zD$CT(-SZeeUa`Y&o&NV*SDaSJUc>GMJC%S4Dq+yrO@mgy+U8-1+J3_eYpc_0$cs@(Qt4WP+;@vmTCpY~qV|@n zl0UN)uZxT1FOhK@mS_4}Cs~VmEw$NJGuaJHTEmzU=JVOoUrOAY{^pTYU#5YpDiM$> z`JFv0VPB%~2rRAW|9r`Ivd_$u6K+0mvoN?B))n3n?ZmAA&{lUV-E2Wf!u4W3?|%OE zC)tnmYY)FY=@YcC8yfqym@oo`*4*sXe-Upa7r?+M)#M^!l}OZ&RXK0)*^}|?ZWep{ zS{`TZZc^e;g~GhD@;aZ9w#%)Oy>e4nQdq)DH$<7f*<9WDy%VP6T-^!n_REMpJEiWMU(3|s z9Q9yZGAtdZ_MO&l!PYYf_frdP<(}J44GQh=kbTFn9`sKw40l!kP8&Ua#whu%Ss%Eh zcew5^PP?}7G_Wf(j%0NNfTxI6B7pT%u`b-eU3i@0KepJ86j`m$ce-BScAAKcxr-Sq zpTx7p(k^47Lvz_Dc*;gxCg3wtgI%KJ+*!QTw*y#=I_B$GqGOD+F_a&q8)|gzB;F=~ zN@o3&rNk%)FYzw5fQOw{%hm(KD!Y7NL5uWWeN5Z_FTDwIq5R*qwmJTO_!< z#@jnit3L}8$%=hEz<;0VtG;-%9$#%&A)JeYaiUEMK>4TFU5-`Dtbk+L^Ixp@03wk+nSGI;y%LcTSu5s((Fg78U7 zZa@As3KUJtp-(=lQm+nVhw2p8zj*n;nZetcxw|Xj?mF=8u8=uDYys)r4q)n@4Q7BQ zPbwYG+0d5&`9(}|ODI`oSHeYD;GN$`4W~QZ?i4nJyS6I5?_;s)QaequN3itQ>3I*3=!Xi@NmuH&E2_?C% zNBX8rIMGt_`3Ej*;Ez?#K70)GV`M&Rx+O4#fotmhv|E)}_APoTB-f%q# zo$%lOobGi;7&4XN{?kp@dpXVHI{P_Rd>mi*=7VH2fMQqCiZa}@Hd|R)aR(_!dh6rm zm)Br7(7>;VRd0JXohld>4NKlzpBV9P7V1jm2CSDix_sCa-~!nLAfWJ3NzuN;u7vJd zO_N`opBF()&Qn_BmzM$Y6Ch8r@_s1@T$upQi}shYk{ceA&M8dOaT1q_Q5PpFFMELq zaXiES{OvDL9k{CrCM?;t}g?{#z_e(XNnJ#8+H?0@H`FYG?-lDisC{i7RPfbiAb zo-@P4exK#w#0|;WEog*+psv=R2}a>PG;Bma>+HV?*^0quW0#h|`>IP>ZnX;EY(@yXtZsr^{Vvwa}m8V4>svz56| zdM{-9Xg9}be*TJJmpp4^xDD&-V)b$3TS5Z^f)rWa3Zo(6oKd zAtw9CnT~A!JyMwQP{k7Clhsx_kf}gmDtmXWz4Qf`3z@s77&rjt%v2DM-{3E3u)U`g zf!I&p4SQ`Bg~J6y)_WU8$(AK6~=;gUO`N06J9@NbW z5{|_44?g+Y>k5_eahtFc`LaQV3)7i}%c=hJ5J6p=i7W2*`)P*X@fVh}cS5#nz+QxCAOYRHPSV=2-fpaLzxET2z)YSc7S9&e)eV}%VtWi6xv8dgYm3>wo+7{>vG>1rWR7$dII77jAWE>&&yB-RAynq*0g&s z;<$}#et@753-G_nK`2;j<*Z5Y3BT*ogO(stwg=8zGfl%q<_i62+h`V?t>M&B%lO;Wn|Bc(FHakzg~M*wu#=$3aDlOHO9;s>v%zHL{`$m;(aQEu z3YYHEFzXIXmb%Cve!vW}9monMpXzb#aZZ1oO57A#;YrU#*aU#tPyW)b6rZYTd#)9i zxUCM4wIxlsHtuB~5b>&D1vYrwd|Q#{Zz)G+&dcq_1~ zM#yq08)!XTJ)C+N8S_8Yo$9r#2%6?qW}huN+TfC@P(Szwa(9mw5|Kk%XOmC7wQB-0 z#yfJbK31AbTal1tDj^gM#e+bBa@V&)3h437zZU=sU@f?m>wv)a=cI~;_m`(J5JGR3 zuvG>NocB`ZU4^PE5Hj51R z9RM!*GgTBMaEZCJHn?!5kHZ*_zHdDWBEung5eChk0+%`Ym4YZobzYW{ax~KlUL#;8 z9Tz+CFyFt7agwb{L1S9YCO4yRNO(D;G0*JSp1`RrI(zQ^DX;EsI6v+NG-E5|nGSfYiB!?1 zr~!B~-u#$+&w_rywZPj(otq)OZw1@?uT4pw{PCCxAZXiA_{AxsHk#_vb=#z>m6CW< z#?N%}Ghig$8K%w>2;`Zws?u2CSJ)z{o}rK;C?=h}ejq+sSYbbLjBz(2n?E;02cuq5 z&~$I^fPAdIlcTN(GW)}aY0ho*=bJdYv(oXkBqny5I3S$KHm zDy#N>1ewB9nK@z;&9jq(0~2nLF}Bq?MDE=|6oOV@=Efv#`N^ROdG{b%a)_4ICE(vt z49si=Z}xC77-!kS;ryhi*MQbKZ89ScB*&*K3n9lKwM=9sd`lFUwSZnStb;PNF)-E% zkR?Ez0i+{F$FV-SZ!O(q8k~}ebhLOl$$}x)A-BTubs#IqeQ#fp z>6p}LMh*O%L0`GUf^l9X$Sf3srlmjWv<)e0 zKfG>}t6R9=FSg9~-nkz-K%p<-W7fTS26FTEr>p_`Py(KTVVTFLQ%^sJ%I1yA9?Ap^ z3JUci{$3Pc&$u#2%xuv_5nQo7cLURI9ZEo*X+>vp7mY_}@(h{B3Y}35R-w9sH|MF? z`c1QQ=f`7>J>I2rEUxMH2)kYEmV_iyo=0FO{*q5vuZUjoay#q%@YxYs9i#8Fl!SZ$hq`9AN4)%K^ zE)jYYl~hpYj2vlp9ctb$h_F^db3C|h_KII)gE4YzkZ4M-C+eJV1iSUs8^~rY(w&Ql z=B_V)`1q{9bX~*6RP>WmTz3aT zW-#=4Em! zEu}=+)2vy=zJLR;bmv2>DJ6MW(Id&&t$XqvY@Pm)0PN$tQ#Br&%oB<{d1eErmJ3nx zDlOTZbS8S8H-9LLdV?4t-{@*UwVqyHkhjL(B@B4YkqT|*L&WERU%Z>66jL0H>;TgG z^=4<6dQeKr)3^-oM}F?uB-P}6cJ65wcIlG98{0(Z&3TS~s@ZtDImOee z(|``$tq);qJGle=?JReH3By^;oqQm-If$tp`I=|9-Z$HZ^UqT!u&~{uY&qewVNX=P z7ZHLANwptv-#!rZ%Hta*66Y(eWw zii#CR@nlZ#)ZORgm+%2;Oi5~_)h0oaL$Xq!Q(9P=n7kFv3mZavT zVvD9kYg=~2A$lqhjs*73`QUFYrHZ0FN_Y>5UvqawxHU6u({NqSc0`#V1TghT;jOkv z|Gu}bP~`{VHh8SACQ@6J08>+J0ORxW0)%7vsHikKCml$KEWa^ zN)`~8s-u-naa-ro2ZiQm+OoKbmroX~&NKauT2NqL1kYO$i+P|$pzn63;Ze-tKAM7X z5YLej9(6j0s044X&8;iP`C-k7w?nNK(mbZN!PbE31ijD6uL5~PD?8-F9gK2|Fd<_! z17_<{#B{$=XK97l0bZh>gD-n`=rV~(Q#Mu8yUD=QUPxZ{?x0sE#|%c`vFgSzA(#Kb zc(3Aic+j{c139F3-R5ZTllL$GLgYzB2M+VvGzCc6DjsR_&E|GyAajV(drJ! zo+pSy?H&Z~0t1pCM$DgqP!jM#+()S3er^gPIN4P(>UX19TvPp_F{-S>TwMk6Mf@Pj z^qWQcfrXfMWOk~F1V(Z7`Yb+g8BnBlhT6ulM0=A*R>uX!pf1?A+8RJz z-P;xW10B4Ki{RA+?Rus30!^XN&IjsX*wnU>G*DsG7hSDBJ+y~TF$(z+P1@e^bcqy$ zYx48TKhB;V7}$-|=fqNHH#UK)jXW32=+85i=D7`9X0=>6vW!GmQ#sG*N0mfx8cuM; z$TD19hx6wz{1lDS`|myS{Q{H~2c4>o#D#UtP`A}z7Gm3Ta}m6K!t`<5^I}vc8z+0K z%TmUHB8KYKJy~g~fCwzjMZwEe&>kTz{{wWbhQBghkof#E*rps0t(TH{f>u~T1av7- zHiVxAlb>VF@bEmQsJVE=)L)9`cnm~PZhBH@DD-(O)ZcpkN)s!TBYaGr22;wu!LkWi_7)#7L8I?GBz>zp`6apo#MGUv;6K1TI8J zfi*|Jbm?Zi(jgsi-IIx?HH7v;uGMYSte)?!wi9X-3{kmF>6_ zXlDXzce>fYGN{&<9T&& zf$gtA59Uw~B6BHSVorz(EHy>q(xM1wY8r})dE*I%Fg=Ns@1ps22lR~PT(4E+qh znQ-lX4|B6*<;Jg4G-@1XJ+Yzi{_Wp(7b+PXQOPpvfoA{k>0kHY|M-9~ z;Rn_%^Wd}`#|i8g{l9(SuU~)j1V>t#;bIU*J3HtVA;kOlh3a4Pyntrb*g#H(R{TIK z1M`<(rT;$5{0rZIUq-I78;qZtEbKkZ753ue`i714Ux41<-UKT((lR#Q&G!q!zay$? ztvB=<+CbG@Ae$cX( z)sd)VdTD(Bf4=6Qf9(4O@-6vyEc?fM9FBh&ivRv)Z}@Nk_%{NCY0ob${IY^nN9*p| zw=C*UG9<3K#oJd3Eor_hczcPp63}#35Us2I7U0%in-rJHhzox>|F@!XG!X(6cB6;H zEYgzV}oxCX>K2La8x1fy0|8(Fo zUt(U$6$rzMfn0Cs_i(-;Q>dXu`yb$SF##%Y2M}vp3^*}kfbr&FKT*+p;A7%<^h^RU zN!J02RFN*_y>xOKm>&Dx4OnEDdjsN`-FVrHeXwd6fX*WifS+xS3t;=<`L%fJIo}pl znyC;v4hvDw>9!5KM-T=OsrM(@ncGtb%L-rZI`?r70 zZdLvQo9nj!#2wuNJ4w^a+xfC~75CV%vuukjo~rYn091r$v_?*H(Ib65JzQlc3G<5K zomWlmUUj8`Dd+#z*_T)c(4R&7H}(5rG3ae@fXB3FhnCg$m9`^&5o+j%H^g6d_A{8ps<8-%z)?VZKrJU47SH10+kWPEckWy||3$7Ey&O<1R%5T((2zM> zP^@E%7HuwQIQNnfQNN{IU?e|#cUV8}?xnz=r3J9%^MS2gN~)0O&Ql2hYper^pcs6M zL;Sp;d6by;%)Da{($K!Win8oXzlz*bQ3{N-X?hklnEKdDLdya?t!_gc+Ms zl1o$tEN)+ZqDcntgvn|0ryeU}DwmSLg&r$@W%rWeOa zQo!WylBKw0TZMMHbYYaS0;;#krQkBq4h}#aOube(6!frADS*#N@>0M`{;Yg&-_%%& zIBEWTD{A}G=g(G@gA~TJWcStp_}ahJbO;<$s28`w?WZqe;t$?Ou$XCy%KZbcphJNYuUY`R@>;=F;iq)EuJvYH257+yijGhl692SIDx`>Q1@)%3(|8vjLz_5h3MGq{CJawdvOh6o$mz6^40pmDW>x3q@ zo5#)8A_MrhPv*wd*q_fzb)GGITe8j zv+p`fNY^Wu2A5UlP2~k9u~{PCwxp^fe_N@c0r?MKCCT8@5%;q&L7Ub7TqR(dQN)Q> ztm`^I2TY{rTn1%m1%8D{m{G3#`nU;36ex+$_?xo=W0jb1Vce?Je{?YJWn7$cR@_Zh z(;EUn_HWbYEpjLZlOI9I_#UJv?JP=C1?0jL86Pb+T%0*x_W7ym53zl~Z#v~?u?I+<@@!oh67>1oBaSM5Vb4L<(eyQFa+sWtX7LR;k%lr6M z9~tnbm>jnp*!$Z64e@>yI%6D`9EA0DTX_U*a*8F zf$E2MaEl_kQWa|GGhfAv2>~b(ootnCEbCLU`NcgTvx<;t@e!qgGX+v)QHP8ucND%^ zLqZ|YLMZQ6`usB{^FY8fRYlsWhv+^jl6IcbTwDIg>_{h#wx)xHL%6~7_h(9bLAPZB zufHsInJ5}!(gbCm83QEJhE*^P*@H7|mPC>j6FXCCMLbg(j$S4Xyas1Tw?W5Vg1wfF zyIeMCU$m@K5a+>h_}zjnrmeC}Q{0H0flc6dOWemDX5Hrr$%`vAeBXk_5iP@1yY$W$ z!-^GWL6jldq~s(O(dNtxJMulSvcncKnFlXb0}fti#Q4}4g<1t!r7a8J*}``wE|kUP zMCth{Tuv`p4FAkQ3enS#KdblNBvzVWuZ<4n^ruV~JfP*La#TdMZy8bj5$muemTV6Y zI>+HrjA5J%lbFEzm>}2lvkTC zplGCddv-Wk$TB6f3`RTD34tq&(cnY8W<6#t*qVzNQN%#!8Ebj((8K%4O@a?daL6r) zrua}vC1m96HfSF-AEqBx0yPLF|E)Op2N0M&2s3Tj1KL|i`Z!PsSp zmsoH@Yesv-y+9fZOX!uiynzrCL$m{$7YT>en4^60d3h_UdFgaG9cxx@Hbiq$&$>d) zoo4aUk68bhK_VyBhzA1rMsu1X_(UR4+3B&Uycce4%O?%?XC>J`a54?JUnP@!~uU$v;p< zdZIatDnB}2HH4tN+gmGW3Dk{zD47k9r0O}}21$ZG|EOStGP5U^NaIicZcAxr9f^gj zs7TEMoqf^dILsgp-OxbEb=xjr7N+9In{JGF*DQdAVINC=kK%aVtg)nneOP1 zAM_bEn~bi3IGsKR1LUsJ*sYC)F^x842)WRFRM*?n4Nn^AQp@PpEyO6os739K=1!nA z!S+bx^b$SfAnci5jAOg-K}3-~cE>5^$S|<6t<+`IJk%4$!#f;yJoWAb6Wi9{h2-y* zg8UzqED0`D3)x9zL6dHYOrJSIH6ng7oj`*FJ!oJKa)@kNdB&rK=ddwx3&uIcqS)-H zqMz07h$TtvT+7MwRfK9E5Sv2b0VvX&so9I8A=kJV9z$VWown7_3E6*Uwe@p}iF8xspc0^f_6C46j zr3tuGFjROPOY~x3((z9a%ntV9HDly*u#<2FKgKnX98T9px4&3{RUjoNY{Dfg)ckcd zlSuB3ra6j_xUfEF_Nq#eLqEi}F=k?h~XHlBXwEhRRbCx(!s|>02>L zPHgYWzn3ez{XijEfob8WnT=#1)he*4*9_I*;&|lC`bddnTlmzk`JDA-==x-0&uy*f zGFF4v5u~t~>8l69&RgEi&6^GH`{f+wyEgeTeb5u=r|h;mb{6(T;?-Z(t>#iuj8x}` zG@(~t5dW>sglnckSo2~vGst)gk&kE66wi(fzC2>p8O%KYr(cyP7&qU#?qU2KH!m}sRle2a&lvcIaqXunP+z~jc0#v~EuQ!j#uE^$41Mg2LP5Gnt{B)N0a9Y`6^k6wFjc zjnG=cmB>nKQ>KXNXh)WxOH;w&##x z<}unD`E}dln#M3j&v8TV`iV~xLvOHK7iE4?^;eny?$ZJ&)UOPn*ynr#bFe3r=Ar#y z?|fiKIQt6q%N&`52SB(tHP-&v602scUUsKp|0NL?Lc9>~MyX7>{he4(6NSV;XsWJBMll5Pi1$6#pM|{%%r7g*-qB^6M<6S!@;Whk24lH2-4EOq{-J z;w*WA6l(LPp!p+qD~~_^BD!c_uus&IQW`RgJPvW%!syWsCtuIacmo~Q*SqnR<-gA? zz^O=Zk)3Rj(M8gjh`=ODSGGJsw1k`)B#Q!$AGXauRS8)It>$O{%*dtW9TH)`V57(E z8D!}^OmG$&u_BeWbW=5V$=Pv;3UUO`lFNwE*%37B<{lLs%N3w|-el9I$6-r_&O%me zfMdrQe_Vg1%}`gvA;Hy%Z`zn2TjTHyZOk^DS+Wxf4+68lmdP#B>y`?qbJF}v-&mLt zO^>gW@q|;wjzs(DY|}J5S6jh08FnF_f#oF^ z^L^5sY!R6}3>MMPn!7$AjAslW{FmF3k3X=D0yLxLt0}lH9mDrY8D~A-j?^uBbnVid zP25jHi9ODwRSPM1)Qhb58B#wdW33XW$u53z>+j1-(GZ07L-46C%8}e*2$=d@G~bBp zo^PUv8)VDD+s3^b2FFdMG@(F)pX{L;=Ava8VAjaXhP;~m>( zwbl|MYx#~h5a6S`fUf|RMWD&%iALDD;(d%!zHo;|B4=WEG$~pErAl!!f%Dq z>iUp=mD(R=b8jG*?~G|b%eLWAqT^`3U!nweJA^t!=)K~DfZ3uj#lF&zGIlJLIDL}M z#r^eYx&;o^)tPFnLR^vB5~^ty5O)s3w!T=a(y`)L-7;x9Q-6w%fCLvIoZtFfpknpL>+nbY(Wl69#wA2LrY+_5*EN#aDCHZA2tX zsiBqDNTrpaYWVy3t&HU`D+2Q9rU3Oc@!ZdT@@<)QTv>gUO0x0p8+kIxmeZfAPto6yk9Qtup{~lMp2()x&fq0; zts>4G^L;^))+%*s+cnI9F1Y*uScm_-%&k#Z#Nr?;@?}l)gj!by0%=|EJlnzP6IUYr z0PMP^8eXwAL+8TU@q-~ycKK%w2v_z{npuBPs(hf$he)xbqLR!_WXXwkY&)X~K3|VJ zGX?OG$?5zv+U;QeZFtYAt01ld7T?96yh3I4CL9Nh?;o>EEw?hZq(pi?dIr^CMG@d) zQ-L|=K`wyZ(g}GNn(jsw3CWKgSK)50EgvISSShr8DKlYoI6r{dZ5>3h#ILXAJFngr z1?n2J1KM8u0?SAU3}a%pVd{(R9R#%0#!yNqHcRP{I}lg?na|F-wZt$v8Tl@R9&sx7 zV;#`M{F%v}j?md5Dr(HB`hnt( zdt0U0!7JNGSFR{9DZU980ku*;c9&P*vK}=zSC0N{;tP;(>P>LMJ0n-6StOf5uE6cWmQK6S%}a)>jLU6}E4> zlOk%RC@j6${YftcKOz75qRZI6t->Xa&i-MA{{P{$0rv4%a3DC0%r3|;vjbS^ord5f zFRZXUAJO+Mq(UPRoH^ZPAvg{|ly7`+Ucl~~-X|ZBJ^!AJw?DcZmOEUQXWHo?RggWD zn?7n>zA!D&`_~xZuQ#mpILP0zGOL&FBOnw+faX&;C#B~*2ql*td6YBxxbnI1+WR2V z9aw{(f+=`(InrefoW5A<$6@bvunc5pu21;|F|MX^*siV4?%HitDZ1|qDy${!dQ`Z) zxXp2n|KjftKKKqiPFa(iE)(SAG?;$KO81vn@=b+*`rlGPS*a4jKT%o4ULFuL&I|VU zT&mq2MvgKu#@3I4@PdohD(ds-x$*C@9>v^C%C6nOj$3iJ?w8o-66HDO4{aP(hUq0@ zTU6%_sPk6chb)!JMl;j*{1?blBt1SxC0E>=D9+2<6MnwrlCst|P<)@-)cI7rC&nv&8IsQM43 z(%^j{m!nBYu4pDypp};Cfu?y?d|8pg>iG#jjg~rG!R2Kh`nsc`UfMLTJI0pruq=Y~ z#WMo8hxK_*e}?~98-FKy0kM#K$|z*{)_8x=$q;C8hF{itQLmq$PZ*YmKRc^;qt|!e zRGwe|-mp}K^;p8}3g7!V9p;+EG(O(tK$Vgxel=X|Ve*+qv@j~Cbl=hZEH%`qNX?Ya z!DxQ0)SbBYbm#Yo5<}qZDy{p$1h22mP(8i$G>>#y!q1BV-0qiw)5UcR-mu>M@m|N6 zio(#|?&0X<`i~b$5w1hdnLxODjn3wmV>1`Ydf5yV>%*yWc&j{VfAG%#&lj+YoK| z%h0Z2VBU6rZb*10fjsf|n{31AO|$&pIu#klDFia~j=D8xWa;Cr-VVMJF7>nnq*KN9 zX|A$FhSfK}r!$FmN_%8Y_vL?`kWO1=u$WkMutQs4Ol5yr|5+V9a6xIRUJ{zhr;-^Y zeZzK{zZo0Ka%LBoHu?F>fkSSo^AuS%KRj#CeKwc#kAnrhy1w`P=M}$H5}OYb_Z^5= zQ*XI1=Cb*22fLElyr>%&hYf_GJuo7}{U@D*HD0`po@ygiVj4xl(R;n^?nSDL-`5_C zREAF0)7Gc_Zf(Z0eK#S$WrTyax@JW0>Y}*7@d=MSYQGch5G?U%?|->QtTe-#LredW%Jx8NN9>0?B|DPZ66+5NDO z#P0YsVSkucu%pW<{lMXXTc4u8}LGN)c!U#x}2Og-ozaojZa8;rr&qBxBbL^ z@WYJ6*AJhMjI0Nkq0aIF*Ia@U9(ME0&Fm@8UTJ38Eg|#Iu=mLdNutY4Zt5Tic`a!F zm9jsOB&`Hn=)wfc41a-;XX6*myjuBSALyC4kI3Z&99ko9FK8S8EEF`!44VH@Wv`V=u;Ps}J6|9LLkv zn7CPJl5ifz`pEFIagH%FD%>`GIUClXp|^%x^m{6sD^n!iS9w3YX3H;9yr)#Lx$kB__LbQCu$0h! zRl>4_godnu(!r+Y4JM)&556T#m&fnQ&X*2*wvxLa1C`U-GC$G!O5x3k zBZA6sPe;#~f-3c#_(z315N+lQB=< zU?Vd>W$AioI`rEXI~zLFp7}rIbdxk%plqhhgY$ltvJc zknUy(XBZk3M7m*!Aw;@6r2Jpb`Of~nvvvP#u~@@m>V4klx%0ZN`x1N@?EB0mnrgjq zOM*$YaVD`oBI}T(tcvJoL7(vwTVk9fZf=akEV$8 z^U;-nhBnVZHCLkZZW1PwE4s4y9~j297Y;Ck)yD}+$vY2|?%FXwuduXhhOO%$BhQeZ zb_`=N_j|geOdy0yF_Ig!><85}g%qaw25|eDGb00UoA2h5K{h*1!OHF!<3ew($XjPj z%Xe?^w|V-BER?*XXFqaH5F$97{pngDM#}0)kD?lC3LM~Z%PtjGsAbg>D)tLsb^RvV z=j+|H1(VhgYnvLEXn;@@! zb6c!V0yi@KvuBJMBN5QvyYY+z%SPXuy}tSHpbx+fGSbI*0X=V{7MC@3$XLB=&O*Br zP4KPG(2znB`NOwrSpgV#f-K67Qo$mIesIJnF*D~FQcq>~%wk&N+-t&zV!NL?Y06c$ z#(n-H1*@)`4QMPh@oG^*V9(^-^34@UWh5RnoP4mdi0RebdrBR}EuCySmkx2@^Y8!P-Lk9e#=xiY^r$NFHf z=)XLME&8@ZciJlxD7qkf@)}kozQ*us87S@>hyi-pLc3XS73Ma7ST4FeW44ANQ#j4x zX>2|*ALhHHO@m52H6P?t2@FYuS>;?|85#o7<-h<^07UW=g%yP>FhOV-=^F(80kDz{ zRgkSm)x-DfHD4u!e@2R%EWdk%0yqsAs;;~CEqW*E?wq?n1^=RqW-Ssu6!xgxdnD>p zd(H2uciXb@-mWXuD${+&UkB*Af`DPMm&ey5Xl1EfH8*t~jgz0-XZQjb#bcm9qzi~N zL$C2Q!1zere1(LF^Fcn*>mxIfbK9rt=2dlGky9RYXmZi<8TO6u^9f06fjo~C3)1ln zU1v>FA$9l6hMFPF&)isJ_W#bFSX40qh(&Jw0xJ_5sv_0$ZP!2@KxG*M;I*d~b>$~x z0hfDS^V$N~88dk|t`VVD*DxI=%HDxAV6-J5^6Qo&wiHC9QVD%sD2ml{6cMVdxo>s^qA~NG$NagW8a8mV&sXly&B~txfB@rxZ?npE?ict&0SH^w ztWETH<=D!fzg+mggjC=OPcm-rzG!*1D?sL>Kt)RO7T0nhBWzjJeMq^c;S$RSN%0L@@WsSzou)@ zllf`1jfVh|0}hnrCjoYx>9wM@*fnc&5@?J=!(OM214KZFF6DnIb&4DgO8Wf*fT&jg zBAww0`JJsXqXBwcl+V4JhJA58dzkB(3g~NU*Ew03 zZ9dE~28J+6YUo$J4sM|_XpsoUmdU)H&zHdUOErEF(5yD$dA|I4YozeK@5Nq+5xbP; zV#ZVIcix{HoGJmtz!ceP+bu@`R&}r8nngG94ah>)#SE{F|x)om98UYys#H zrbfB(w!6tXI04d3!?i#z9J=z)U{v3L}x7Es_Ux%hN#bS{rFQ`7tKsg)6|# z@}R@LxdF`GJ?H}nD%-(`YiVcuYjt{~PI4a0cXQ1^(|r?=^yZO*0e}z@gVvMahoM+W zot@!;YX1oUtNph84UqD{k23<`KwP~9UGa&D%T1&B1flmINx(l|^PH{~w%Q=?>t+D9 zRh1cleTv@m1V}yr79nR07<_>bX;T2cep4j?Y^#~L0(#x(>oH{$$AE^B*>(5M5YV9Y zqv?GU6H;_N3>*y**G%eWrWD3QJ;IZiY7NHO0%T4P2JWP_zOZJvhJ-fVRzDTHb%iIv4O(mJ zN&)mg{%`mses%AAR8UrsFMy^F`TSNqsAC);F#6E@#qBXXB`Hw*=?sL}C;%a|G1t-p zQ0KVyPXIyn%o7aB-D36p2r)N+3hVXdLEsvIY&ty0KLc2++qNq*&v*UWb_v_wa?EZ6 z(m<(V{>>-#IL?iG-zb{GuBm}Lea8Seh@x&3fA39gusg-Vy^s}Z3FDLPnM=+fb}8P* zJ*;$J0O@hxlyEO*>AubrY{_XHdJxbZY5FW37)tx}OopO`lwJ3GJBJkF4leoWxeK8A zf9lO^JMnbR-baDqEALmSZ=p4`ufIR*pj-%6s&B{P)YV-{#*${l|9&k?zT-8pN{|B+=REZRPLPh$ z$B3PK+3Ntt@VMdL-q(gm2Swo#i8sFhV{kq=2oSd!u3TE8iinqo0q9{6lyGYiF$w6}t>9tO;nKa|Gx5luuMVlniD2>2s_}#o_}IO+00JU~nYk!*3bEI5#Dqcr>g-K07zKX zpc2rXD(`F6wsa0Bp(1`zNvz zzn20k18=nA1Y?{A<=hI>3gV=v#hhmiB7XdXba)lOaCL^>2;sN2hvLlY5;3bB%3m#X zNX}L+_obfybb6NtVf-cgoxag|d%Di=UyVn98|0StAyHj_QGGaF_k16jD8lR#fOYP5 zNDkd(b&wI6DdHrz&*nI}Kg9M>OZjauzo}3mlk~mLu@HjcN3DXzugS4V7hTGafru&I z4};|ba>sw#@SW9KCd~55-;)%1LRvg>67H@lS0HToDxCi!y|SMcKlTFKh!b&a)Eda)Y{Iop0CE)h~7NM?I{h9L%U6 zdVcLB33*PQAiscHQ3do*f6fHi40C-ux5D4e@LFfV$u0~tep-NSx%$2lJOPZw_I-Jy z#E(!$;i}dHXTIE{A-EAz`QZQd_rkn_j$RYfhmo5V?tTI0H>^mC=XM}Cfi2~A&*MGs z6_fO%VTB!KRPW!8%f9LjI*vtFZ@@@iBV|!Q_bju5O|yt`fb`y~Gq+O_W(DtlIDuDK zE$RM;C70R6i58$Oejbsmx%hSA!At!2Y|Fl5S~{NDF^!OS6dOPTgk3Z4^^M*Jz}v_) zBRoo9WxwJ2_W8n6sExnQNJX){WxuHn_g=~IOMb=ts z)O$5Mw6QW}D~lhvNCgfc#m=6(TeGMxq@ny9Sao0i#{4pHh)!DZj5GjbZza&?ek3t; zk2%fc8c=OKP^{m2@A_Wq#$xRI)3sv0FHGx4tDm!ET`Py{Ws3SX>V7@lU6fF|b~F`< z;k3c?$}%6#&`4+#^CTc70l|U-;f;IiypiWp0xDM)Pf=C%r2Www9dZon`DB9+KW>Ft zX+CQ2iCRw$bN-Uyf92Vhi#za%!mjp<4!QIqHtBx*o@it!1$7Xy39VGTk6W1kyB(n2 zhgZm0U8Y-=BVF-!;_incwW<$KwU){D`=sAP0D41~3*nQ6_)B2$s=(moP2BRXeCLvs z1fZmZRJe}Yuf$HV1%G`hH}E=%`o#?z6m`k?k9!sGVh(u00|;$EMvG!kSaIl8xOv(< z17lRAialOFPClrSqL3i*cm6da+4=ajo?SobHRrfhztcT&{1lKF@`-#dAkCR>O0YR&fJi<5)cYqa}h0!di zXt{$h+&WE_nop`hDx{ONRe{BN5C{ag%5W)P3_jlh5a-bndI0R2&^39q8AwSwuHEXd z`P;%v!0RX#wdL|)z&6Qp!I#s}wOXrE3>kTM9Qss4>H2wF_L;=<*B^W*Wj!*CJ|N85hPi_Pfw+gVlp#9x1{`H*BTCEbE`Wx z@nOvIKW>8?*IQ8{#VKzs@UKPx`-8x=dwJ&O=C*s4{Y-4*lkN;F=sAtVMPY6#g8#nj z|NFwg*HW~&s=U~^w#B>sV;i4y?7gNMnz(RgmRov!rjPT*FeyNv$EodnzdAAI~iQ|jm@ zNhrjq@HL%*XOGJ>$dC=>6@zb&fs)QYeb|J}?O}=b{+!sM-tAu=jZtaMqc)nHR`30v zb(QF_LTaeThg3Z}NXMY#{MDOZ@A}u&k9F%8KGxd$u@X8dqGGNnqtuoQofysMH$BD% z6!g-n!LKIk?Q~`UT|8TYEc;)ABh+W+Z?2m<>7T6N98SuHzex}L&%HC*z~7|?Yi_e+ z_q0-My`Mau77zk`0Fl^2WfBtB4?rx%}=$>8p2%V86)}8e*9QFUwJs5p9AGP??ZSvY9ieExzCHYqcs{*YMXBeV`wnXm!FXw+-NJc@>Io&!TDRlgOF4^f-|5Qb=t8;dm?$RM|L!~`i z>Y7ucvVP^Tc3FR=f}L?CJM(N*a^3Xf0evu4dhIykSj=-G5!PKf#FJF#%DALanG4^J(LvJM1Hsu&R~PvZ=x`$m3ZX46wmb&mfqo>y*b70I+c8nB z^iv2yk|CLPLIU&NTbVgfAMzOV(Ab= zcv^B@BmBENo>ijg=jA^tbecCq22olnE)BbNEr4h&E23fh>sl$3SheD&UT57Kp)3SYDMu_DcWBT;*+otM%p#sc;T5g}BSh=GL)t2-dl zWVb>&*czf_M2U02Xf>sEIVAujxtAgqZv#lgDveZv+O{tp8bvPFZd0sJsgdx;#va&y zW@BI<>)T0O5SjoQ=ruTLD^tWP$!_^*)sJBPm6@aINg>cdJw@Xv^_41Fi(!>dilLte z#au@7$z*f+od1QLGNB}RbG!P&vDL-JjMi`IJ){x-?B+&oa4uMdQ^mRZ#`r-x zBr>T2p89^nq-n|y|N8^?y3vA$h|DyhAreCCt=1<5LvLyxj+cW>C3T;?Hufd!4zJ=0 zgYKC{*o_w%pvt$$QK3#h!V~8uEbYCmYh=ptDDH`G3>HOLwYx@d4h_1wV;Go5W)2Bh8{Upc5m%uH)s_p7VBEW|+5E4! z{nz6NoPRX{`y#=?x(?rsMJUkKoKkVG`bLiJwxu$0F(zgE*>LEdJe<{s@W9h_bS+vS zWj$ucSqI#08fB0Jo9xNQJq-2F5w{HEe$mMEoy_O@D;b!Jz1Le60~(>WNrD6XUwQW< zOo@d*u78D_cvP8E%F3}RF|_FM(tuV9D3+^wmx9_l(?eWc1t#8v0itx&=a!ro9c-|- zL_hEu=BR{w3b+iQBA{^xgA#8%zW!L??lQ(Cg^?~ZKI4c127mtTf~2xu;L9qW0YYeaOw2baFs`amBl$wAs$Pktn|T(X#=p6~<8_Ks1%W$VaH$ zok(%+qbhr0l_Mn{uP8B~S6gB?`$OWf*iUC9)eJma^;8UK>iP@5F&wpIH?ZIyB3_WQc_ z++a8q`9*+`qXJ=|xisP_b2vCKDDkgA?ym>>sp240?;L67AS>;a0xiRd<&c~kn#+yn zdHu)pJMhhx@6eq3sONcv~QM6fgUV&jAp;gCZLg|J`t!K^}5dI#U$ z4Rn()^t@^TKwm6t54QsCPWU;?$uygfc%{Q;z=G<21gL{IHJ;w-_f*Eed z8$grl7l>BaYw-Lsp)?o#J$4TCOm#DWQgj>^aUMDI>#3|Sl8VE;eA>TR=|FWkTBaaZ z()MZmkX=Ibm{#unkdJP*72jlB`y%nc>B>O)r!=+ay-cm8O33QrkCPFf_MUqxw2|am z9L#+)_|215j#~aj={+1FzCTY4TJ9US6>?T5`F~u-tf{PMY!Kz2V>*2g0v6iwAjx8m z2KpiMC?%msFZ8jN?K`%IyQ?cizS)?pexkd39;x4*a-^din=dz%j45vxHfbNRp{2`p zG3GN4lAxe!UaxYLbQUkc>P(X&I3;@d_YnIP#=ShXYUd{BNLK6t-XNN<#qtOIVUl73 zTnHersvGRPBWgw7Sd{TIm<=mnoTTfRtd1U&Zp~js`2AFV?0}~mN<*@2C;G_u<_*GA ze;%KV9x-I=3PEW@mAJU^$_)QtxV0*7nk}3Z#vk`fHM6CA_L1-iYhmFe&`1@p%qV~O z1&Cg!wsxZUu*qpp>9$?n7D_g;aWA;tVy_j5(KA|9`LwOe6UrSfl{_=2a~GZ=zXr+X zQ$pSCO?K*htm&pv;Mn7s(kZ| z_EgL8^{I^{J(9+I5)lxijq?3VwsJ05^HWou1c(FBs?0xd^{=mppFo17JXxg7)qEpC~^DwJ@!~Nu2Pz zw%>n({~jW_p5ER=9gdR0ekl#XN{Gk;oCA${x8{>(;Y4gp6rMIe?A;bRznrVJk^??8 z{ks>sey6M#gmO5u*3xRWSua#-Yu^=@^f(v#0{yY_^7Cv{?rvtbf!J&?G@n4VTAb93WiJ0I4Yqv@o8t7W$N!vI2fzHxEC`GCU|d$JvZl8{?_dOM>#|8e-e7QCJ3CaQ{1SR694DI z2(d8+7@&Y=fZNrS^fT95RdmXcz4@i=&Oa4;K*;V-i518)Er{*qdUfKG=!Q-O{0M?d zfjK*Of$Y=FI1Li;1VPF1+{@G37lleC`UQ2Wb-ao!#{J!SNv@eriRxc`SqtbymW`=pqWrHXJQ_G?Tk6HSu? z-xoLgQSDwm&nihA`|jCT@OxDx>lbJ$)QplRa8&y#D?(n4bn};4SZrw35Ra?;(-wIp zcrWj##4QyieYltFP~Tv&rXJh)i+S4&3nkY~@Bw>|X9mo;Y6{hGxncOR0Op#V%*hU) zNIvjsJBK#maN#DdrJ+&z`9EU@xk-lmSQBdEZ4%h4)i@L(pz&`0Vl!|*q%Sl+UF(Y^MoY>`B$_;m$}IU8@8c zj$R8&v@HX#^H!c7a(f3DzG+vVpobVmap?i0DP?X34BOb*S?rcMXL56nr`E-2rGEO{ zt(j@cf(y|H6_RQr7u+~IK-I$v;s=BPBEqCAzN+2Lmi*&yXF2_cd2gCBW#*R3aS43Dt4aqt-A_r2Q(CoZAj z-36e6($+2~5t?z7K|=qLQsFybjPwmN2QzcMklmqo?23-m7R7$aU=|1uhijjp;=%%J zK%!x~sl|+!&C90c+F4MNKlF-Te@8S~!%A;?l?8cxPM0`UAD)2Du1k!H zxV}q)aW8=*PZ6Tf_*%5j)6{}p*9PK8W0CA11uskH3?kZrV_?71ZZ!1Z1bvD(q`i%Wvneq# zc#kfSc-Wh=hH<+|;S)Q0-B4rJd3%WAy-co+e&W_nIz$*i2OPu?taj71Y)qfT9;e-3 z>r_d4<@TzT-;i6m0c`VXnHnjY9Q+D}^|?R^VPZ(GPZXPK>aq<^ z85weAFHtak(z(U!EB3KCRSmaJ5K&SlAf^byhUY2M%unABx6F&YJ>o*=zZIWcK*e3X z&d#UxmIt@nN3=!vD2-2U*Hu)XDSiCump2IF^MQPWGgY4{3%V3F_Yg1vOes=^Pkdx0 zOjOpzs}vJ*Z2LGiOcYW{iW)3yd-HUQ5mfpm1-7$1Qav|plftMw!}#cU(4o&N+rYo= z7;HOSy^zAxcoKlHn_{BkVU}O7J79t?OOf3^cE=Kxy>at4FoyfTe!bwwD5vjxQxu5G zm3D!_kB@}%F6+jbKEVub@us&5#Nf;CaOIJh_hIS{3k&LiWS8XRBzg7I#H4AOd$aQH z9O<3N=u7Gs*IJZhQ?&C2ggXusk(N$3GY|GRcvF*uHRHO(E_AvXajnzQ~KHnf-O`J7nZ;eQV z&E0{N5Yl^^&lOO!psq>%m4Y|NRFhY!8?&ko_oAR>S+en?9|8c97F_eZN^+&~!H}u4mA?u}iifChTmMQMn zsU~rb8Z8jcem+zwmRIFTVnK;u*R&ebPjvOV(zx!%v%?ptTDpsF8$%i8LSG~2j67xP z5wqv~e>ZX07VmyVK&c$uq)0Z*65Ah&0{x&15~rqHn>?Ig&#kAqOJPR(G{Z3Z)rKe`*S z@ZEwvL72O3^jIm7Ri{k(fjQ0R`lA|L`R8#E-7lf(aHG* z^=rN81?|@RW_tCimJTgxK4M5;SIjLOW_j!5+g?d9Ff71_3}8`4z{uU5p4755nw03Pn6ArG>njxPdej{#~KV>QwGTK zzT&}@a3B_-o>h#l`hCjh9Q;>iMht&*U;nzd@5Mi3(BKTH3l5N>>^?x*_!V46IOVI3 z>VJd;TlYzmU!*-V^xoz0Gb+aMf+tDX+<^YgmPLCwzR(@F+*yi8L>&a1KeOV#17i{wW` zdC^GnnWAd%`p}PO4N^U&d4?2XdCC+!&EqTKr}+QnZ~yInzlY(}mZuv@gKDd!r=CN? za!o-$tj8mYr-Jk21nncVb4Y-rwLy+OA!uTB(*1Sk@aU>{GIuG(`Ei3GGJd>{wVEXL zxJT!gef~uU%_ao^1%*d1UE$gUn`v?A%NKWk{(K{1v2Dmi^VihNfyJ`~Whp zVUKOCA*-9}hE({{Ut*V(Fr2fI59RoP{KVSzQP@p9+oI;y?wvn<;mX_fDY#PfYWK$1 zh$Yg8*Kk4`sCaWm2=u>^DRlo>i_-o z@qF^~uGSnwcCKFBpRCZ}KH0R756}JNI%Oq}TD3AwxAW?sjw|0Nh1euj3zfQhm2_UP z1ebUBo(~qpYNM=AhLDxS%XvKz^aDAs5&(_P8RLdqhT1N1JXfVi-SA~vgHP&C)aV!b z_9uVjB9gISnz65>FiQaYjlcVx^cphulTnb56$q!}sw-56K~oQ?hV3frjRbjD@2jxK z5??Am{4MGOj!hf0L(!x-FMame<8WZK_&tlH;aIBpV?83qhP(-_91H6(%Mrp*@*JiH z5&eeEG*71e(+KX#$_5lBxW|QjRh1gV_1sv{kXqm9F%mV|d(V?o4$fNt>+FEs^Ee8d zEXT70v6fn6-16BFy};J9I54o|K%+%zoyc|R6#R~@KR(m{%%|AS+IeRnMF`aFGEzJ{ z?k$Y2xB0a*OJX-wcrw2jXZsU%cOYiEm)CwICbMsl^~~1GDv=)=w$9r>HC#0;TqiNi zpXe%P_ko{!0OYqqY) z+&Z1@H^&0lv_!qe3_Q>4@|}iq9QV1ze7dw^FE_kN71wD6UA9MT_)~4xH8Ov>9~5(2 z*haIDuN8`2p5K{iR%kxR=4sy%x+u;z`e;d|$^%EdU0J0Eru%jXE23u0OXoU(k67%l+Pbrsdcenx3J2|&SC{&DKwFl5cSZfSgaw@Yt#z2( zmy$m$uEQU9$-q+3bzGzuNBc)Fq0x9F+=%eEEk!FXBg8Q64=A^)xn9=&f7p@ul{Y#h zl8aQf-mSu$ROPY22%1995Sl@Jj-xQGmzOo#u9KsR6c+Wzy0TB`!S$!j%JBQA4LY8E znS>jNwJO~Ji*q!{3S7UO{Ct>~J+%~_0UBc$Lez6Si#a%^ScNNZAgUZyVexBTkXhXZ z__wC%oXnG&=9H84K;ed`%7wNCCC;Gk?DPF*im^GFT;8hflKX$M&hsP=ki2%4V!9|y zU63!SwcVB*2@4cd>{j{GXQ9Il?N@9623%FynR2@~J&nt%k0qAH%*l(A|Cw&-FiR4D zS)lljU8G*yn>O6x<1yZbHm;dL-N*nR9k@GLTR8lHvIhfq#3~|>!!`${amzCD&pYH<(8khXwAB~VPSv8 zY&b()B5QW*gyz`S0gw0pojm^Q+c>yj@=A`dGUJTcDirdQtUjFcCC^V6m1Y}NNA1KN zLziSvZ0>j*Rs|==<>F#$+Y3EG;eVz$K69N(Od!BVuQ+tv-@>}?BXpg1%7V03kosSm6)f}(EaTO~SIsUp=9ssf zmXe9^&MEiqV$~q!&+F3(eYahqXAYfn$Kf{LAF=k57=y!4T_OsVZ(95@F3lAGtg=K} zF>ZOQ_=}7DBOP|-53-8eiQ8JbEady06#995LK;3gnQVbM+v$9vJ8@=KULzOJhN)Ay z@R}PH{mu`yYaG`n41O*OE%*GyiS7m2Aw$qQS8KdKyCgT3@?8_k;YCY2h5b(w-y8Ai zv>`?r+|UGD-zJ+a8(c27qz&JfBRd?0KD`+hgru3TfMr#oa-aW}MgKe3GzyHro)ZJi zVlWce>8?>!R4g6coa2h>LCIg`vN6H*+Tf?AEOD6N!hzWD`U7!@F0&tEB9;}Y2<*NP zt#B%sX_OKyFp%$2<#v{=QxHXJzU$fx@>I0OXM)-v|`sAp0oRVMZ*D@O>G;I1w>8^zySAm zo3`L&QKNNI%J>`${jX#+>$)j<5D)3Z@`S_`nQ#NIlsv8!78deD${t7FTf95X?2_y( zox8Kt$t!Z266dA)N5xu%UKUZEf#m6JlT8#>nyS<4@eF-dy)g#q>ZyqtvtyJ$It^~i zzWhUC_b@DAV+t$;g5S(6|By(1Utzh3zSxjg+Gaf^&q1;-PwvD)7U9HEZ&!-Y8ke_L zOwP+}kyGz@Pu}@+qELf&c$5`dTBC3duaI}~PR4oiVIHw$sc;#$7AFdAG`F-+u(4s? z$Ypx_n&S6E;1G{FZ6LQXy*7@y*_M)&Uyv_fpq*c>O38q$IF%=p`heILJB11y-aTDPX`A;I_I#FJ7i6(7IE@>jPZGh%t1p4HYu)K_y{CUG#I@3_R*vIBH!kq zf{?stgo$Axw!!*r4XOtS2pk+1rA#>E_<)VSPQyYh|jSqLY z3j98Ze>1+bTp^r#bhoVhC1X5e(7T462i3*tf}j-AB!#oZsyKbkCjy>=`5J7uIi}4O ztSCvj2!30!KvPnzAl_+5L!pJUd%7kIt-M?K$*>t#Ur_P#cnKopoy(;0MBayCOb9Xi~f#^>CjXnyvhi?UG5nmo&5WsxxFsI=q3 zZz~8?A+TyAcw5RNmrK$c<9J2fbv_m(fLR2pX!~g762A3j@scemYy?SkjfrzqG#0E( zf%nGkv}{1xmRdGyF!e~=DVFRscJM$C%7jDJhe+{jMS@MBWBU)T-UMRYO_{^N>P7lK z7-5O<6T0-0N0YgDxI;PBQG`(I2-i7V2+bhJ1jrgX`LZ}e0PxV!38Pi*aT+l@1x`~b z)@$VEXT?$!F-$^qzx!-ieIUoL`tUU&&oyJLm6yx>VB=7wMJ!80bR>`EVSIrPEQ_@( zDdf}JIC#Fg)5%!4SYEcCahn96p)7~Z`v}&dpAm#cO&86&$CfNvkIy5SM=d=HcdUAjhcbqQD3=-Efl7$u z`?^_e$nE0dxNK7ptyUW^EM8&!DNKM(F@^awr7bOQmVtTDCLTJSV}7pMz&ACooxA5^9?$_x6q%ZXfwE z__?|_a-AEkD#=IovD&(@&krRS+0u79&L0dH5Yt#!l$GCDREy&Vjf=7MCMn?#lwoi% zu#Lc!tv0fg1#bNwc#E4XUT-`cs23{NwMDK%ux|yYK6nSl-ADQb&(cy((JmSh%^5!x z-mdPRW%OogEUuvt#0^O~%Q&9AB(C~gDm_j-66!9pJ=yvccUP*daRScSXs;p8=RjDh z>5PXvxH)<3iYHPlf_At&wX@)W2(vKvH-@EX7!lx`vI-uq#EoA`2mHC{3x$h%ld zqfZ<40h_5z%SRhb#y*Izy!DCqU2kXaC^&7s-91Qo!aC~+H{X4ms?a+&P%}`T>yIbkWIe9tx43cHBsP-?u5o(f?4mg3 zA>8aYAmi(N-YySjhRqn!D6>M-34EM^;XG59Xx{gINyl-*1nD{CgR%wM>3R;9?@gYz z!QL)oT5D$Gw!Lp-`MuB-NQZ4LpG5LTnvr>`1WtXI%thL_K3dmCv$Z+ECC^e0QqP%@ zIiHl+8-O6Woe#AwcTwt3N;$JEE``R9mfX2VZ5?kC)r?w;Pd|Ds*?1UxRPFGY8Cqxa zDxL+J>%^#8)<^90Cay$7oyV2jrd~y>uM%4{oe+Xox{7nVOvpw~J;zi&yy)BaQ(4j* zI&VK?krq*W8UL35wd04aoU(LW`fP(-zg=gX9(*^IQTZ0~@Jw4CVlx8HV#qHj+o2g`?QaG9f&0cP zpK?h~;qWl!;SJrvXfDe-g+se7Lm&@LvXfos4yln>$$Fj|FYZESPZIW6e)s$VpN9ys zJhG!z+p@F2>4pcUDE~y|qv{*_jDr*9UqB6c>kL_aH=?yZTT;(!wz!xV$jc^*e%i`DNg#`tL-I>?4nDp+ z_3%q(a`{GjJL(1eb8kuC51M7u{^O@?t?=BVm0rg`xRhMx@2tb@6OU}dNd3V^a^_>& zq~v->37CqeFgJK8S()d2$_eFgB?sm30L)2QvYH=vfE<=;*`VS*x)PuGV6Vm|t6(4y z71NB8mDTl`WgsTV6e%m|x09k_0a;k#QJg8{@V_4=^NwD3ma5>&q0^HpG!M}L;_(+8 zwDVb_-0Cw7Im#G0C|u2&j{3J#&MA3!M!r_`;n_lYaJtxeBmI%*D%zeh0n2_5uKp0x zMJwJU%bO$zb1H4z&xrn-KC#EH1PABN!kduJ?RO!h<*we=MV2E)ne zdMJ5$zO@`FPPYqof!&G3ODm@(s5e(%75&667q zE{fGm{x2i>$MPG-y$Rd;e zm|+v?WWfZB`> z-)Y(s{!@K8?G&RNQW&gCN{GuCME)S|>u1)zCd0PaW|aBL!)Wg%%HIQv8yFp-;yUm^ z1}Y`0Ej06~;8ZIm%^&;#t&zBu+Bej%h`CP4AC?Aq#VY4_?X$xAQX%Y}-_0HHe>vja z;5O45Xr9WJ@T)mas`+iI7ew}N{t!+BsW&AwFhD2Da-fZ%U`t?v9(}^sb@)t=yZLGE z08sdtd_r@!2Ki*-lreo2i0QkHo(qMzkN14}c5K(MWKq!FTQwP%yuE|;&9XO|G~k%K z+OVwKYrN;wCO~xOjPLl5?WqTt7)Y5Zhv=l|RPo+dRg$89?Ywo{z0GeCnpVavr zaHx-&W=~qF^-8|z{207=#raX6{nuHvUL5JeG->ZA^wr8#!^702>x#nsJHH!+o3~Xc zf%3%`<@Qr(mXF!~+x?`0bS6>UhUn4SODK=I!Atg!=*Akgt39s-;fC!1sT{SsgOEDe zbzaIp6vV{8-zIV(0h1Qsj*s@6`&I0eIj52(EZ2cEH+skydmmxOrN*+Bx{u2@t9(hz z)_B@PFw8fCw8laz7ltiE+Na7KC%IXl&z#o|tV}LAn}g&N(N=r$K6Nfv)}G@hiQ&p4 z2yFOi2?7-zRyL=>rF8`5_E)!S(NJ4ZHONh{sn{xV9g!hhA02&%$QY4LoofA5)%dcz zRNtWXxoic8V(IQ5>s}11v?d!lKQBcjGv*sdD=xrqg=QN1sky|wUl)2`!cted@_s1Q zU5-PRrYEKMqbSVAUTe8R_m{-%VsOcklL)0IpL?eoi>a`SPt?JD#Tz?w<9Lhz1l@PL58oRL#4EyCmB0=_TjV6qVtL+Q$#9#(pc5dPRZse++}n@73-X$}!RtY|MhYMxT+pg1#JOh3I6~ zdDY_#J|?r0i_{KciuTS#Z%H78%G2qRIGzmcmP&759LW3}GD9y4<4()0;FfXs3E5Ti z(^jtY>qkO|i*&<|r_XLVjkoDrd}{u+npzm9{LI&@&+lh-#(DHPs(Yq}eN($`nW`t_ z*LgVA{#OburzNo2*Ee4*VS)`Sds|L5Ax^;PmZda3X}gmh(b{%bhNhLaNW`85vTVp? zYtFC6=<2{;e{sAf-H+Wr;uJm4_Xl4pUU;3sL-O)h&0f^I9Y5Y-t+PvG9zmoQJ_p%= znvx6$NWJuC-H&_e<;N6waT?esY^-tR?PwBS&)(FAm{c2=PT5Y34l*O0l$@pN7UR_n zAys-=R^!eqxXuPeR_C)$d*;PO;2{P0C1${$hEPe$-BJq zvO#faUMm48t~m$1FU z*<*Rj%}-y}mUh9HOeyhA8L!Ta#hCF47cAybGgF^tt1U@o&0S$}r6QeW0sC$FkImA0 zetlnOx&ASS{?sv>1gJ7>zFYxu3SsS5akgiJn2|vDGAB2hszX^R0Jl)KjP8?wnV_La)*V1^GFSA~ZTWsq9l7SKl&Ld_u5pNhT%M zcZjCjoE=VGI_ee^J@bRteT9+s_5=(TDS}m&X$uRW=orxJ{h; zPeJA5(wET93RNQ(6x!F&bxQYKQ*zwP#*=nV<{}yWx)Q$cc@d;&8s2L*~tZX z1G1y#q47sLH|3rhEk#3T*>Txk2edm2GN!+@_ucZ8Pnin&`DqM+6w# z;BxhCleX{Mxf@cAzttB2&-cOxubZ8fY;PcC&vBI{CQYsNC!YhZR0MqaLEVaE|sT%XmXeIdWCzUR7L@In`9VpBGl45vNeyhlri;IW$*ZH9ec;p*~ zHi?y%6S{iu%W&Tqck`3gdXL9KqUhhZD9{E^4}kESXn{Mtq7Z?EK1*kl`)_5Db;U8r z2g$)YsLTDHnqJ2-i(4YmHFNo(q2u$U?*GTvcgM5&wf~ov4pq8nsnS7PvqdOsv}%;n zp;l~aCa4i=SNlP2s&-?K8nHs`(OR`fNW=(gi`Wvx_}xCw_xpXGPkDa-dAaZ0=Q`Ip z=Nj+xKG)H`(`k+&I@6!cZK*4@m6G$^n^Y}jil@;>Wzca+7$8H@Bc^byF++vAdogzWD6E4K9 zr$G;~2oBUcPkpaTdf&acs!BJnVtalfg}dWo5AMrrb?iI~gBH>|6(s)im9=8`4LPww z`Qx9&UZjff7raE|FFE~n(>^JA1je>udGZxtQ?9vqpV{>q2H;m;_c7D9=(elMQD?U( z?&pD4{$pUr8W;iJi?JAcpQX4qz~{gc!M={aUSb}r8^*1k@B;e*&6gD2KOQln9{?Bn48-Df>gX=;jm6P3rfmc_WFGl^ z^GR};nxWMlud_!!Ugsana2jlWlfDtsaOcQ39BC|H{+EfgbdVEdSXM9nIQpj?9CRvIv12%obqwLROo(0OwCFpp`c&`1tZfPwo_NnT{o_&cfWh8xfub7_iMmb4Bc}ls~ zV05YGIL@eJ(baskFKXDs#N`vE3YBHT;U}@+v9PyS;hw*!+O)_@foNnPbn?&?_l{8JFuVbX!c_7G64N|C0t4bq=sR=1cGZYB=IFC) zm#^*3p@@(#l<26oEVZ?%-mTP=9(CJM=d!F9q;rc#o1Y+jG2KaB;R|&8s%xwGh%0~scPYxn^pybM$>B`rg>xt}h^p`=1)lapyFuz?i&(z2{>B5JyHCqWT zf`xZehe&()+d{0GF*n3wsao2e6PE~oZuuNQT8@iSc~6I^}lEJp1b~3=umP{5FF!MOaE8+$LdAtw_STUjf#j)q99xN}~d^ zvNvs>pO6%3`rc}ITZTn+aJ|@>QMvzHH2*JxbmV2Oqczx1Eo){G~Mv|?Jh3b`|4 zO^}?o5azdh4sr^-`W=xC+m)}63a!dEjnvKhgNcF$`5l-(aG|6K{jwuMyzPv*U@0Xd zK;Bu(&4ggXhpo%JEuhDBxtY6)mt!~ui>%Im_Ig(;R*FlRD|B9acM481l&!2nAvKCN zt_3QuaZ2`gRBcEtZj{$j<~9OlH^*c=DxtX+VHMNooCN^quUI!K6q(&KUygO$PNOlQ zyEDVN%k~AnCk#T(OWEqW%wO5s@qkRJBNE(Tkl-p#rMh(1wz{O~l{M1TcYAyL z*)OMATRR!}jLI(@%_D9hgn^9 zr@0sl7(3$VwM_p`J~fq7X|k;hl3i)aS_F5@yddC7f=8QM$O@m2(q2}Xl~{Is(FYEMCQRzjbr1ONW$iF0-b|SO{ZcK}cnuDw zk6K|&(t{;d#I;&&!jochg7hm-*pTJw2J1|P()T5TD_BF_%p_DJ$77|GAUx*(iy1Ga zwzo8q*+#UDzd-~)dutSk{WUhFa&juque2fn#qRMgd8q3nA``O^z3*wPu#+`HSduE% z&J**<-C3^>9JX^_2*SVfVJD z*Zm~~c-b!$8es|E*vRZkxWVAHa&4=O_9JT>>lC(&xG4TrOe(qGN7I56SH0~vvX(5z zKTssoo9BGjBwhoKyW0||?jEUGTE-2^v2v^}-5zw%Ixa%?@If8;eE=hhqm5a4BAw0Iu!Pw&2+DGMkbKZl-&fyZ)KWZJQYSr6}sAXck@ zB!2)xsmPd%o$)QA#ic#&XaqlSB5DlL??-ZKElJKT2o;fqzIabSnncI%vLg6kHgwR-iHoV)7Er>odj&3&xQ8JUtb=_<6|8ewQEWSHw1Xd5_MdHlFf? zegEvb?Mw%qLLlhzd!u@)!ewj|L7X0&%{`i71lbu?Vb7enbtoKq7y^Wy-s1_Ftzv!- z+hji#!g`b3T7X`d86+L&swo2H<7YL0F7U1kFHI_k*{%Dk%i$DJPkoX;W<;IfPKk8LP^QFVz<;hj|O8$Qf*2K%Y9 zYS@{oONod(f-{0^^K%^|jYQyiO3cfp;R9#utb-`c8Ek{IkXm1iEW}bWJJQy_o_iqo zhm3oeMxQp{3^6`?{8@sC#B%41^$QX-8He4g>Y=fSeO!CV;*`2)0SK3u7k6((x3RoX zsT$u2`bzPaF4vSZQ#WYw^^p^)+FhtwsxeL8AvH>_e$1OeRXGGqeEsr#%c~-4_-l{0 zv7ShEc>6a)Ew)!WyM9D%l=&y$!R^C-fFB*%;|u#x%*dCUL7v^V!r*!{_Noh!s#05R{gO3S>rQ!_ zGEIw(O_Q(uxNCmbJ_K~LUo(PzjmR5J>4&YTo8qNJ*f4o6N;9^GLwwL;*tiXrFjABM z>N@z<9LAk8pD0qXG0wYLWJE|ENrv>%n`$}G#Ge_&77#}Au;!QlLjTU|!Vi5O#bE6g zkboI`-(2<_`-8F-A)&_^2h%7ym$5VteKbz27djN>ek-G{`JVAmq{QL9v2b{^W^?ur zXt*aK;OhS7qT36*UX)ffy-sv{lS%&4MI9|XYu9{VcTVkYz%bbzx}8)y`iY0+T3KdJ z=(MD0e=s$Pq6Gs$Sd6g+72dX3Oh(y_d@tIopFff*z-8jc)aAu=(TzTQT6$A<&N<@?_*C%qPwJCjD-Vf}j51`-N3w5`W51+rsqpncUJZHUmiNv|}oU|5lBB8}<hN@>08V0{-y{ z!$V>NhYwuc@%^Cu%suMVS#&|?Hzr=aq(}w-`X>S3+CMzXr>pSF@Tj0m!6(xV>I|%9 z86cwQW`e(Xh=XdpEjm<`>H5oG@Oz9NU}wH8)WDxO4Pjxpv-qqaAjX z4lEZf#?EnWnx#-RWk!d&H)|^|Vgdk4?4zA3is$?>M#cFYL=fU9vNILMgkzycL|JFq zLG1OC=^0#EKUF7gPVjPfM6a!jzJGbVm4lajT?6jVP~bmCjP5 z_!}_#ufjo#aZ`F)NW8c%O&rm@mXHb3MOr<|sd8&exzM6SuC&QDTj>#fv2xB?t4oug z->w$1tEXjX9()@^%yL#aWdXE~9atv{l;Sm>l9E@Lk<#w8Aohu1Ld5mWtpQn>Umzo2DYS&i#{%0M;0-0pgEF zHHncyC`w<`ibD>cF(Me@^T;V=bS`CUpVH3n98`YMe(F>7S0~u-VuwttF~>H!h$+pk z#V;p8zmR06G~C9(*XEPL3x$PGS-FOjn<*(yyU#_S`}w1_&-P}r^cLfw`!Dn#s0e~S z&s(1*dHxPHAykiS<%zA{0(rO+J+~&>3W4KzY9-4(okW^2k|d$Pb*6IW*NeSEy+zme z@e*HYK4TLl^`&x0RIBX@H7rVCPmAw8_Q|`0st36gWS0OzE%(wRNLA^Ok>pAA&bCBn z#FFK^@+V%lId|LpKdez(YZI)X z2R5th{vv)z6%A??9I3EAFIfPu%pdDBTt^`k5Yw>q^gGm_ssbpzv`jx)959wE!wI}bP|hsFtA%~ z!Z7VwwbURT%5y3}l2))t_+_zz7mx!ic(WDN%5H3z@8VNWF1wLkKa??SZx0o6UC|+l zv?fMHRhFzvC8>Yjr3ta~o8~-hjgL`N)sE(0R+G|`kun`%Irr_D&G64>Tm{;)#V=q^ zy4COZ?v|Rs^bI?r9>^Mj&yh@tE`LMk_#c}`j2}es|IOZAWpyHNf*|~p?!rybw`#kp;9s9`< zafR<}(LP7OQ133B-lX;hPJo%X&OBSsamYF3;sF@p6VpM&1N|@u|Ad;LJL-));m0~? zRa4mGbq*l{fE-sU0BpFfxHI|qs>)G9MgV}@8J}UeV9DC#=8tmaquxnA!0brs*}cBB zze*1UWdIDJdJaG^iIZh|ix>V_ZvWB*@PRydTn0UH$er%U7b2e4fml!Z)juabdgl4g z=Hd}A8Cp>1UFujOGyMNPLSq@O`1B??vgyxM@?Thq_ars2Wd-Vet#ya5oKfdUto9`p zwgin)43tm(@LxbW0RW^kS5i9DLF-Cod^r9%_!Sp`0fNKi>#6oQj?x$7tX`ctfYY!R z%)yT$8UWo_B4s1alg!qB?s7k3wL75No}4-~*@GpuBpjE_ zpNj10w1B1%`%U}v^_ zzIbDb{&fcWwRvb)cvdR%w%s$a9Q`4!9u8nf9Y7pq%|hD3a~-{~Mo>!L@eaZAhiEph zkpx$q4w@U?^tR=iq}lJ!Pg9B>ca`g8!aHNbZl$LRntrMfM)^z@XVHxOlpAtZlS^F~ z9_3x*n*McLvRTGYkIY(WyaxbjVqI@X5th(s^?By!Yb{?aY4Tp%BKK&SoCbG~*st)u zs8^?+w&h-9;+kl%g{F1hbB0Umdy*Pd$=f=jT@K#e#9DoxnNI#MKz@(4E|r;Y^BNv| z!bykB4dWks5$@&L_Qs_-c4AnnWzVHrhs*nQ8)5-rO<+nZ zaeg`(A$(m+|#&D+9h(X7vK~k36<@A2Zuk_5&iq2Kaa2d4X7Ir9EN=ZscRd z(yn#jjx+%NHa(Hq*yg^feQAD;Dn~3oWyS@X6 z5AFSEy$>Sg4`so^4NBL@Ql&#a&zyB7WchVo-n=|mJk}{~`;}c1rP7J6vv$<)Y89>o zM(*HGA~|0x$>n;hVfvXgsv18`Bf(5}0dk{}fxm`@ zwuKY<*3hIh=)*Qnfyr57GCyI2bKi0AwwRkna!weMvelMz*(N(neFP_NW3WRMXHFS$ z-+L>MlWp@x=OCdM-B*MS-awArFM6f-6pd@6uq#`mQKT^F6Fk8?J5rKW5*=-pFMs2 z=7DTF-Hc87_cuKIy_0h#l&L$RM8$_UZ4Vn~?~6JdjDWA|Q?1oV_qiqcDQCRGgRZj1 zW#W-&e~*NDpsL67YfPWDeK0toz}xCPJzRvZ&6S}0MH?L#GT*w6j^uc&sX>5a>t^Jb zFLOln&>A&;o7DESR%0ZM9+Up=Vxu7Ld!!*#nrYg}NzGDt2Kwc3qBuWIPk6F&{^)HOXHkwV;7 z-kVe48Fmv`E9S^W70TA+QQ^yx#y7@FfZA9YYKr?+#Xs33Je6`OMSCV{CJ4yfz{jMW zN{z(y%%<$2g$)XiX6xv>P|mWDL8ZtVYP5+|YJ}Fw0m-=AZ|s?fa}Ma8-9N%h)!!b2 z*xF+6xxJ#-Jg1_3EyKZCLIp+p3Z<{*HE|6wS;R4ianooW3AKrs!LYwGxYwgpY-rB$ z^?lGnXGv?EU$-~Zz+XU*)0otg|h5;s6SVjygOzHUr!j`lh~&t2HW_0)wfKbIPLT;|y!v&7$LNEc;EE zXXA`pd{sh+GlNfP06bZwIVm@z$j}CdVJY-IxyIF$-SCC&j$J(8Pg@|6;P||>xpmL_ z<#L5n`L%_Zer>8!`S@E6`8cx?kh7&`=jH?T_%m8l17y5t?Py;5$|$m=530r7+c0j8 z%u0&|x0pj(aV`ks&WwE5iQrDv9K2h?Q43J7j+w6aCs?Jwde!PSz6?P$rCXCGZA&@3 zh_Ni_q6~|5^T(ru8yYu9nmU0Ry9EvEK^~MJ9rj5M;0*GXjh@;<^Zo!o59lrOnC1QY zkG!nRRv%^%J!Z*RTI=WczZwzpa2_}obEia=!PAa9rV%@oB`jnNxBD4itVW<_k;W|M zMUPe$C^&JQumvC{IpI4(963%F(YNC7%Q+83;2bu*EF^C5;MH9DekYj`gdSUy1 z75ee&EjYWqJasFPkz&4Q*q(-^J2jpE^z({nr#2LEdA}cl2vJyRCQYF91zcllR;ij6XJX#F+j#T?XpS#nr6{ z*4Xbtw|)SQ0G z!oE4sW#DYE@so!@yN!rf4PTP3>Idm<4h1MNf56owcE2phFB#;I6K$pO&=_|HxJ4s& zi250Td~}aFQ81fu!K3EV7O5k%?7sCBiKV66LI3+`$vZx$2l#$R4!v4Dqg0Wcq$rAX*z~M!$98|DVr~ktuo~ z%phrOFCs0Y)Pq|X&TVP_LaQdwIfAFdS(p-nJC&su>T%Tj(yoq`I!i==pk{9b)`sOg zT4|Ch)zEe?(b{Q&`d22q-lprWGO0A9^%~+9vF;O31=o8|4O!`dk`PFz?LKN-%_A2* zI0*?cpDa7mqdIBOkncV0?8u!ClG{|D`Knww%BvT>aLYE~rf7nMk(TW!#t6w>ZXjK% zQ)9Q!H9cs7F#9(5$?&6ujB9XKP??WcWDOK@de0*Ipuo! zPR^8Cek})YfiC!}x2o{AUO6586Lf^nY<^nDe9mvgKQwcrC&d5Kog=&sy(r5UyxW?vx@;9o|+ta7n7*f91H!$#ZqjC<=AKM`-ZW zx&V-X>fGwccz;A1aL^VYEx<%&?}2js*9;Z#LUAg@wDDtJj7Rq7Z>|h*7&RV$c>Im6Kz``_%UkHQ$m2?6TEVK?v*JdJzu%g>hZYRdBILjcvmR8R$S z8tqgn?%vJ7BV0D1R%hUlMR{my51#z{;t>c5pi=vPQNW0|_m#&;hs`Y2iS0`VKu&qY zQ4}*kglgc7u(dEccntu4^z`(`m+b-ijHkW5{oCQq6+y`K^H>p& z?eXhhj+*(WzD-h7ReI6o@32Vugvxvdz3lR!kXnG7*?6d|9bbaM+~x^(zHZlN$&?}6 znNGNIXzkuczM#?Xr{&8(>C)5E(rWYBq7&V&em6@w3Yzdvy?wxe0Pw}}?`L6RiQ?k7dp2opO-9zMd6>_*nP|c*e^s@=F%Mq|AY+PKr>lb(Q-Q7D zSP;Ji&gQ6+Rq|CInEu)Viht?xs`#dke^5{1^tFhLjC`%3!4(`_SjQqG4YRAeZlN;kN`!Zu5Aqo!)m

*om zVqhKi$jIED^d9Q^((dRLO4&=ybZ(8vAy*Qk1^C0e3?z`vQb&b*At(P{YR_0h7&~Ng z#oKe9LM$!Af$PKCnu1v?p9l{%Q-g-g5^+)r##%9EGbL*bW{!W|T(zy$8NHkmIhom} z17eKBX=|&Gbl)i59|ae-=0w1XVNWMxVTA>}_@pe*Z2{=RE4BXE!wam=j^}`iwyw&Y z;Sy-gmOfN?4{qv4`gPK_x*u)?-1GHXPj7^PTM!yD|M8KgdQm8b0XRDZLobNa4t^M8 zenI@IJ_H%D#JiB*mlF|cm8x5<(^gy9(Y%OH^u*PrS_6UowL-NKLbOQZ&s3U-RiM;| z8Fr5c)vxG7YeVG~pj){4vSeiOc*p8sh&WDjh1vAKnjUbM9<^5-W5gld$P8Fdru|+0O?mQp;)H@l4d{Fl~)R(^YaeU)NX9BK($1 zzb~QE8L!v3vB3gIZT<8;zPS6b#g!rT+=whz^|S!Ms7?e}LCJ1Kp^2Dx9ZyhrU--}> zv4pK;QJiW14X%8`;WlWDu&uOHZQ!u2B?@#v} znJOJi;!T093`#JQ%Ai)2Xk>rT5o`FY9ETwrKi^Z<5Tg+>zV;#`v>4?~&#>xtDA9y+ zJe34Dq=dC4Q9i=&em)j;XZxAcuQF!UpUTz!cyMj!Wv__4%K z$_que2_2omvWG40d*o0Ct4rZb`y2WO`6GHui`GwG$z!FM@-{@kAVbPYt>DgWjam3nP%@h~SH-On=dW$On_0WpbB5l2zi`Rx=XWwdp`a?WA zY_LEyS&Yo$5<6Z0^QtZMu5FlMoIxl2GM!T{bq8xC*xKwyTyWq_sC}20d-|}C#rDNt zK(2nxO6@aDz3K<5X^?UD9xT3iIU~Asy*2tAh?dyZ1H7sE$3)wp&qC) zcu#p=KJ~i=JQw-20Fj;9YV@5aYRWuG_dS|=6WqzMRa=7d4SsCIT#1|QC)0#{i(h;H} z49?NV`@WwBm>Vx8OU!DuqF>~M72R*;Zh#K_+|lnQeO5&j+l|CdOlr_qK($H+gQc|= zGd>SGL%sJlR|-X6z>Hq+pc=5_@1EAIcj9c2DsAoE@49?Jr{0?pskEylFxG32=yyI^ zN@+@(kH@MY&ibU2obJ+zYftZ;{pS5Yes6gB-#y`wSItI{TlH&$Z|xC>sC@HR^oQBv2L zhs$U?BJl7|y$77f0Lu-&)5u`!U={z;+%hf7x|0EbX`$W?axz4k6U!P%XqU?+?iCK|MUs+Vcb)f!*W-Sts8M+rha zrHtC|ww*eP!oczFHVshw>umFS=eKXyN(XAy(@o9I8OpdjNRqGXTj%;oWiO=qTHPgix<)^99YL8fV^~*~&qnjTp+x*(IxMe6CubKrD z7+LdOl~v=hCVa0FUZF+|=>N_TGHdj4!;>1!bsF~b6i_X)ZRCp3*Tq^qYFHa(5q`b< zU|z+cqS78jrWqDeU-5+`0u#pBL9?@HEJH0g!B1`5Gtv_>A<RY<^7g(RW>Dpsabq7a%SF9#q!S+zAmX@g+T_(ZR5MQv-%&X|8mkG-2 zq1ix)^C`z4|CS!9Bf>SiZKe@x)^Ah_k^R^Z5{U4tViuu3! z`1p{Dx{N5-vO?faDdveD8h`^9&t1z1cBPN2*=eRXrl-mMlKTX^yq#Zb9S*8Aj(ls2 zf&Ki=lfe`@tqh#1)2gS2Fd9x&;TrWQo5JbwAStH1yL!hUuUAWCDp~|;vE~z>0l@|EA447+k z5qFg`=+VL{In#a;xTQ)!B6*Bu3 z#ix^2A&y#h|1GEU*$hyQ(T%~s{4X}hQ%{vKHqrs^lwWN0SZ!x>a|%5fQL9S~ajPds`}3h}slEWTb_68Xh5pp< zTA?s39&C*A96*qBdzH$exLh?+H%hQI8?qMh{p;!{#Nx8Uok_)Kd&#Yp-rvxq!ZXE` zXpOQF0kjDTbF7Y9VQrgaAS%J0^N|%E&kreaumG@9Hd+jy@Ja=Aq?lRcnj=t>EW0)W zp=M5%P7{Z(aPS`;mpq(_++MJJLm6Jd@~!_p!Ws^=gsf59(M+HZtwZfl7Bk3)Ct${`>YG)s>u}3QrraU z&UNwy`iLqchz~qFaJf0ul2dyX9{(a-PHlDgL9}&NxE10WQf8L26fgZ9)LOx&$x~Ca zOp3Mf5X?!_N!1h2*AvvJ^uJz_u(``Tdvg!PA1lhwVi-YX>$QaYSZ4`G>sWUb*ZBI_ zh@|&FK}v!jmm2FJ=<4t(L$wtAR<-Xt54N=mq2@q!M%R8DBR;krfJN&j%r{H8;(o-m zSmz2`?Nt_#1M2Qi2JqxBlo)4~3NXMZKu$E3?05akT9>q!;tz~r7s1B@tLxB_l-=_M z-i#3*((So>(UB!tU7Rhe=X{7#x~lcHbVvaWj2i$+H2fpuW4T2(iU9ch*{B=&Oew8y zM&1O;?*)83yWDkXU(wdK7u0T5QeGFcW7sQWUob=H<}Pln5OJ=mS#&FCHr=1lOiHQJ zHS352O|ojC+{?cezHJGs?PC5T*Hb-vfK1Ow*#GE|(Slk{x!F{-sHbcHGF3NDPfuTb z5VpkdmL0-9Ciz6%qrX%0b+2<|pG*jZo;uffL6L#ZJ@+S3F|_7qO_;B*;~ zSWDA}c`!SMu>;t{RXo`n+(%;jRgjg6ceOl@OY}I51bBO&0`}WcszC0&U`>zkzGhKC z(fWRHWB6zFOzXiaZGAT)$kM_h>@(j2ET;bB55UP_;SN11a(;R~PBGC!#9HD2 zZjbz#D;%5F|4yNZd*Q_7@2ErKj@@QHHL$C5S*xMK8LWL_BJF%ZXUa?`x47kEH0u#4 z%sGWR7_m4j?B2HF5ikyr2Fcj}t0Y2wxfFO+*@)+`$Uwe*OY>j{o*v*v9GUSSAP+WD znJUCbQS>Lj7_c0TJX{f@wz0|tu`Y9?x<9^VeT8r%p5aGBz=uyq|Km_xE2nLyA*IZf-lm+K{lUhp8wc z0psL8G8-X3bpCP)l^o0E6E}Z#x63~H$;ff~*)QE2&p1I<$QNaiO8tnN>Tg~@o2Ota z%E<|9qeAY$ZQML=#9doz2_KoOiahygC9upg4(7)nj!{vcIK{{z^M8LBnqT_cFB#1Xk2AtE|nHRZmpXXPWsZF`T zQ2_d&#JX%id+eVLAH1YT_Z5<6Dgst>aKIvu_yWiMIvQ>m3>FSay9`P={_Q96;$bmn z!Ab3ktZz>AI1(3FTCV@mzk_B>Iry-OZH942bpL~lwRcK3^5$oap(c$ZeriqK*lQuD zZQcW|JKQqI-wFWrRwNiR{HfQ2b~kV(LPEXUJVL~Azv}@fXF7xB=$PeYWo0K_t-D}v z%jai%QcY@~MHmkn`8D3~yS}ZJCOUHGSq>gKD4fIUGm7{q533-+L0gC$1V+A?& z9d}ly91M0Pr@Io_x5^fZ+KgBCD9!gWzg@V<^?Wv`-r4P&?}o|>L5Hcacvrr$y}k)1 zA?mEb0}15T`E+MJ-b`c)Z@vwxm|n zHv32MqkV02cz?Vsbj`fz+uG`ydDHR@*wD_^i>Lo{<^A2Bla2SbmB?U-g-94DoG#)0 zP2lE;@^mEx2iPs|Y{AsGIW!R24im6fpSQu#tKum0;~&dkFnQPc&`-f?7HKX9%AboD zC@)V3ExH-23~$WN4?xz#Hgil!izI!G+>tBJmvQg3rCi^IpBFW0O?Y6`-fbi8F!8C) z#HGIJiorE$vLs$d!{|ZEGs`hk%LS!gXSFDo1+nd z5`hHW*IyCWVRhFZ?0$Igz;!cfp~AFL%!WGzKEa751;Bzk?9jtgh4#dZhtqYEt=9dY z@M{C}G5dWLf{>-iW*hBH8z&>-`~sy!N5Owsp#xn=AZ5zs2SY0A7CN;%&bHZihQwNw zauT^V?6XnM7Ocq^6l zdML>|ef4GQpw%-o5F(a7INfk?}`H3u<(zS#yScBo>0$w=}{9}{fac+s+?EPJx zan+CTn<=}$(b10>ulBXAq!QRe84^4dtHw8Xv(djt^IRW#x~!$3$Xmt@1r?i3l^M-O z7ZoChn0XD@4JaM5Q-t8gYQ6R5t)4PVhvxpokt-{hi(HMP)vIInUi!^brIJ@o6zK8u zp0;tws{8L#&Mux#i+r>9r_FRS_Ew5=GOPDyd%(n}d*$k{)^MQQJ7KnE*j&OaK{`J* z^OjJaO-ywz`W7oaw5@D;USaT~MY+>pZ1W3V>?=x3?j3QDM@US~W{#LCBhgeLG_)%b z_O!}!dRhistN%d2#>2uiv&(V#opU%1>RsdZ`{rkqXq-j)KI*pv^Nzby>}h9>^Rp}> zZ`Vd8S#P~*7T7B)IGaMxo#M6hXzRn~uukabiOi5-7;D^wiFAcP6auFOA0+J%bJ&BU z*{YqZwp}i^<|2o~>6rynJO90!|Hm~f@Kc-0gK5!DPc6*>E2zVP7}PS<;cycq<3ggn zi>pEsLO63+CXmwcBk0hQj@sm3v~6|+o~ge_M}=Yhd>mL&>~+ zf&*Za`G)Y7SW)&@jwga2Zv9#*bcC6_)xd$s{J|3p#No%q#J7!s3zxQo9YTVQGT`I3f?8&(M$Vp8Tk}oURxP~UgrkQ-(G#%Hgd7~93&pJ z5!>57eBW|7itC(q!j|LtfwnQt5QCATyHTh%sWuqa{!xLUN~X+UP)bm-cm{jf6zqD} zQNuJfzrrWC{ZiR!sok}P_)tFdx*k6%QQIh0J~+U-L%6Nn15|{Q(N}oUoL24I=sZfU zQe~$^w@o*xJM+&KSMT~p$~9@i7R2n|ZJ2I7%F`0fop^7hVltG0qr(sTI`m(!P{ieWotop zy&|!R9M?Ss@v1IjEXava6=^qENNLDSxL2RsIhxQF5lFZumBx>;g@%|zthyrz-)T2k zFrtz4r+t~GGQ?O!91~lx5e_lRfBK67JawclD-dZ6;U5=QgLA*sb3RZ5sx3{N5#{?Vuw$>PqtG3a=$5y{Bi8^KOzXgT)mD4hy)8EnS!mko;9=+P zg0nqz6XyyYbzi!wxHAF|4BSA2+b9)_d!~|-U$_1YF1$T2e(&M{hp_cP$HHV8g7?dJ zNrEx3)_a)^eOYQI_$rq|VQ&M(uBouQU{G=;(Fon5hP^&hQFg72U|CwxPG~EWOGrj3PvvZdg0@4(m@;X|li8 zA6+npqmVV8kXSbI^6$*HHFJlLR>wakeA|yXPP&^faq)Vs%T)aeJGNaw`KbdD1$8t$ z>^^C{i<Xd!J1jc~?(wdf$3;dHjLL_WHw}<@z=dI@_?E3vAQoA(km1PusIv z<+8VTrTMa+h_roEAA*JHZE2fO=x9EADu`Fy#&ZhZ-0(~;vs2e$Aw)qVQso$lRW$c( zis{KNBIsP~eZiWYkknhOr`~Gz@IKHhbTYXnVfx&C*ESTF{rCq;zurhLYu-=a&EhQw zzruAgLLlLi7i=9rp_7V9T2Vuox2CqgCi zu!-cy{|J2tBu~t@$sd=i9t+XBg;0u0kF=GhEC}-OebSBP6iJ_*j18?}!M#b^w%~5n zTyy_ds41N}5k%}RM^=8K=T;I!YCrZ0I=gNa}ZLlXa@MPZbWXmR*=CJL`G=vpKA*+|4QD1xA>T6L=TNV@Gg+H z@P%Qri;)mcx5LkAt7KvWqNd=mqF&maXsSb@`Ue>-^%RR`d1aQ?osL8O0y>j~l1b0Y zd!TazyWE!DF$#osS+n~TyvtTITmXB(d<57q8m~bOxuHZ`jJ1ed#UUP8R=7RIrbt2#M9h$SN$lhqcO@-`|>}EJvVh8a{fa===r+@al-sO3OYBQ z)Bg10Q^Jk7xf7?LNEx&8Y1`*}m@EHOT#__Rs|*+@T2p*U+ftHE+ew{oW?{~}0~pgf z)S0*U`Me;jqRoFy?_JK9w7tN*eq3!K9u`c)djfCZB|gJ>zu?4bDVLW6(EI;fPTdC7 za%T~*^TILl8$Cr5GwrEdWNe9q02TFdMUMaS`79BZUuMT8+2{!@u$hH>ae&)gSK?r0 zoU{%3147tVPU0oDa|}9IKI}GNUaI;>)Bl>7PG4m>KUv_Q975WzN+pI~+6+z-nH6iD<$H#aT1!>x;^b;EM8^%Vo0x z5U{JLF_yP{wqWDrL>#1!i;jc&OB6 zxy8NSfVc3#! zR@uAF=6wM#RmVnO_K!jIix`%DSG;8-g>SS}+gbYh{&H}RN&g`IRMq)<+GbIy&4A51 z?4HCP351?VwUtgL7S$|Qv;NMMW72jwFVi+Q^f~#{M7?eN)tPTnsn(rC0-LaJyRCC} zdlD#je&>X>be`WJA%>j(eeE^a@2(J!gr$;iVnMfs23&filnik`*?Nc z*O_1K1q|-Q?1wuaR#vvi->#ip5*4wDuf8j#cA`|o#tJ_#UIUv$tbEE@457n&*xLF&f!OeA_P&kUF^{QAW)f(qH+Stm?vORxu$fntuo{p5~WPrr)DWiCU!pFaN& zBjQbU_hm~-kKo0QGnU7f^9_bqcS9dtX5YgsP57rnwy%x(X^vb}<66ABzg`pFI9W~p z9A59&z^W?s2=6sb*9vRFfNZ$Cw&JC`f|~%$V_ZT1q96N1Dc`esfzg+=jS?ki8Z7t) zxpHnruTb!|O98c>Td!6`^AoO{$e#;7Ew{XIamn4XYI)gzs%C!J+NCq7Y`daE+in9E z`JLzb7r|1w&TIdi%lGfoaBC07GMK`TuhG4mGMH|hLS7Hjc9z7?93x8=-PV8K;3-#N zMaU}KpyuZ019xjdAU&vAgR@;x0*)E1kp)VtM3#X;a%hjfhC`;-# zQqy4`#+@c@ts9+#U6uL2-au;Q2aQfb6AUOZIe{>&Z&S^L$jQZ3w`v>NnRCemQ7Hc% zYp?71heDyDn=dUf2nU^S?Q=ddw7+b4D~$831`7H!^Q-LiW4$%#gWyiXKMD)8#5(uM z(oKLRI9bcY2l>1fm~n!r09*n_Pvc3B_6rzWijpkH43?Bivna4vIS;XzUu~!?vC!)F zeDw0Gr_{)eFp;vFyl+J*Mjn)`!x|Sq-N&5Tt{4%EX=(B`Kd)8ksix~nJPtPtW|QRA z*FVuL<%ns2zm~}#OunS|VANwf$e6zPyc{D`e+NtN&`NuvzXY8iZEcM6SrcO3y_Z)r zHU6u&!{-I4;r|tL_3=#SaXcfhB+|Vi58aD$?h;Fp2{pT3(PcQ5yFA1c&BM%I9X4z| ztgiA{N1ANJ%UJHTDPv9^R>zRBRi2uNGY=*5G=|vjH+TQoe*JyFfA{%)U;BQZzVG+@ z^L_mu4#b@&t&UYi#jObQh`*2qOV99*%O<6+j^o=hjLcH+)68+N zlqpog8Io-1qgaydk^V3xQ<%DsOFyj@@qjq|`1u9;wdXwl=xRm)`h$^5G;c%Zb22FH znpp&V(Tq%_K`9H`VLJ7jJtg09yr7Pc@JHVWTuJWXow>Z6y%?T>?e+zvJB%ozH`xv0;g_JDTcNUCnKjeM0acDX_zv5~wdD~fKQQLX$~1Q>S&L^JiCX&HG1MTJO0hAsyqAR< z^K@Qc{*JkK(h)21t$auJgyGY!Z*WS@SPgOmQHNHqkxGMKv_1@3qQ+ft;9FRA!(? z)wshWNu2(wKJNZ*gW?Arx`qQBD601F;Rb#)^`VJ5(hkF%qF^dU>_?3;R^5+mAQAM| za$&$eCqik+2#ru5nYi@E``{5nSP`x#vKKol4vqUz|U+(au-LV$L2%BG&!>;F%_JOCZb%38%p3gx=b^~LszwkjnJ*ud&>o% z_L^+Ujh~QJ!VQRNJN;vT!F+@z|uqA%o0Hh0C5Ub`r|cxl7Ib){pIj(Uc_FI1-oqgnnyEA{~S%)g`$fr2toN zJEMZ7@{p-ELi|vg`p`}70exhut&p^{cHoWS$%AV9RYK;RiU=^nhE<#O9C*BO!N}Qm z_M8e8z{AMy_{p}L>YD2S?0*K}I`dTYV;rg?Dg0tepLd)uIgc zs7FAd&{wBIi&e@)VbG~C0JQFcphtIMiH$FP>o`vqGO!i^Wis77jPsO@D&u6X#@?KC z3Z|%%!&z7~MtkJxISgaz`cEd%#gm@fu)DzRZn zHp0U06%59x^rC(vTEs_Zxms4tQ0OCJwNO6r+i+BgUUN-BJ(j+yywA39qnuf3LM}xS^bXe zCkR$DBe%oh+BztMHy|%F7>uCoIDBpL%GNTu1n)ERV4aB~<$?)%dw{9ptl(YjWZ%~t z`qWhuiHzL_1bU>guoirJ3Q%<5v7X%SR;m+f1@iv(aI48iC8w>WJHY<2Bo#;#?v!Vjq`X8-f=E&?~4k8o$onxyG$r1r!4%Z9&@tcryYvSffha5CtY`n4-ArAv-{#{|Caa z*D-#2aGa>Zj1<cDLYYwgt0i*l03QE{h0{%!B z*w9@4OhINtQ%RJHmHayoToal87e`(gPNiqVGMtS?O z=HA^gXfvSOL}yK?@><~fz`ZSnl@%4oCNYDN)1eFf%vC+#fYTY2qFW}dY7sOsxi5tiOBzE$_@ zwM-i|0jXZ#)`YD&^OPv_e)qf&i1xQ>%ULqV{AL}VPPlpk+0?H?B9_a`Me9AXA7%@*Q^%NOpNi2uMa)hO@m=6W`DRm-`x(ZoG-jj z`<(g}D*f?MGP)d{CbnRe@Fw)mt0de&jK2yy} zl){%t4MieTv#FS*Z&9UH%6%n27HJmE#&bhOF3j|0yo3;5@j+(W$;3E zHitfc;Phu|)g#QGZhb}ZS6GMt)tJxQagXbl^+h#P%c({h`%QnYvGX(kWa?lRHDiX} zAb#MiDDu>)Eh|c9VR%l$-iz`2qRKf}!=@?`Td;xexm&nZQ$i#rnlS`}LxSS~T{Y4T zO&&J&Q)VhJ_MA0=HCaJ=*%l4m%lJ8(R(D-Shhw#eE27a!bDUuW|4D@h#t^&mW%2?- z+qDPE^b@~CQ3NV>gg|Gb;Z9eH(qVUm>6r^Gul{<69adumLQ4dSBNZbI#h}OKsBvv1 zo=AKc2y_stUxCsb_y<7`&5#7UP!*s|uP~yavgN4ifmufQUlGxv#)CM@f$olYdPrJO zB}dw)lH#6-;cdbkNRkA2^it~bWL&|$bd+6$CX(~75Qx!aB%@w&JqrGkfb<5dBs}`_ zgRe-_xMR;-B|1O5Zy=2#&xLt^M%>^%!IXtQN=2h=Q}h(-4Hk5KN`*WSCUfvy0G};z zqJ7no*B8M)cxFlAKuH_3H84UIAwN7fFGjWYDQP!8N!-il`MH0;u70IgCoiHH3$&JT z`&^p$@vG#h91p3I4EJEIrHU(u5NV$ryEMC8nUvNin?a=wDL|os_nvaL06~b{DQCoC)8Bo*73t z31qVMqpgQwcOE)U)jyx*n?;YOUkjIwe?Lq7B2)R~B5^H&_{-^+>o0u6`ool5X}yoC{=E2Wd*toRdVzC6 zco8C`F{hmMo$Zz3XM=C^U;LG>^U);R^+NJERTxzA6jvfUNMa?if_{EESM~h3`tfDA zQ@34r`4f#1Np3xEHJ7ZnS!<|$7Hs&o_&2 z*E}qJ!5#VeL(x{~R&&*KcC2<5y8^p1(@E2eHA$|-SufUyxi=^`A~x$bataj>u?ch@}IcPrQ`cMBDg5sDS!cN5#>az%C>apT|Q+ZG#@+Sb{Y*;?J&9Tfh~FObAH z=E4+UJv7>`w!hhaIew{nsc-_@FdV1l+NDF~_{!teuF7iSYxQ_YT9BM}r?NySbEjS$TIk zsJXbgGOTJxi|2Q;1)-VdYH=RQJaZWHd z!zKTp*Le}&P<6A}SL`OuoMRNMeQ|U4*JY;iHqR0kp>QaU?ma=x5dvrHVciNA? zoq6^0nPX4FCshPBGkN+E`xOj*3>)@27hwK0{ykUISE@IXH!?SKh;I-p5J?c@k(7{R zf=+{^B-tbtf~SKyp`W1{!459C^}hD39#4O4l3fNgp!%q%Qt~VFBmbodo57Yjn zb>_brcC}hZYDaxXdeUWU7RI&uJU&d96z`AJD!lSjaf1%)6ptClkK%Imt4|w!hyCAr zFA^wIe9RKp=r?nF-j$G*>0j}%bL_7_-gtcVm|7t=rS#2oV@#DT%-Pbb`zVg2fEi2k zqzvbcPE~Y0li!%b#o5K<3kR*vGJ95zr`kG$FsVWO)3B|uqn|T2A`?4Tgx7h0JNIgo zZR$#{-#mKyL)+F+PxH!)I$-X14wXOC?dHto=;<7#9{Eq_-PvUy{?&X4gTI=fbG?%IVl3yj!CEZ8mN6B_?XZY;ldUU1rI-9yv+WCexwOiNj z&@&t1Nnr|iiXE2u3#$~fiL5!v#*Cu0F>{}CALlcXDayLKZHw`N8u-Z>$v(W&@Q0xe zk;aJ5wx+Ibp<&%;no_M%e5zEJ#VyI(5LVn;+(ydH*xwOtbiY5Zw?4Xc-gewrieo5d zz?JKZ*QCy)VYUuVC*{ZGeV38M8pSHwDHZ2Y?OjU)AKRO_W{go$&mI+PSRy`DhkkP? zJXXf9`!kI;BeUyAeJs3}^FqKSkGrx8Zo4scypA-GwYlt?CARN&^9wZ|yOX_=jnB}z zdDL}k^{8S&&tG_acY4nC_wLIc4cPY*QEqQ#{o{sl|GC@C2Fh8=6#M=kQ!dPB<5@E< z_RRYjdm@b)PO!1&OdkfAmROs}Q*U`M?KAmX)zf{=R(>Y^Bu1ZWA9nFLf8W;l1=w7U?*?yLL@vqTWj#`uQrBgF z%UYTZV6w2jo4S4n;c$j<+U6c#{-c(SkiCfX_~Ll|oMmxQ>Oa>paJ`$~TMF9~gJt+F z=Kk;}D}TI9^ANC91$apUC*X(>MkI-5ECi>sa}m)egXjEh4})%ht#HFq>yZZA&>&5? z2)9WHqx8v9+7EK`7*ug6+mw}=vQqsz*G^P9rCW%c84xm6Htc*V9$ZOvSe);lU8`WL z>)gJ*MP*_~qgvxVvjFJT#9aHuTO}m~Ca{fxfQ0w}0U2x|g1-n6RtPBn*hWBj0)8SO zAiobsKnFh`fPb$(A^pdv=vSYR|6?0T?e2?`YEmy=fS+on&gSO!E|w0inLA0~Y)E2O z>e{Z_N{T|J4t8wDW)3FiY@T+Gce@~ncnX0{J9AfKh^L*cy^D~iDE&XaAq2MX-e#wV z{NpRGHlp;}O0OVN4$kHfJ~j?E4tg;x2m~VHZ1z^@wY2QNb_f3xrMGl-brfP}_wevw z^WbK4aJFFQ6ciL>=ip-J;$j8gV0H1bcQy88wRd6o=RyACIMU`Wrp{K5u2v5Akh|j= zn>e_+iqg~Ho#;RR{Ii|ro>u>LCVQ8ET^6`N_PbBmIoUYa|KE0VwR-#ivD@7z|J>~# z*Y(fSiQFAb=#`bHxvh@0l^y6+&@?e#L0*x6oaX=ff#;^5)=@8|q)AN{vo@2){e#n}p+&iJkq#W+RS|F8G{^?ecc zyB_{;J^s(z`Hy$O{S?CzVgJu(Lk#Ql$KR9)2oeY{q$Slo5qGjs>q#`o!pY`_&4ZDc z&?2KTNtu|UqY-oINtrG)w4X*#vO>NZ6*GP&2pPLl!g0px#sO{QR8HCrcrWH)PnO-FKdOW(UP>u?!+D$v-k_-c`$E3||$(S`2t;dSYdoI><*h#>D)>oaSbw>(S zQl2QKr%vvN-W;xU_EdDzG|Itm`-S)A-;uKJq>Fl=B*?~6Oq~_x_%lry1~l6RoG(S( zxHld7RF7b?)mjeGZw}|byE++B={OqGGE8dnTESmm?MYE6RL(+yY1ca%Hym{{o4zS~ zogY2=O-&9@Xya4rihbu;r~m2J*lIL$sbk@5gMn<(nM<6`_y(8tm!F@9{r&PmxpH+> zO(jVRUzDe^B=a(c-XR?Q#r>Y~VOdDa^YkbM&um5&(bJFBb~Bpg)I8Er?cE=rG35B2 zrDeMG3qnV3w|32YlK+rfWYWDOW3PqxW(n6(Ikq8UhHcKFn$6Zb`HwgXI0m-{a8xIL zy#j4V4HhnT-<^K_MLzM>?p!MzicZ@_q#|NWMj_%^K3ZmI$kuf5v!1M@%5hcIa`?-O zM00$>zXV=47wh>>D;=EcA*V|$3FJ^(a$@_&UB1onl8l|n3WkEYg@9XqA@`l94=L;I zziXD5^fOMsIEi^MHlx2Dj!$c!M!)EZxQ*00)j?uQ0=s2)2$31nys z!54c18BMV@me3Lx!+1G{~Ku+#m@mt{i&R$SB*bb;)KuAC*zeN z=GPtze(NyCDcQF@j5h39>vHiltZD^{QC3HwqY`UuCZ>5Fa5B=n6MVk?O=kEjQ|E{4 zx2B7=3{vw}^Bz*G@#3`nNx7oyoyj?m1y<#qPGT}oL$-#$?^Bim!GjRXuw) zU1MW0M(DIq=de)BvpPX8;Ao`y1W8effE2EJKI<|-A`n5$ZZNy$e=;N&4F7!3mL^Eh zBTbQB>34hUebFp=<5YG!D24wb0*~5O@=H|}m`L>d#o)HOq%7Ktc%8+Dey19(F&V-% zsF07jY_Bug6AW&Sdf3T&+wl&GuO;EBQli;(+if!rt*nKa&e0#AsmWrIv9ZWeIfa@( zYEwrsZlB#RZKd@;|E#2Ou#p%4GM0k3nC)`#hno%a$agLF)=S=@;*4xo@yBpy@-Lj0m(W-s=1=lC(zFc}tUOMCSfi;A{A~B^hR1SHBB6kGpQPhi0W}6i zHe$A5rk7Dy^_+nRnfP7;{)y-&B{w__gC^?z!kkVVVi& zi%XvpGRRJ9v|Jr25OwFVVXBgvSWgzH>PuY7gpH>S;(LOWVV7KNz`+<(}yWY>$^P6Er0% z&yx>BRWb#J*jM86s4bo%!&ll(ZPV=yJb77^iRo%J#V{E>P^jP2 z2(US86;jxOpP?j>im7CaXrlEKmoBId&=t_lMKOsR|3qoU)w+Q#1k@>YlE&fl&$)h} z4oUUd%uo4|dqV-kU&qIokyoJoyG<5d6?6hqNM@<1@|>?{dwtS+Hf?isJRsh4b#d&6 z9w?ynNCYP$KAMC%-IF@=J)LoW99UUp7) zJoH}A^3eMbTyYDY^gn@!$bZ4nQhL}Ld;6EJ-_+S>8R99E+ZSJ;yP5W-^A0z8xnK(Y zToh@c)lQiD1lH`EbIkEozWYu160&y#Z9rfJXLGips>b*B8kPaUAiVmT7uW4sGv{hH zUaa%vQSwvOpY$zDp@C1`F+LE`w_a{BOBS7h!SL$@fpAQDNBp>#Td9r3u%#&~Nxl`& zxWWJW@(7H#sirN?BsX-V$GIbfI`Iwj*u+nbRte1GH(aErrEtRNglO#AA))Q|O^33E zI}w(#UoEQiOYU9a=e+6Sm+@MIS%)TFoVBHB{CYm^E6DiayW%ZNQ zNCD^i``!Ad{*KSxksoIALCzU)ZvjBrs(LZ5IqD_Az@q6>)Eh z(HehcLdh@js~*#OrEo3XNk}`1mL8pFXJXXy2Nf~S!uKV*b#M5Pnkd7tNFE5}1nL4z zX8t>x#C<1>%%I#}2pw(bQzj9db6&UB)+$Y4kt%SV-(g;)=T(pgx(>7DJ%`l`3`u!#}eDbY05;fzUBoR$`x`PY}d!GXZ!Sv(>5gw2h7;j zU97K&x@hs$Ya-6Ii-t57B*Rp`4el=-b9Pd#(Bj(%wo>LX@u92yw)NN^?_)3A+_-l6 zHts>va(m;`N}aXWM%_OwR5kEa-;{FSeo%cBHrnvJwJZo39Zkh2g2wx6*2u{%XXjl1 zkQ}YuI;U8Bfii?wge^rtB7QAj$) zk~Z>6T!^`lVwjsM;aFBB@9SZPm7kl1IreGjdfnHl&l|naRpD&E#kufMa0j5VAz*Eu z5!r(tN7WKi?-lq-2nATMm585lWw&ZH`BqW7q#EyDbw^uL_M`0vm0C4r+0Xq1ObO^l33gD)bLz$1?C427m#ObP}ioh1FqJgTE=Q z8iMo`Dw&><8bzDxVsTY}G*hK+Jhkhfr1M&u#We5b+3vRR)5XBIeC5y3%l&q1CQOnl z8(eQtu}MEFur)j?pc%dt9(n3lN8S)#9`AZ$4|S35_Nxx6K#=nG&KxK*))T}QQ}D3$ zD4l!wr++e>TlS+Bach9U#^@rgV!3LeGPSU{7^8b{-QzVu!^;HE464#7a8X@}a` zj(qO0DP|m2s)fER`4rwMHTls zM+Ec_BCNS=+HFFgF*So4=|{nP0wo&<{p-u^GP_ud;d-Z)kBwy`yxKlr11+NXR??I^ zA*oUZYQH*}>5Id(XQ#qgu^K+D&6f<4jgyx=|Ro2u( zt_goWFuqe>fjGV#d?sG}tsPIemn+by7jldmdq3F$!i!`@a@C#^nlMpmt}rCHwHhb9 zmr%FhKNH=C3CRj>?bo1fscywLroxssksfF#;EXRu?q4J(Fjp&7CXwc8PeXWaErk1R z)8WVFlc2?YoQa@UG2aLW#RU{Y?N&Ns-7#P&L%xPJcU-kfE`aVR0`>N|Usxqr2l9Rn zUHXbz*j<}bq~{%bi-3KS%sbLFBGgjCfnYVHPl{tj8mNRDh1uRk#H??43~D~c)A}Aj z{inb(qk{$aRDR|U<+0ttHWi+?GwP0AYLCFChl`&tB9jZd7nA%Ja9C(*W=%2lZ*bk8 z7cE6rX{zNG`-JPyEM!hv|50|OzxSCU4livI4BEV-g{0KUGApe23 zR;TK1)OW)?SzO`RE?+b9R8gPWs3xg!989g5U3gjR%@6i7siuzE!_>~%sGmK_tRL<$ z=O23{Q8CuV zyEz(oevD^4XF!%29HO6;{ZUJ2KT)hRl1-Ay&#`vsOEg6ra771LUp+}s2uh|Imks>q zr1I$c`^O61w>L0xU5>|X4z1TuM19VxzeyZid7o~XPmvgSZ{|z)9Z=m5%>+K^%A&?1 z7@=E}vQ#0hh#$S^=Rj|)GJ10rhE4XPozq5Io{>wMSL!65>&vnS8sd1G0(IO|Q!>H5 z{@p0I+neiMF0g)AKEvnGmw5Fn&O@9r+cS5ec_>c~J_E%SK)Pd1$>aW4=Z8==A6Lp} z)5h|h@#~Aj7YxB-Q=XXa#IQwAJ*VF!YqLQy*h{F&v>HX~6-9ubB$+#7)K(SN4QW~Q zW$@Q+;?DYyQkaONIItX6Doc-H)6UG83eVmdAr_P# zGdD_?nRbZtPb+%M#WBEY7wH0opPtuQkG%wJdo!mUZ7cXUkY3CZhw~Lu;skTzvKB@5 zlq2J<$q3JYxKpP_0MA+sm@-d!vGg-6uSl8(TNY;MHl~qiJ}c=#qYNRqh84^KSXT*f zyDr&cD4;STIZvg~P$m|@Qu`Wkzn!lp&!}f~O=n`UV|L4r@hV?u`d@C=HeDU{kngPk z4o50|+s?(!$Hnst`Ci!SfR>)%JCDUu2VnzFNP-twDBe&UiTPc2#gKEG0p;fiFo*6t z?F&o=)y|Q7tpL8*Qr==l-#WBh6o`PuowP6ZWGzd`?fo55JNo-)B_)3Psb+C1WV#uk zVLtxzUr|J;c9M<; zDz*o+7ScM!dQP{;&WV-(5bdeB#9W2t=!nB;I+oc<}V0xbYlij%hu|!!QS#a_S$u9JW76hp2SfoM9X38`KU-hX_6_k2P@_rbaUF{^DGVxdf~kdK9c~xrAgR7S z{sk8K?RZ5hwqwN1aQpftG@FoHy8DUe@`{id!W#nbL^1z^MQHLRjmvNZuGJi8jv#&d z+|@Fa>rNNV&jG7gNJzx$*7q{fFBcx)ewr$WyAw#pSsoi<`JCXABSD_a%53Brf}Rh7 zFo_tp_;Ru)v+9mhVT9w5oAe`cGLDq1HG8{3T~Xb($4~e1b1lFfWIgrElZ{&VA0aY43MUS|Udw_Y?1Q?V31 zEX4fhOes|*^4)ss>m9$pq7z5K`*M*g1c%I6YB~VsZ>F{5VYeS!5!;JPa5~3>SITZ5 z>m%KPBjDTi3~_}zj=7vkqevdmx7AMCfRC^9Up6r5-4o|Nf+Q%&)NQ=@O{D(P6^ou3 ztP3S#((clFokR$E(`oz;*-?R!!{JX;yArr)CZyC4K1qfYKee~$&lF6ip?cOa*R&Wl z4Q&A3f%*DxnNF4LD|T)_<#G4QW560KHt`qD7T0`LG&{58G(%f&E_-snBnr~)AS`z8 zmh$Fa5eCJ(!XwAyD${vwy3Y?+c1I^GOi|&t8oJHiHHtZ6a~%{^I*SNZjobt~;PT

Zk&39v`?>n@e4qNGLZ~$0}Y2Q1ojCdHscNYK#ho`?P=2 z?uRRYzA90R@z)XxWZL9PKe#(|_O#kBLIQw=FL#rB#4lAj>c_YgJPD@pqV5Zu5TEDw zTeN}mhlJE1QWF66J0SfxT(L#N7e;^HKPGkDwYQ%vH?B>ap-9{F-`Vf)=T_W3WPvJy!L9MZ|Be-&;DQ0HSiym4(Z@l4>4EEBFW@OJ)4s%BVPhxs zv>tpuO8(!6nGrrgS4Ni26%Y~cgh(qe!pyh1r$I%gsUsAZxE?`|MS?Q}S9rdi?TZ0x*>(&i3n9@HQvP za{P$e96V+~NWkoGSE4gOU~&f59WmrbBUKR32wkL1YG=k^t>7g7z$AH(K(QRl99#-q|8wBty_@&8h@jy@9f52BxeVe@9cm zxG}?g1h9FmhJQ&fbSE)qth52DkPnpk;VKK2Z$J$%Z}78+1A@<8kfQINCUWuj!)xH| zR-X~>dj%n(J_@+G@^8c@Vxt6c zA4_F$*gkt%(@}KmKHovhtoL@w@MkIX)-u44Dr0EH4Hh3I=qq$QZrqL721dZt9_m6M zu)4!kTypNwOAvG-d8*;eza4j&{oPhGpe)?d%B?5E3P@jCr6i>Y_~X4f;)VpjX$K5^c6lYA z1k%eyYbVYfX3o_teL}3D4Y$bdy0cy=`E1K=A0wmVeFTD|!MyLWlGn!PhzA-9GYuRm z)x1sL>C9Z);Gq=?_gQ4mJqWv!7vtsiF(U7j17HZ8n*rmB_0A&WahRVSzJN2jBAaaT zFZdEY6O(k6!m`IGmYf061!}_@YmEcP+1)g5i*6ucvl1NTu64+oaaQ7;Yz(yzS2;{o zJ}xK$NrTB#M=CW^qpjfm7Pzvbh(<^o4)h0*O@h@Aj59clg9C{!BU*qRN5&}w3CE)q zn|<9C0bj>@@&i9|+9>K2fFWBSK;TM?TG1Fz4na}FS^VSahoauq`JwnJcPb2U?kB`| z_VCdmM`v4O%xg`C9My=bM|ajw9>ItcKEeACG(Mwu77y5E0?I{d>CW`PB^h#AAM$gJ4bXTMp-+%3-2wnb=2302(fVJ9u0Ibx+V3OvXi>Ikf>vzWy%MD~>C zMo}F+d)T|nx pVjRjwhuPehaqgMgw6@sFqyl8U6a*r)v*BIwqgQv>-1ltC0<(cW zOPut}p*Vb}Dj)E9XJEi807xCIRM30wLDGQ~8m?C5m=86rmI z6FiV*P#MzJDUKK36Ht*6X^n}lcQ;tSIEy!y3S$y3Me^%CJm~-O;)4i@Ye{i`r@f6* z!`?Nps{K_@R)85*>9WCn=`<*b?#1MOVXn?ny~6Yw*bDXR)58~Cq9kvxoe%lMF8&gg zfjMS8{JYPb+k_X>&ZP#P=*cS;hNxz2_NKw^Z;Kl_*yCD-SSn%iDtUp|$CL8zv=n~! zyURDT&RgFKFYOlnhtM;nXra&5#(kK$F3ofK+ExI?>UAsvQ~rFG$NVO(+ziFV-saUN z%^uWg@MJ4n$n|{nC*_?A2j9NDWAT^2#cs|`%RCR4HPQS!ocnkuJedUG;7}ZVL)k)^ zlU5|LdC@X7YU_mGzblg`u7{xWo;D_FIEi^qZUYaFT1#?1u90&bfL+RfIQ;V#kUZdY zt8f_*g?XmqEv##o>IvcxOp@ertO$@414j66Ux~Hosb4-6zDVY&4h4cWAfX+La_rWy^_E!17~b zTGJJBklqK2)y!IlCgGochS!Y%p}(%R)p_?-?h99LJmG0^aGea>zBN-1v-HsSwr5Oe zx}gH}$Aq}CUl$eFn@1gB$-4Fx7{(;MtXz~X8SNaRl9O+0tj!ZAHrR)k8IbCM0K8@c2{^BS7xNOV-lpCDKw!v4DgbU5?e7~f&%SvZ3es4*#5@nb zO){qjP}*iaZj2vM7VgU(=JfCah{DzfxZRr6cDVtmoz$w$T}Xj9blqgIOtyAcq)FQ{eZ8q?%#WucVVPrT66$TFZlMHb zVvgbRu1z5V5nNF?ut1-tt)lNJYPgx6%TX^zE+b~7I*=880%6=53S@=qt%w_Bdm34G zELBJyd@U;H%Jn-a=V;45I8vmTj^DJ(vUgW9*0<BEXzzhdgZy$N zOjcQb-!M?SgTNUJt3c>hRliWr8q|G^?j@ZJ|L_kR_13=wIfh#xYH$}2$^YQ~yqY;D ziFeq&y#irEU@4SHaZq%>=~$ZKr%c|}wfYLdXF6*-a}VH)$}sdReTtY28{LtV5^{{~ z;2k1%sW;toiL{OyPPS36!2APNw9@?eiDHB`#lZ?wIU>D~U7KI!0KCcYtON?IW5g2F z`F$+YuE@o#lBdKUqB)KhRsa*$t_>`L7v=RVGm7EEUw#-(1ZdxZ(Y@JA7bkXg(4L#P zk$gF?&nRr8N2!ckFD{_KV?Ekc;2nrSPFghQHaG!hvMIL9cmBdbby5*{0DNj>#OpD9 zP)Z0hiuE^*a~AOW#T_Glf{xTGTaa;#xOV|ERtadh6u%;SmiP{E66tB_;UKt1Cr9J; z#WJ+HZ4;@p$~5(uCbw1gY^K?`>3K}2Iyr~iL$0@umTjky=ccoZS-}uIRdLIDet}5K zaV!^qXahvb)--4--I1n^d-$ujkF1t~SH4jZQe=NW7k7bvmf5!S0J+Ek3-3t80?sdM z0pUe2M(!`io%|ShtQ>s^#v__WKSIpV3eJ<;8c=@L`&hgmYrk-HdU;A2XtONX3NnMQ zf4Evfg%Q^YV~#-xWwH8nLDT`Z<3#gQoXY9odc^jTCg}>Lnpm4v=syCu{|m`V;KtZd zzZo8!07;Am`Z(U}hNuqc6S@M1VbW#wL5PV|=ua;*gswQ_c@I5gmuhs}pR}YF1c-#J zF-HXZggUrE1TF~-?`abk`XPa+xX+(TdtgL#X%d)QlJz4xT4PY;*0!T{hL%f)Mj#YE zYc^Fjj@w&oT1#t)PYriPA6ltjxVeURX8z=bc$&1pOydNftm$dGFU}&;aapFD%(@e9 z()w#comFQZG8%1DA?cC#HpYE5tB$fDHuo%jiq1$BIm4keVvk4d_*dejz92Ea$hs3! z$-s=-fkQ<~fC@0{{*3Rw(C;7+LBw|2ej>D#TcV!^5ai_Su|1o+9{zKF`m-e*0-hMH z-d5mdR~1@E-gF8)HxozkH~Z68<64B^!sNJX_Exb`XiBFY6Lr4>cv@D<^p@B&4^cNSk;PoA+#GRyi>;cHUWy*PasQekMdx=pdy>G6cbb+isgC08Jj(mJPh89MDW$fbP|?5-f3wLV@3Cse}AJYN#}JQHUlFY{RKVMkez0kuq zA;Bv?opIE)w2#kMnw?j$Mkw0|F>TE9<;RA9WA3)SzCin=cTO2*`lm(hDY8@nzS#pP z1$GA>Rb7JHBW^YF9oyL3$M?{ZCn-^ena(>K5ive5_`&qUM^c4pp4Sq+Othv5Uy)S! zSTvU;d^Tyy`f_Ghu7ndN^!h8qG>z4fKE?Hfflps1LhhE?+C%z+rISUJ)80`Tol&_* zg1L){xnkMoFfqXG?4Iw1ViGOu)9*Qnl>9b($d_Ulk? z>cGU65KE7TC{uS%i~En86)s|l1G#A*j&Ob8Nhbe6n3dYFixwYxd8d=K~kBzp4L@)o{u`#Kte zj@_%(0ustbZVHU?4dcdX4ftlN7z6roV$an(sOBXz`0UbUd?L!IS&)ZBuY3WZA=^X= ztUgJlfmo)}Qiq@H9O4kpY5%-Nqb%32Fp*Kza18QHlRt=P$IQJ=VbOV@M&_3(9?+7{ z9WO{YDVekMh%oatG)orCE7j!@0gp(Pqj+* z%DuzwPAGNAUflrp6{o{NjWUm#f(VgX<)M-XxS|CyZL>7P;L*9*lIX^KbE99%5NQFS zRHwHT1tQ4`}fa!N-`a?*NHW(@aA0*j}VN434;#10&sSsTdX3muEB$MUB5LF4%~=#*d5 z(RDGPMl&FmkugKF`{boLo@%4zP_BDC2#6#I3ZbG**Kg#$vn@g?UYUyJpSOg09Q|R; zkopCp)4g7(*U)OLhpqNP*pV%fI}RfSu}NW`@m#43qTP-(u)qq8Lt2+_4BeE$HQX`O z7&i~UJEB4T_-)2}=d*Ar`RYy53`|9y_x>hE^_=jA?XiNAfSxu^@JJYRE{j@d(ZDb) z@C0@tzXQ11wV{*0&oL+q_S!u6L3AF{e7jc;@WLu(LfXxWPf$g?;*(SgV~}a?z?@R% z*jvI0fg$1{_+6e5D!MG;kh3h}1RlP)&(NGi$FvOfn^p|dQ*q#|6pigZEB&#>q!Q-A zoR`=$4+iSqX66{@svC{FicEk?m=|*)Qcqv<*a2lxhqVwlv77GZB$1n%hXYpYT3JMz3Sl-CSc;)L4L zM7LJ4_K**t&!*9vdI$5k{Idiuy}MsAcC!4=rm|&Abp1$}n``Z6yl3hhJY8dec$cU3rqaAeBx_`zS^r11J_C3RvxSg6U&{7 zadz}zHr9EzkmGs&-1Rr@Z9^%BG0{F(0vB5Uu)*+tBcIR)m@U`Z-5%(f^Un%1mp4V${ryA?NtR4I z$gsX!Zrx?L`~}^tE5w%5Da`G58#u$^s&v!cCL+t^-x$J@FmL2W_IxN#_YDoJj3*yD zn)6i7O(X0-^;wQ$6se}=A!!jMaTSd{s{8SfK`5Xx7TXc`O(kdSVJKWJf~GZHNZym_ zDIC5vxoC`o!V0Doa$m|pMv_C(m`3Sv4=VF9PKZ6H8%?>GNz>0+pRy^r<#8v*1H2$F zE`<)vQ?;e{-TZCMrb}X01=#R$w0Mu5$)WEcZ`pfoR+ytY?9b%cUQoAzI8paQoS4%e0i=NB6^3t2H8d6G%{eOXfO^!r*B*eA6= z?9*ryO!9hq#p_XYg2(C{g`ZlY7Els@KOEtPtRQirLvkt9s2=^mpRo6g-Tq|A^s*!K zr&I5CDIPXwr`nWikfC}T8Obf{kzkTgDGvQICkYb$pLl^~jGnc{{;-L9i;q&SZj*PS z$h$XkW~X!!R3 zmdHJEnxrAS8u3uDwK9A{Kzs#T_1`OL2}8YY9*g%o{Nd;us223Lh9FikWpkuZ0n7}r z-=&o*{|iq59t?Pu%VGQ?x+01=-b*Y!q`!Ly6u^C!ig*c_K1&f*;x#cG1h!c3!B7z& zU_3Ai4M;$%8<9m&_b1j=x)%a%am zL~-uiak`JW@Sn9<5V<=el7WUu_{BXI+JE-11X2YslRA$UY5onb?=Qxk(e=yJgz?`n zx**BG?bD{2z0b`5FNZRSN)kAcah{4hkN%C;{d(|ek+vtq=bjyX_q7BD;B+lu$NxEQ zYIYp+gT4P-u`fn>#?p-PKK9gXePo8R_8-*NCl29uvua)Y_~rj_@LSfqI)J;HuT7A4 z`^yC~Znva;#Cz_bVB~exTP3T}uc}!2+@$27SnV!qbw{r+KzxRqw98lygv#)~=`~aq z1JIJQbpcWye?h%Y60o>-)UIFFaD%jA#SN&?;2dMpVS|BK#lynJ{T5vVP`5)a7DH)^fGiv9x?cRRhOF9yE_ZjngPX*ufOHT(9_1dRS9F{cFB z8cs@lIo+A6LVdnAU;Z!4&7$eOzIf3NK;tF18T<~)nxeYEce?Si0tu{Bk~m6iQkDdO z9lJq69W&6W;)t2lpX%=J_ur)hD<#_EDna#{a!1UIG7b`&?Inwkj4CufWCwnZXP_MO zAGIf-e#tpvxb_B^p{u2hd+$NHgU;d=fw^a#sym2NU}rX4kA5{Sq&qS6T1^V(TN#V8 zX5{WXxcd!MnZ4@9Qv^YhL@8b6Ry}`*m~WgiB}Pwwl2Dq@71(_q_Ic~E)E!&t-99er zQsKA51S&V`{0fQ8&n9c&svS_FXNDUOh(vc3JaTI;65e zK9QcY^LAU9z?<%Cfl1o4jF&#&k~cvan8;BV!xM^?xg0L@XhC_24Ztv|2+!O?K0H5f z47&!n{PC{73g7rr7k9voujCU13y{q&hjr{0y6kd=;?Roek+K_5O}R!rc>FirrZf@Y z-N|5By&wE+E!7eDJJ@Gdf)Q*%B}g&8$Q;N$a;`A> z&Bdasx+9N%4YFls1pY%F(&h0~g0GX=4cQ_Uf#98k`fO)q3J`J@A-wg(WRP2HjUy~2 zaKqvxMuLx*3=xJBR3^0CnY!;-LY_MC^CIRdfEp8Q9Z=!9@c7F{gi&})L3%QoUzVUt z!Gi38*j-}huG%n$T6kPM9`}n<#5uZ@Iz#{!9<*|2H-Z?euFg(GpFa9fD6o{Pd?eF= zU#RV~A^xi$6iB{#<4)#QZ#To|gGca*Onq53?mvDP0P7oK`|&fvu~_xJ$kYze`X2(A zlC;=3xg3{q+ih$8Ss6TtwaJj4chwFf82B0teoNnkW)AzYxUtY&2qCj##yl)~>PU#g5wkvYHJwAlduib#=R zsoe?k3gcMy&O?0QBk&q8$mBDqwX@dE(u0m^RLl|@<7EY9)$vp*p7=m{_?v>=;sA=v zI1AC6u#MmPqt3~X?k@^J;lX0)aBz?Ja28m7dw$|Q{T4^!+vpezCU1*#nnx7d^|T@Z zA`bi=2XoGNTmo^ZRCg}LE;@O=U5r|$%3Gtrv%?Z7cW8+0#~}Plp8C}`WPBcUd;u4_ zbnVtX3?}p+vY1C zrwR$rg(f#2Gu1C*IRY>T$v4i@%$-&0Jq?#1G&l@u-+tIt9^W>V%n3$w^1A%V%jJ3W zTBm}~-#tM|x7Ma0a8;9UR<^wPfN#39`C7>QTk(qrWAQdI=PnH~IaMzSR%r$XpZs#H zZtzud&6%({Lpq>3>U^!!WNq41z(v)`zX9y3IwzXbnRS0PRnB<_k8;(fnRDtFygprA z&9ownldt{UCk-mS*ST?O-#rbOrS-I%q!L#{?w}PYnR|G>|Jf)5y6e@$-!sC#k)=PwX#Wq;%sBM3Tu@_m6(y9(n!U5`}t&m%OeU{ zRdVr%s7B8vnxQh@&p!GM_6oC31Wjb4?LHh9o0q9#1#bdYN)O>Pqvn|0+E>l~@ANIY zuf`I49S@~pR3n0W{{vXN^1%Gs^`4bkqv)f@h!X!}J1H4^6^7wMEgzM8#fnVNJ7N&Yl}l>W36iBK~j z7ZzV>ZF}r!-9}Xnz^I!mUEj+CxRYiY8oYgH@~II{bz4o_-ChbrtoE?CR=Y60_Oi%y z3LkT^?MY$ZxB6HafNQ%blpYIh1Y&bZOSGr^O7Hp~3&d*jvX%^>+RK1JWW4jdZ67NJxXUA|Q>@($d`w-KiiTDbn4IbST~3oilU{ z48P6&oaemGd0zK@e&4@<`G?tiU;Db&TA%fKuPy+Z>phKv4;^(&5Y^}GWh48Bt7|FE<^_0vmT|@JR)0k37Q+r6i*?-h8asrU%xs17 zwmS>8Ry${%d=-W*g#zRg{K=2*WIF;1-d93-X?bz=XZt4jYV}S#!>+nLpS{f9D9wri zl{4eN&6FZ5?xs_F?ZcK#nB~@URbtd4D?mZEiOITD2}G4a(wn}a4YarOj8Z(jB;B?{nLNtL0 zwKs0N0uQeCJ#ga|g%2ybKF?!Ab6#nFK=%DDS?PN(>ZSm9-70e3dOpB($!pYmmA&+VNKy2@#p0=^4!%bj^$**SL*6LQQW8$9IyN;pP zTQAjhsl`uUuG}D$p&^%T9?T-JksCKi8lH7$odMz&7rKewb~Z5OwJv9OSd#esZ!m7u zkIGJA_(}Rzh}ERced}xYQ~c(>OP~B+!WqKL<+r$(SLF`#^nU~!e8mxa0?o`bhT^7J zG5I3r>26JRfnJcZ9R^DJeYUuEz}Uu3uH3P*`Dle$__<$Pt?t{bLfl4jz0^+Xvch|} zfY!!IYJS}sUiDJ^Zcm*3YEbAh)l5c1zRu~s??nao*71W$>qtE0Wvr#`_ImAd&j@za zDNAT2OuoKy`LeyWTYa{++N_Hh`uj=_T2$|Jw>nph0VRR*eEFUJuocU=jH-OUDb7=Y zFxwPqqu?)dcL=-c?=v=z33eMD;m?e@I^I8Ru8rK;elgczciC&F*;m;)f1yE1)IxlV zv)<}h@}uRnI6sD?%wuFpVMXO>%Y3QpG8A2WsOBuC@L?jThkts}Wq7nw%=`A$gV3ym zSG_A4)daOSIV=N(V@?!(z%lhdXVI%G$`XN*q82i?W9=Tek+LeLBkP!w9h$Uc)ByTo z8Iw6})zTyty%K|@uM$w>mnSf6*pIuZ!#Kc)mDI$0fk*4HM_|+wIj3k>{*I^XywdI% z-^ny=U*7?^h3Q?@w`3hVGqLkw_;<9g)^5Q*<|7$4pGks1ip@5P5?rGg zHV0>bBrSsB0;aNW0gQ%JLAHvOww?a6?-#i3F(Lw1QR9z+Qtagi*97Pz4z+n$ts%K$@Uj zEu3tB4HR2veQLf;{cS@z#xzFRf*r2<`R|~O;cs+(WD5{)2*B-jz{r|KYhzJRpAP@C z>hBN;_x~mXw9X%;G+nNKK<1`eKUWsNH`!Vpb?m1Nb+>=jp&e`fW2|EtN#sd-Sx1)~ zQ--N5tgjk4{k3BbtS0NVNa zag9-pzW^J4hU|4J0qQa8Oi*xNlb7rX3Pi*UBg!53WUW^W8<~H31MtHAym1SD+6yPZ5;E^b+Ol}{92!+!v5ITX zEnwV1i`FUrTLs8t8l&HOE2QV1Vsn=*!XtJtr-25p-%G!oO5V8)HXfkOlzX3rqK_%l zryapg`BGc*%J28}ZpOiFq|#1dizAoGn;bf=e*(XOy7e#;0+|qJwz;2%?c8U2`H30~ z;I0vv+2GqsT&DnW>5~yiy0K)A?2XsML6e$&dZoos8LkW_MizB{6PLBGi)Ozn$ z%xd$(xbpY~-$qz(b|U%v!#+{v^tSmjLFd1!<%pO50xZ{UqK5hXOZQ2F%U_7nlGq}( z5~&5SNt;LQ$g=sR=laB+o6m#@+)u$gn9P2Oy#JNoN>GcPEVM9&LcLoOoPZ&fx-0Y<#GncMaexBVH-*Y_v! zS^0_XssxubB@dU<_6xn8rJG@S3!H@F^A0#z!9B>!Cch z>#oApLQbYxc-5y&+82oqvF6ystEg)fBRY6LjJuZf%q))CjgEmPZZ zAONWFBGJp$#_|85!oXH&Ds)gG(i`+-A}usz_-H>V zFtyYll&7kP}on^M{nkiJby$i?MoF%1ieQ2ljXkw^a>qQcb-XzJK(-~8kq%z z?=w3vpI@T7ec>e?R*g1cUPdZ}D$s&ZZcJc#Qpk5+g+vkC^E!+m00Er4Y|SwYEW?}c zL?dMs&P$E;VajC!;1RpcS zeBMXNeSAUM2OtahnIjBa2Qd&9%xS^x(P#g{^C;G?*rrKf;E)4vb|1{`Dg5}i8W3LD z0lwBZgN&=vC_7|VzeE1lyU)8V-J2ZZvh$a{JnB!B-@Pz_LNum}OnVUO*5M-<&Q z0+$l@xH<`|I@vDMaqUm=h0eLh#%xqThhCqQO=ALJ7A zs!&Ag%H|p53GI2OKl=CmyjG9+Ee`8(=*2f3FSvigj@A&L`lq!UCR;J}JJ=Y42IeqEnTfsTq zv|)hYpYdwFB3uaGjl2xRArL^qqmc9z>EgkC*xnGvaiCstF~3$+yj+N0!SpuoOo@Ye z`LB?8L)S484KYsr?g!0#$5nN%0T207W_=WLd1#9~Y21eWV5EF>fjE@X>blTfkFb#w zY#ajnCJZtPj#057mt=xJ!WXee)Ip9IOJUP)--}jb9gP%R7Msm+Di`M_QH$yOkh&p@ z0}XS+15IB3QqrTZbf{Hs-gxQryjvAg>JOcGdbvh}$170ZKGty|I^~5^*J1OYYta6X z*Z%GS5dC{&r#eL$l2mlz#W<{KSB$eCT7{}IliW0UVG*4HO#(9JL=bx#S?TJtt zpk=FyR7wYzeRVeAF%QFP=iZuZ>|K6cG1;#T!GWv8DWR#^*)E&)1Fi7FC?cZ6(sHsb z>Irj~S6_j$X*Y$!j1rJdNT;({V;f@+1AMdIH6WnMILH6tBgq>W_@L`h{=-(+`4%01 z+2|!FwNGkntXFNQ-F!`DxlM@l+T}OcrQw}tt2)+dx?>wjnMjP~G|oXFysX(t+sX_V zu2jsU0;?FxE&E7}0=)z%8}z{I<}Ekb_}E~MroznGQQZywCM0@eW4@y&2v0se9iLI# zoF1mnK{ya?1+nFB#Z}4V7$QV`;epH?iT7AL-w?*cw|x~SNyE)$$hD9cQ;xpDgs)=+ z4BL{w^06gOnrA!pL4WZ0cKm%NbOq-JlJ{Tdl&v&*hfL|^6i#t{)s|1E`~7pDD)f8} z{v45}Tl#(w(L!`!ny$Ygy+y!#{$x;$f>%L3^Sel+)1OU`;@o4EEe3;(#g1&p9#(WX zT#l+CQ=R4Ghm1Sx0NxUxtTb=;kAh{6Z&@{X+YFe#{L-%ce5p5jD0+FXi7o_BZ*RbQ zkPT|9Tm$sjiTp^j%!-xh)mm6{zzWGw5rIhB-Vw~HxbO;H=k$unSj^~{O(pZm^CG7| z=tt#S2c~D!lmQ;lV)YlqB3pJOGbaV&iTttmC5%=_H&&ZX=?3+d#RfR!%o>qCX2H(Z z)IRo@DMce<)tdM1>(7f!%omkY>Z@@hC5_!SwvSxXShG+^#qee}+Zj(f`p5%EdNWsrnw9^b><~fj0~p@w9igJ$-lHrnD_-cf@|>`!F=G(Rd=0iPOmQ0pd1h1 ztcL)X^z)!1_4gAGPl+*xNuBKdMzP60;@_=F;1i}VU38ER8l@b$wNyr5$>ziCfzq>* zX^DmBaRbya1Z~23_3mt>-c5gEi2Mw&?atP#(?bdLkzy!@E1#^enhRZo+Tf6KwNnTz zI@G(G@~K4v{jj3jhA5#q&L69;Uc3Vi+YqO>qcr24bCUEANA_$ZU}Vm}ma^DL3PZef zLbXwXAVYGfqFJAAg>v+P`qy!`Z67}G-z_Db58rKsH|>dLb@V)QDBLVq7vygE$m{;f z1%8I$(q2-3`dMJX(7HL$d{)o>v5^jy=V3yhYejq*efO#K#cG?~4(G}Z2Pv;Z_~1fq zwBv9m4*9XbH2Bj%QvWr%R<^wFpG;`x3Otyd|5I&2)m8*a0ted%PJ`+>X*K(fN@>jo z34IS}ncC(At&_=WP|vsxjDl5{CRW#b-QRJAQSRNch;TZ7m(6PZT2XNszCeJ{p^-=j z(#yY{ZEsiz?uMJ}G$-w(CmUj$WV4M?B}X{Q#p<($^s6SuxJAmom(DSmMv}y7uu1k+ zpHyr8>7tI+$j3D;9VyQdK|<&M=cP>uY*m zW&)*7-QSoYf0--3qg=|Xb11*Cy7_6H?F_1wUR~pn-ocS@4dp#NzAm`d2HK7owpj=j zvv{g$OKvq<`Eb~o1JUWqoPnXU=^b*~G}l#rQ>Y!##gLeR6I;*CqkJVyZm;>lduvV#nAIEINnk^JPRG^ZbRn!G+P zA=Thanz)U-3|==;>E+;HZRuSrwmW%r)EPPHT6#}PyaFM&M&TB%=>`&v18^c=tNnXw zd(LBhW2GDixt&ISb6jb-zS9sNGC!BJ>&Op2_6aRXh}({KM<2!5V3@ruall`;*)Fgk zRm^0!LsWc)Kl}T9xeB-pagg-ZB9DClX|KvU<6od8yZGp3BZ<&K9G;g8acy6zL#o}wU zdLPy+<5f!O?KtO%)$k_tP>dEkAZ}^`9CF-}#_IP#dOc^9@XYKSWRM>5#u+P!w1fKd zj%ku!-9np_3KTb{LuCVy4`Z8%3iX=n1S$5Mk>ht9xTeg*s=y{L^Uri(N0NEqu={>6 zZ1bD)JUqtnqleZ9=pq@Bjep#0=fAJ^X6=3~&Y31)TYJUG+)``jRZNzB`t->l{`!h* z%CW&rsI*FmvOS}*F2sy=R$NU{>!pD@FRFKlP(LvUZ)MJ?nQmz3i{=xO!sB7ui8o`C z9*WYvZEm>J*qGHx$Ai-Af2B(Yig&%CJ#w>EhCOJ}H5pc&;@LAgX$E{n5sQFQ^~zyg z`njNE4h?$@+*R1TuK%3_4wc7dyenE8c|RE*ugSpdlIwX#k0RH@o&N-n3+o+3xj44> zlV(VnTLv9^!aVCPy-pWjInK~1aYiirh7w&QD8t9~8M3Eq@BPm$H#08xvuT3dD4}JO z=_(y@@75xstLl=c_GxeOVp#@`MdfP$82@`gx1t$CB538e(Kl;@wzqYARmNRrbCnJ| zRX!8MOUrZ_$96M46GxPug9{~3gYL4l+Y}03SL)ZdQr^n3QB2eh!!4~KD+?|ZkbcQV zQd(-b@Pn9jpTmX#^hLu_Oz_M#eGX53Y3QL{@a#y%p-`v=e%OORxz(7D0G6Rm-_pF` zv(+@)opD3qo%bMSKV8%_g)+$-M&+N#Uv-iob!Q640m89ctg0U>Hm6W=o=h{(n3p{` z0W6!m%@&$8;c=k@K{_>hlhZ6=H6rX{{C}hTiV- zs3w=|HS;quUIXr_p>QO;O|MQ0;T+}1me)H4x^zC2r|IYm;&sCOfH=zh*vtUv$yl%W z+v6@t_wGC9QiPMNBLbF0HJz1gy~O(8eeg({jTPH(*)A*Qwi2MGN#~IkCq1aq69$A11w+PTbykUMbW&1^gSb9OCe9#cKade_2ZuggQr6tYE! z^safj^ffSna`*JjmSXpe;2}>7C;0uzo!<8+i^9tsiv+LO+PRdtf&&-aWDaO_0E^_iR8JHo(?82@HQXMjCQi0~6ZmQQkDZB51@;rq z8+Ov>++@t zoI`wn0NS}F>-|V^;{+}9lw*vXeLBfQn7Yk9B}UYay|g$`x1HEQBJ^e8=(vGrPB zoi_=zM|bt~J5T9)tq$J_#yZ~D>W{{aQv@91Zjk(8BnyTjC|gxTJs>y6g(psZG2=CR zU3nb!VGmhh@bMW4Y57jMu#(w}PwYG-kLfu}Z4U20tFY{L|uU;Xyp?5&+fS<$ke_qrt+uPn}f<&rIv|YB&qpQ#(4flNo zG@>SU9BiYpxHpFMjJKMGswAsfb$#Y(xg`-{;ygmOSxOJwPd5{~9#XQAK2kCb-srq6 zxEN`BqPy_%qq+Gab*>UFmx)Ki?YY$50KHXjtS9y`<4KSk;jpdGLqIdXu9uG-N1X7> zg6&L%TILv!fzsE71Fb1m{J%MOiV{z*uh+rTH^&Nfhxkr1$N84omg&-X_jLewh)lj56}S z*1{dXQR|-AK6(D>z4J?{HEHx!E<+roeH6b#BzufezBFWBo5J+!jiz*+z$Z?Y#QwVd zgdemipSTJlZHCZGj7fgY*Iv_8pk11=5e>n9QUjs(7d=%kZ$1|C9}Vi?2TY=~i>KhA zO!sxnNY!U@b5P7dz!1&*1|$19@m*yN;Ij_cT?fwc+MJiv*XqgVjkacQQsextR*&+sDq9qobUiPU7I_;Wy(9syfo zYs$wz^T*$dQQ>4$ygH_0mnXqJxp@Q_19-U~73h7@?kQw(`{a9jKs8cdVK2(z4v=OB z(5bywja`h9oQi%?Y~Rav^Qq?l(o0%?zK%x+Kf#d@klbYc+bFntkLL1(b%7dC4kG!* z0GVbGlhIwQAw8w4!*ezy%X3E0?GI25F_(nS35lWy6#yi*X<%>{O0>EAHtCLehDToX z8<;KtglQF*kBmVGcj->KU%LM${bnB|!RwM>{4B2%q;~#f1r63wkGv4ZA8MMvQl|W8 z8Ymf=!2QmiBzM3RC7gwp-V*b<)J0+LIBDOyG5l?%2p^>frFHz?Ij89KRCND$xxo1gXj z)&+D9)7vxn|B@5w(v}RtTNGbrg|)l))PtXkta1fhy-vKMoMVo=04X86hpy@RyAceI za%yY#QJ^f?K{7d$74G8)Vy7EB6_Wc`fWoL_7}h-)Zd6~`Khx616?to?x&toEVjYji zX&u0Y{H--tCuY9R|1?bY?0!d`$fOy;C8ZA)os#wni%6+N=PZ61YY}n9ES_p8`H}*I zX7h@&Rbqa{TXHckPO@zV=5d-O8@1x7PgMBnXL3|prQ~h2ell$-LMllsDYQm6^>`z< z0}~yi+jkK1+xddYqsj8w16ln{ITL-t$PzT>>%oY~Bn2wXt}o(1_l>v_dCl>!rQWxq zM761GUHCGme`47#jeX%fF)w%OiI&^z(S{l9{W|z5B=z0bP{jik8@#K0Od>1*mh-*N z6RY>=K~UOW^H)+r*9650U*CNDUI5eW?B07a^ynFq``Hmy5x#_O#u=wv9oa9@I z!t~o6!~}qB?k0v$cg7R+say>`vFM|T{5gdUlpZ)_moPjY5?4{(KRFa^4NY zPt8y1nV}L=6Td&T6-A)2pBYa7GdaKe2>Io883tucJ?MO|ufC|ci&`uHz!AY?Y=9eI z4e+NO0jVUFG?2w0NBQ?!2)XsJ3A5qRTe?SlUbqZ^yidBlI9Q-q4BM5;v3#`#)cQta z!c22IdZ}TL{TEI%#^Nl_$KhT9$)I>_yk2?HXjV*^oPhvecdDC!-hF^PM*K4h+U$*a zup@0mbHBFkQ)EsGq_{n#J;X_fQKraK#K{~?2z1R^JtyYMQ^8TWw*%<5QT7of_+Y#fsw_Ry>o+e#+^KlOQotg3 z+%epk=qc{sQ>s*K43`}NG~H@|Hddg#sKqFa|1RaoYWqaFfq3NSKKLfKG0taV3YD}s zya4yAlu~XQMZ_Fqj+E0VA6X!PhK%^dV|_OAQ@1vzkaE^i$>qWYYa$RqpstH;*UPnK zdTc-UqJ9+5oUk1xbaNbA?Vk($U36Wr;s3pAf}BsaBN0$V8fwqgdHvqZ-bC^e1!F-4 zt%E{|f zX6Hd6cSVp_1QP1r8+inqibS6moHGr#A}q1ssrwpZTU>6!#Nlc01luGHH)h{4Zbhi! z)>+&vvmwX2SNg76pALEUX~sR9HxJwQ(U9nE+lc>iAAAt|O--XF$~^e#X1tc2h<*lC z@U8xhS;P`K-H5@2v7!MvncC5zF1jI_%hv{-r7B1 zkGI#>)Nd%SCHw|uuEixZ9D%FrnM2*KkwpWnSRLFP^chz1cVb@V!(P4HMjS`sD6rvECv|R0xYSBEdJ5 zxEU?Cq?ThICLwGix;kphRIvQTc}sw5E+9MC=Gdj8`g3!*5cEpDf74%mCDwgtc?Bk) z{*t5KRCsq0vQr~ci^I+cU3B=`-bkBbTJPI%FU%>jC;&)9@h!zM=r^a^s^(c=$Lmfi zw^{s83zvC*2VcL=8o(grlQ*lHpimUaUSpZ^MgKfY3lhwdM{5- zG10lr9Fs8e0{8|myiHkd_2D(O)m`)S3lw1<_MBCQEC};J@F0k6FOxk}iQ119rQ(@6 z$eLu(JOy-`z{2LCK=iGJ(Phl#H~#Fj@ek2<AtITjo>~>4R^U?e#G(ajo&vu}&P6=Uwlwjeomx`R#u1 zkNZIxVE1Za$ru%xHbaXofI+$+Kt1)mR|8K$qEMxuHnW1V|E`kKt>G<&m!OAZ>sIER zhTaV(M7t@TKgGLDQGABYZx55W{UWW^$0MRxw?$MG8`bYG*d!!V#_2+cE z+-9qTpdHi}@i5=vLueqmRsl(@SJ4P~rmvYWLz31T_v3RX;Ylaru@O@~QL&&y_c|H1 zPqe{abN5(GcW!D$TiIpRA9woLvI;Y*z9fGIJ$SO{3mD}6kU-(dzEmI{%B)%b)QTqk zDQr3%n_}A1^HLRaUK-LN4mkZ*({+I6)lVy>06+8HT$SlY%!J8Rco@Vax<6vg!|3kK zs;qD!S@xQWXIq$g^dXLoSUQalmGHzXv{k`XpBI`eP8)Pnzq52*dqwv-X9}?*2nX4N zzeceqL{~kYL5-h}%zR=`nZ23*96yg?{ew6n`u+0XFD*wgq@013IU9M=&M-CkiZOCa?Ku>y{ z2eMIz_!-CD>n{>*IP2mEg;>dxA*m84^T9{{l@uSE0W znz_6K|KDfl638ixAb+a(Tb=KvUpW*WzLySYrZ(7(-5$J7sjyo|Jj@B>3N#i~J%|hI zn0qPkw|DfS{35bg>chx~)#Tn+E-HczN}?txec@dyy&Gd&qz74N+O~N^twt$I zyM}c=uLgy`2@SPR6oQNy+{cnD5o)2?$drSxNZ(6ymm6D`)03(YN^|eF#K=%4dT!%# zbhXGmTl3> zBDFZv-wpnGM%$x~5@RXdlN=J#2j=L}-l<)=ZiwOT=3Tc~D9wT*2qK5jWb<4iO-Hz1OzObD{2gRWidD%5|Z1`99gkrF81jrN~?d{gZSp zFL|=cNR89;=@btiD)*S=n2k9KUU2)X_g67SV8bNAyr;aciVOVZ{AJ(vtd4oax{mm5 zC38+%;(kZxo<=(0yWOZD!&*hWr<%N*^~EUmFY6`GsKSrN%IXE_W9O=_PKDLi5~*UJ zK&;cbky$K@CJI1&^T8m~FX$a_6vw+DAFdmC0A$-sOlLQ%YueJB$?56s(+au8%26NW zCc$LD|M^?W%Zk~Qx@59+{_RqgM zIV&G;w1_FG*hJX>y%pkInWl`+E~ix?!0Qu34Y^cx&{aW0`P#LyF9w`#-x%9SL+X^g zvhks7+c{D|+M8&Pc3E!f8@bf`=68qBNfdOG`YN%rOR;WIcfWi>y+ghjho(=Pg3n5` zVz*qEjZuAVv?PUJqhVYhTP`LzrKi^~7+ELHRqC1T456Vv2+jy5HRnhBLF|pHESm?z z;F_?Rqx7>sd1Eh+b!~SX*fyUUbNN-ujz?R;Q$M2p0u55@z9hqIFPY%Zdum7sq6Zy0 zP7q*FQ_!h~Sq_N@H!YGrHH3G?*Rb0gV`QO$xRAM|615`0sD|EwI2WoyWy{j2Eb4(s zd6Our9@lEXG{7jU|9=)4utW`4Q6{T}$B&X$%giv3MQDCf!ZjA{;^lRcx_!vLuH<)$ z_3qbl?w5*EVA3AZPRik6-HbUBolnsVYsS2h@lGe!WA1%wAmfQ~Bcum>FWJ$)8Z0=K zyR+1)mYtXTid2OgE_gP5;83Ku5RLsVFxZQH=hanBx?9ZBMDhsJo3#2rN(BO?S3)T#?KI^LrO^6%aR3=YI#QQ8i-WBdB+?x_EY6ujRI(sFOLej z5Cziv&JgDx;Y$4}z<#q4ad-Ylz)jKeH;_T}`@4o)Y9o>Xfp1X@>f>&WYZ_4~Ppx6M z9~N;v^V{BF-cua*mIhLJ&!_mx0moLv$# z!BuIpU9dM-dyQhs`{t`kcvBKkzFah;nE>yk{}HtV){t<1<+7k%{?TliTLfI&9R2E4 zzr&!ua)MNj7Qy`7_&FVlG#{i_(ufZ&%%aVpdmmJWuJ=TolI6_x+TpU`~ z(+(CIizfUJ%8p*ef#e&@ya6(VQUe`QmSPwQ#KqT8BIRH*kh*(rg>3VaQ5I0{H}$(r zUfI7)yzf+8ZdQkVP>vNl9x%KMS{VPcr2oGy`~E+fK_hp5A)C0i3(Os*UeWEbqjxGq z*IpvmPw=(;U0CotkL6l=@Kx}{Rp)rRi~cecO=Rm&WUu>lbRga5l6)sr;6~BFxI;(v z4!b_?U_I73Sr!dRJ|={6N1Mo8IvY2ai8H0x&a5~%R{B=VtP@^N=*1FUAMv}knHV>7 z^lF~FDJ__~9`d49OyEVc8RZ*uVT|4YlB~*5wGpMHP_zejwh9QV30ZYEw-=O-#5(Q} zPliTl*$r}j&b4b21<>44vZNHeM%H)@em?cHC3=X)lc*&FMU^Iw7b>a>nU*9J;z>B@ z5xrW@QWpP;E71XA5Es=K{`b4k|J8flUV#^8IeXS#_5^>nMJ_-n$XRH9d1|8rm(v#_ z(R$@XPAr;tzeKL=Lv+?9BwaMXpm~%i5adGydy0cYMRfj@j}Q_bBRX&A`7=b)E#gK- zdmDUR8WwBnIxmW*&F*S^m{E6jj2`!;(W`mvRVHh!^TNlmm7zhb2W{JW2hOe*B1HOz ze&vIqI%rThR6h{ZtQ!sb%yuVt?gk3gpG9Q=QFAy!Aq(dND}S-(-_!d zLy!W>kPJatAs`tW1eGyJZ~J+H3U)SV_r&3)G?K(2w)5LH1C=r~(AGSvL@r@$@wf54 zjhAl|x4pEI8cdvyn%yqq{~06vmsyd<3Ym>XRz2?AZjRpnnJ`0zWs>w| z?c17P3t@MSr>n7ESiBUF@F1GQneSI&@LIO_n&030W5F}3ROt_s3MvfLi$1q`-jI-x zFo)gR#`>_I>KLuCE;r|=>wd-T9ll7T@<912I$<)f7=omEU2nYD$cJGMTs{Co@ z#Dx>uB^r-xhylj?EW)dDWNH=4_3LJ(cR4H~Wj<}>q zftzTB;+OPV9vvIRy9oKSS-GCh_qK(HS^lkB>E7^Xnw>v8J^+>TY+|kUpG@Y$X7z6f zVaMfj3xnJ%ofP*|&(~?SUJEm##kVQ6UTb`yc)x$L$f@aIAay!Z*kE8lTiJvau2oP_ zo-KaM!l`c5hGKl7S=afS61egh+Fj%MkQKMc8`ouhr&m8`$~N4c3G8>CUA0Wmv|FMW z=BM|9l-8&xra_I?O4zQDF$y>VOozM+zABs%Z$h!y`@T+`nNV|+V?h%SO2Y%; z)P!4K$lC+s@em_qtR))IhdfVK;V%K)Ac^Y-$@?Kn%Mj-t(HDIqOT3a+=>zusb6PHJ&>?mFFL>YimGG#&&a-AZ)x%}ZabES)^%DnWDu12 z%Z)2xcVjUx!|5llL25MxA0)}Xi%))_#iRVpcA7eR2fkugpVj(FZFEDs+l-EJyVvc5 z1%KkrvUpU1Xa8CE*J>UQ@*M=k&Uenp$REh)6GJ|=%r zyLjHa>}MLPA2jlo;cUlN$%rb)a>B&(FgPP)Xb=#Ig721SZ%PYralahM*!0PbSFg54 z;UEvU9HiBSfHd#wH%w$f{C90c4^ltlq`^cI&B$Q|*U*`tI-$sA;>m?f#@4>DbTGWRwl#Kj; zLfPpTMZyTjBRXlC`nFQ<7>V*4`rX1QRNCdENOCZZJ8XPCBOn8O&jt*XR$sLfVS#BC zRFwaOx1v8T&xJ}rvm{>m;XL`RUygNGqfPnwx!Bx?LClIaoI#=>D1LQqzW~snNOJ;{?2tgg;mr;md zDEN63$Oe=9Q6#g^*7HmME;w;ZHBS;!=?RmXb;IEoX?r$i*W)AH%O}t?Y+W9{@U1;w zwjDurNP4xyk1T-#df`L{n#BNtAGlEhJE-VE(xBb*w{H0cO+t67?h-u@qDQJlc%TU3 zDH@t@8OMBV7%d1XQrCi=hbEixi^XEqY+xQOCQt5ZvZbo7L84Y{0MI8XildMH58?}n zJe2jyh9Ev^$V|jc_tjsfH~k2Cr)JOnTWNKcLEav|G~b*SX=LDWF-^X+e$jGW3$Z&I zXnqgBdHIq(aeN#Y)(5xc^bNa(t%urMqNgpenwWWVEXJV7@*u%eD~<2!;$Y3y>unRD z_M?01F9H^ztTB;Jv9=%~Z~WcU$eN<7hnGYTMxfDg4l1HcxFTNktI_eY5GhK0Rb#q` zqM>o_GX1~X@8kax#NtB-G1vb*DTiPo?#15P{m_&Jc<}o)T-t8?G0p@z{xQacaJ)?rCK5t?s-a5j1H{9e3G_j6Sxj5 z955aKR}*JW!uT})2uuNp*7XqW8^D43Eb4?jm==U~aAPO+paX>vU`o*Y1ev=Q@3bj5 z@t#BQx*wK=c-`I_6=aS+cv3l|(0S=FT+hGU^R+BZP|Bo9%pd-ab#cq7+LhIn z$Z}2n@>GQLpFA^7lx#%S^)xiDjASMpvg8_mXqTuh=hnM?2fpb_G+ocsW?u_;k=^7B zYB$iMWh5(F@jTJ_NsTOymfvnCbuJv-Ut$|T4onfij_|ubGKyNE4}vhISWyn=Se72VSl+1{HYaNj(X_VQ>ys3l9EC9Q>hou5qDsM()L)Vk3YG;ojH;N z4{43o5R$|-*a0QN1X8-y8{8A>Kq5r+4n*$*G8pmX4W{A&v4KvKbG>un4kR_<*h1r^ zjooJ!LSX>d;AWZEME5Vsw^V1=r`FTOrHTyPY7!K!DKb)Lsz1P}`6os2{>c^fqD)%8 z2Ck^yV2&Z3vm`z)WGsflg6C%~kDzQ#j{Mz55u-rvo6l^QUSBj;%L@}!FE%;PE-$v_ z!tB{eB_(R4n0D6rfyI3Dul*1E3Fxkl_U@M#juptT3xOWUb~l5@z{N!K<{2FKq>X&N z%@a;yXGL7@iAbt}=Sp~HV=%N1$V+FF*>tarzx-M1)m?8L=)O}3d+;Pe0RCl1DOxPV z+e|Ey|CiPJzp>~PFOl)w&!u)D#~PTn^NeHLuHm7b_xegLqx3(2IG!jLC-vuep73hE zRk2A-uuUaW^Qw>?|HJ;9ZvLGwJhiu_I<*B-`T7!0u*3{3Ug5&w(iHVbjTle)u= z(V+Kx>r3>|Cm-s04-8sT*wZ9>aI04^;!+eOoU%YJrNpU-q-~}=n+;y zaSmTlxEgf=zSW99MLmvvIJp;yVSrIE-!_EYuO}5sF^8T1Q_Qyzqc9o zATB_a`f>e4P@Ec9{vQ!WFdw(DYd%8Pg>M5r5Bp5vhgM^VN-4~R2J>|?S8>&kY)cbl z3?j8EB8q&fss8Cs0n zy4BSclGZbjF{i%FB2tVQgnXh1`85?h^lIJN|K_%|h!nnVyV%U{&y0C76W_keXgGl3 ze&5A~5ie1)BOV072H)RLY2cIUFeAdF*E0+CADTbqI&X-vuFNq7|0Eof2k*D#>fKT5 zed;|TSozDLdKPk{>M_U*1Oa{>Ojw9O#{T<%D<1uiz@Ul=e1V#4wPj`r@lHW`eaPiw zP#kIVccWXK^eMl31Vr;&cOJj|STb3m>d&_KQVvqPp>pE4TXn#md|VQibeg66udxm< zky&2T+%CjDu&9kPV*E`p}pKFvne*e9~MLYr)S0MDF)eccBg{GBUw<2%ki;i`#Ny1GRJ%WZAiAiv@-l@dyRn1MIT^{0n{=hj#p|3?7atZCs*KE=QCwz_pUss&TqvV6 zJ^m|FSLAtA%qH3~TB^u>aJ)M&Lr+HX;FqF=r?IIlPtBw7p!-Xzzl#;I)HmJo-34B( z*)-?s4fKu1+^fR3)pGY|_l}3ghjHJ*`r@55`tw`gl~K5$7{2dp;oGNzS1fUz%~brP za$)|wOy&~nt!QghJxx~aqX8xUq~2a@WW+IE^ITc zZI)Rer=FXUBov(A^7-U%ek`d@r!f?TyE@@pK_^dDnguNP5?~(83P+tA-D+tm37K=6 zT|q(l|D<#);E2)PiMOX;ulMQLO9|NuSoI5;=iBT1rPLOa_OoZq;pS%F)MHAMB8uM4 zjU6Vjw>`+0%R%1wUpgqpB66@B<51*R?eY9z7Ml^ zZGKmN@%1giK0}RXl6uL?H{L;r2;mxKDy4f*&fd9$qgs>mE0)N$%Mee62g-TlX;y zb-$`37xoWo4wGhVacXlGIS=R-@)?I+1_P%}dPTTbNI+XE#?C+4L(ybt)}OxH892z@ z;vBJfnDmm)etTOsU2*q@sI5u)2c1kclf0r@pn8#(c(-r)4EvQ>tL6`3DOnNn$sT+2 zf4u;j$ip8Vdg$2h>d~KW&GDgcDl#j69scp|#7g=btXMeb%aZL$(64Z<{G}}E)P>x$ zKS_+lSPEUjN=a3Dx7{`rPK0-qTmt`u?ZqX4jAtMrq_4|d3Q-wBP{O6xPsGrpzN}38 z9!&_j|8+i50xUHbwXOR1rlzK<>go(_kWG!f4_!$A^{S?zcm_e-;cn>0jw?>7o1#1E zG`XL8UA}we7S~D6AF$_q?r~+cykcC={+?QS3FHLZMA0$$i6N20>RtZbiiZ@a5QPBa zf3m^6!3(<$96vdgy*Pw?qGRnw$a@qzQ@(z)8Dyy{TKKle%2V>XtGKP9K^yBSA9fv< zN%8CAN)W+gARFb5EZX@ne^lCAQJ9k>J^0}h9VHPMF_3G$M+}^x^E*n$dSWLW`nmB+ z=cJJ_c!kh|J>`xzT>h8;Y;}SpmEB60%tYX7;rDA_%AGSs6tQ-czZE$|tE#EdCnhGU ze){yKSc-l2ZjZO)yu~)BCZ}KV80Oe7gK^0mvyIeWyHhLvg(Oe!n?zsA=Q(4 zF;;sS8Sdni0hV1jq3K5M-R;|Eqfb-!bg}_|Qawrp!kPUU&5Edz&cp zTB+A<&`E5u+n0ti$tIG)gmL5icNep0^p{%d>ZHnA2UD3x9*v^^x?K#&Vfq25II>El zzu@@TM{EeU#KheXFA|I0-uizp93}+H(AImhmR4WlZtK&Tg6BCR6hre4@ninU+aXW$ z6uHC4X8g7v!q8wA8KzJWVa4!;{xVZcp`}kbX`cI;s-7Ol*49?*av8WI`7u5*ZCe*?bDbeJ-LoH64ZZGsj6ZG zPmAR(B$ZA0=4lmIt-QvDG}K9Yvt1@-a!Kv?o(G*=dA9mUUYmRA+y0ArO`ihl_6&H=FhzoH#oTznBLdYSy6WKIaed?;r+5sesHz^(Z?YO z%oiHkdu_YO@`$KuCA=#Dkjxf5hCeJVFjqZ-B!A{Gr|?;Q2PW)@?=|VZ@CImCxz*Kb zK-a-tPAD*pDX>JlQgZpSu0AC|d{0(4xgG_!fS4-xH9ZB3S;Uc%k!@a-Ls~KQ_~V{F zbm02ZxGj6+m*_UWIy&y309Z4LkE3|XfbRS7eYx4U%;^krlr9O2UeH#{t0Oy8FR$ji z1A~XJGp)xi0OL3zXy^*M!U-TAYFIZ^r_k|TR|m5&yLok&0buJUFyTD$_}KPLzgxJx ztju<_ZyQNItGHMp!*idNhnJ5prD*^__`qAfz2Zn+lwNY-zw#WWPC@Y&%M<~CKmkyx zP43C?5y2e_U^0&N@-occTls~4i#9CA=PY$4?KQ>XMR-eNh3`v-Z+5-DIPq7-cFjlp zjeB{iCII3l54zCOwu4M3*##m-Vj`o$k^MiP#! z0|mvdhLn-}t$;rvn@*W<-C@o+54Ih74+(tI3lz!H^zllQTIsrGX;wypsow>GPJIf0 zSmi89AEZZo>)c}RN}T2$zHL<}am4ZLC@0n~;9vy#<|wSVrwyi$vy7C@vWefHeg0VG z2=8zExQv8}O^cKe3WIU7vc?w`u{q_<4i-3H2jK5w%g-UCH&Wc)B#_Ikzegey}2 ze~i5cG+h6;|E&yBV~~XCEqW)>jV{qkLPQTDqW5m}5v`^T@BjZ>%bGQ2u*^B1z0cYEv)`|`m{&M926AORIes;vDKz9hz&L%Y z8;e0;hk-$EF7(-HFz(6Bd(y` _(7dp5{9IL4?A@9of!Y%x24t4>8mnAl9AU;XXd z@4a;t+79sMe#Lli37=v2u=M`NlP1`;HlP%uWIs6dy?tdg&TLHR*2N6Be5BYi!HU_{`JNf}`C z42*CTyWg6|0W=Y-16&Kyq{mq}A#;$+U&)tdR_^X~W`M;m@@wHW|L!3B;3r^qe%n(E zh3Dr|#1QFbMLn!wsxxbcf8TYkQCqO%#NYykMMGKuB5)uD?&{sxJKVX20M%%5J&#rz z?}i;HLK|MJs;!;))W~SqxAT~hJ!b)sO5S*pk}nJ$Ukxt1{S|-LYej-@cm~3tVOY~n z>@|Y)c~3KdWs&PwxE<{Vy%@1fz)h8VQ*M{Gu$Tla#uannDYS$-7C|lrn~&F? z9YK|g>dwy2aPtfd|cmokt^D!3+gQ#(&k z?;F}(^lc5&>BE>}PTGSGHlV|pse{NiIzwYt%jO&+wXkvvyV`~o5ScP*Ab%c7 zOI1%|f1AqfE#~!H6y|;|eQFJ`eFy2(AW=>_Fc7C!^_9@9YEo$Ba*&cqzr)&jbKrEh z1!U4UH|mm|@ZeV)667C+&}mW&|sqv6*|6H3eoA*FsE- zL-;wK@QwmIOM5Ng*zK+uf(8QAW%z41FxPkeOAk=+3y}`0` z2$Kth0hTZ$KrHyMpY$4@b%o(?n8#cJdJl0a)VwDFqWo9RxYb*fD4BkCJ=fugCB~rh zdf%0Pl|v=>hG5)fa~5`sNw~l5b*Nf!R2-=O!C@z=V~nA(CL!=SzQDmBL= zu0T}zO?iS1EtGVf=stAcW^=FsVZzOf9t}pa-$gFGgt$a1cQ^qQnd$$L4Q{R=S;A{VYc@1!_e1KY__l#>dSEA=(4A6KM(hu7+-Y5fUek$ zDl1G~ubh}!ok$tI6hV=bd><7e$<%%)^#-g;{h-Szw2rJG`mjc!loAwUORPl7J4%$O zO?qw{i443?H`#glrJ~WE5mqpUk6;ZwsGkNT2_spKo)p-D)2?@a-LpFTFc6ERQV}U;q z!H%Qp3yc;GwHI4Eg&(kuKi=0FmBq)etPm#??aAGgE_Np#Q0r8`|DKn%MB}3~%REc% zN)GF6kt2R2$Zl8;*YDbjTS!QvAU^gW?6D*ay?_YNB~0}JD6bvIK6nKwRXwQQ(tG+W z<9)>Z*h@{nm=(W+$YA}><>#-T{&-Z}j!y9@Mecna%emS(cttRveE;pPN~dapCue)E zJN5uHW@$>;HtlY)R?2)`Nolq0a2+ktuK4yFoRfprv}lB&-4M*$pTCDDi@qN_yrknd zu$;}({romU&VDAelCtt^w_jfe*6PgL;uaDg$=*)0Fywzb-JQD)e2>YlMw`3v$VE-* z+dpO8^eLP~GU)g~l2;2k=U?b$UmA=%bP784~D-}Ayx=^sCTmvXDI zC7zBB^eXp)E069YgQe$O`?YrVv%dgXJ~$TBsdViffXu@xnq)GCs-XZa0;o z10*3QAQogPy2&+Ym7a4a5#2WAPf)e!zdg+8Tj0SYWFGu88@1P@Qr{>ND7ztYQ-(=; z5Epe)&%-$}c68Yq75O~nFJIeUKChB=$$9F?q4X`Lcp=03DnDIqa)BCfI)U zlWVX)uqFfIcoMETDmA4deV%K9f`pvlW{>}+PFgTa4mj3z_?WtboLC$oReecC#--M~ zWu6@o*J4P&9#iQ5^+(?G!idr%a7;v~;Ysb&>W=|0)9(Bz8d`ZZTH3N(P%92vTDaCc~i6llw2?en(v)?ZsgJDi<=_y=diQYFQ&NR(? z#1nJ99q;)Ppmy$QfBsTWLDDHw4F=e~twmDnq`0s4i~M#g+)IU%j;AEQRrxr+a~O$* zViDL6;^zvb;_^B3tml^_^ulW0AzPbd%Z?(6hi?XrtK ze}eX^Uo`MmkVm#OH~#V=i>*1#={t_$5*rpgi96UU|GtqebI{!Dbc^z~HQF;fg_#nM z#*Qw*3a}63Exi0)p=DDnN723tzK^%g*t7AHKD}_%G0G~Y{cW_<$q1SCt)Banhv2S2 z(nM(N^Y9Z(?5Sn=!Grm2fOVFR0&@I8A2JzwrTps0d+T;qeZuAYO5B=RZVXL|OtKhC zwflDDvV2EOY9PG1FYgchd>CsOFSwr(`RCaHg+Vz&|KwLf=lT(*ak~wEUBmz4TV?Vu zJS7?S8H$?Ge04FT68qwal1aE=c|G(r}+HL zNwsV6NILzN2cGr3{vxC250U<*+uky}QSn50hh7n^aoQ0xGy|c(NRuG;qYP=xsS^65 zQsgi8qz)I=JJmbDY98FHpDVGs@T>AB8GPEmEX*eEbQjy&t^sK`29vQ#HHGwJbe?C%o+SUF`U zGmi=6@3~Gd0_@*auC*if_ui>(JGFIQxG1R|5gEyz%)L3=9H`U%2w&>qW?vvx;!#dt z%cXwAP(<+M%{g#K9V^>HfEp*USZ3o^+9e;KJGmsycY6Dths(3zyZz;u&X_$PcI&)- z2^;9?nbzMWG!^{T4Yae%)8}k&hO_7PcR?UGv42L?CPg~yF_kXH2FM{L3Isaw0dx7k zN!y3iU7qT`8kQJjR-h1Jztd*EZ-JTbf_FJ7^=E|zf^flEktCp^Iu1%rz9(q*nvLjh z{*kFFV?Q1K8r#?_q=3WN*wG=Cf+;5hsUE7&m!9mz8fRDMy5o2vL*_s>%EsQn@~eL> ztWks^*`Msr`^QUzTwwA@jxWG&hWO`iGOB;tUO zOuICRPGm;y`(z(+=@M2d<`wYr=a}1(bz@M>@rL-9pni2Zej3k=@uFDJZb*5rG7kA9 zn1$s<&nH3)daw#hQEB(j{k|{{aH8}f^1PQBlkzpbUmYS_rQg|;OJv6-D*!!;x1<#^ z%OmQcr6XeL>Bgs-jt+`q+qf8;KF^fFJIxU+~fe zo9;?SZ2yuMCG-5IVgv`I3qLw@| zHJP^e;qNd#&Z1F}4Y{AE47F?oOWsn8l}tL1@Wl5ctJpA9K_dV1BewfnorCm)@nG@s zlsL4qMLU2-OBAU*zSQGnug8_mU8;%{1uv{D1t)s-l1rRSDY^&y39KJr_i8MHl;g@)}xXB`~#QWWm>};f#lp=`NYSEUzCDzd2$h~ zo9;;OfFgLp(9QK;v4hx~mNvh8iOiYVuL9uer@d?=&_u}(mnDIw0P=o^|7Q0&0Ik?S-C=y_})ge6y2fZaK(c!nC$j{Vmo)Rv2)6D?m6fF-2=xZpdHmU$c}R> zQqR8W+;Z;0i)`KIFu!8;73~HCdB=AwRN{RYkSu$^#%JmZ%@0i~2mUK5lQGB98u9TU z<@(6rFG5MFf){w!Qfj5jX4vkplg?B2jIy)vlh)F*@{)nBc^4e~@U*Ag)t`{Fs;?6E z?R8xqKQ}Vhf75uH$vN!>?#D{dB|%3H9$kF*MP&WSMP*-8F0E|aFv~ed^N#;8{>#V^(9NvGB|A7y8y4O6 zl(>XPCFH4^qj)mgN(H!T1IrxfcB*V_qE0jpYb)0gX#K z^m?-;h<%ZE5MuI6E$Jaa zkc-c=Fe}h{a;nILRmOoExZxfd<1CTww-6d@Qfm*zXN&ab0+4Jf9CTI7*KdjNUz@3Q zMp@&~70h1{CF%)-LYJaMGjCdJ*ZAFcR`_JgoM%#hny`j1wR95^jM4a0B}V!Qy*hSL zZua$4>9M+vn0m3V1F~Lm?KquEs3BE?fTTELC5NaI&X*eEZP-K!WE+2h12(elbNco6 z@Rz~6hcaJ@QFv;vAKUL};viHx#QE)Wmg1ZV>;pP zwZaJxx?lk9Ou)1iDyo^H8x+kF%N^<~NGIzad^qZ>#@cnd05pR*u`8E) zLSE5$12U6zi!=2TkLr6B2rn1`4ejf2Q3j^EN@n3W5jJVSGl5z!REkzA#!FwdKG>Db zwKycogGk>Y1#7j3|BGYpyM3NO-SPsHKfWDBiA^^j9n4zg{RNO0(wFM;9D)fEPl@hT zmUag+8Xfx8E}h4IPyei0i#ee67NDi?-nxg7_>G?D0q*YoX;G^|71owSm`C-124Uuh zy@AoJIA=@p zh!gz=Wh!AT2$lf++1hbNQ|KrU{?D)3L7LLFsgu2$I4kmk ze7^I(>CVB`z~y{HM8bL$n9h5GVDJjNSkI6voEWFWUHU=y{T{zq&#v38;TOfPbJ7TEDDyi0Y znKLYUfk70c3PK0{CB-^y*L_wyV9u_lh2B2hSTs`R&--P58OMGa*z$7|bcN$uac*r2 zECh93zbrI8GywO1)APtTWc}qu4|*)c^36#^UsJG|3v8Lq|7t3E(A=`648_!msybdX zWrj@e#c+ct;7CO&9uqi%SbalFp8AV-D{`2GhaS$-Es`Y2e%qp%i#An7ER?mCJ@SDv zt)uvSarY{K0NwWF6c@J%<)WpFw+gylA%0TRn*ntGbcu3Rpa!!- zS8*na#76Gq`5_|>)_0}tVW9}E*FU;DpHm$DDaZ6M%=S9eFcBI*BLCAcyXwhbzU(_u_N2TduCwkaxBsh`TG>!0wTiKQxI%Qa zO4Iqye7_vI?jYdL{UtditTRLG3HXnd-z`PlmBd8xrFohafnmW&A1_Y7%HCK)V&+gH zzGP>s$WW5RQ%sD#l%XY_pAehfv5SgS)TVQkTi{c#uc5J;gED8=(eF7v!qgM2L*L&7vFPt#R)ecrlJ}y+IMQvZ*y8OxWL1>Inv=no!e(K9 zF>p}A+^t+u0lxi*3Pmc{IM;RS%JlbI z$(5u|j*_ACH3y8nRx}IzQfa}Xm1!*_;+sDI5wc^w+so{s9=FzGkI~>2>DJP)LD&2$ zkL%Hqf3DKB$A1q2O@AEdHW%(Uef3s+s<|wczfXm>S9oj^aUnZ=enO&*_d%SF(34uH zEr)?##%^ki_khkjCI@r$C+6u%orbNl0)%L@VS95xOqV0G!>53LX2_?3%Afp?k0`<~ z>2(SOn?f*3Qs^B^%?hFSUL*ap9xZ74tf7KT!IrB}sKF^YXAca}XQQ!f);nv+W4CPC zcelwqoJUp@_v0Ui7fRkE9B1SD!6Y;?yU=U#M}ebdLse~Y{K8qS zZgpeY{X|RXp6$i9w`&F*Qm+)^g@XM&4OwB^U%dF}X*aj%W5Z!hagy{kDN_bhr?t_! zd5>CP*)vN(co{9^zv_Q6Oy?f@;uQe@BmD8H3epmt30K(Y1g+XF)n; zI=uf(e1s{^{K|1tCfbGYheJl+lJK-^9l3j6ZdY)b5xrW9!yOZd)KF0trE|$o92sR4 z4VLP9Gr%eC!cu%Ry#Qs+v=7j78u9WYUb3NO{#mKFw$fskE8zq!%?vl?uN#SZ<3umJ zzu7J0=_+EImjN+Xz`}R3f_Q{~P;WE#-P4h--z79m_Gw2KTu|SoN8sY2hKiEZD*~+62!Vaah()gg zDvE~TolI?)@EvcJiZ6R1LN+ih_)lTAD1``7<{rr3P_FOlYjT_-gy70T8S0RD9pwQE z=y|V0R5=ixE)mjs^cQz2oxsvS56#!2&@%_9quHSq%Tc%W>G0Y5S)8O~vnf-F;HlYD z_KmOoj&1GctPW%+J^24ffxU?d@%^$C*VOaLPgPY^?>h~D`}}P>)U$L>HbFo0#VXwh z>hYx!6A+SUfBRuV1jX7jA}{VztMvGwMBClNssi6~Zw}L1&CG$cjNGcxf%O*FP*;&4 z-_tCsSagkV$H+J9uE#Pa8fRBd`9I`QwrJ}ag<~g9_`9mnp><`=uIK}GTSI$uT4$^G zP2tcQxc|)5^!%xa?{%)f)9Yj6WF5Q<1srD_+LExM?|(+%Z#&LrNrp8(c$sy$^Si@p z_Z~4MYvB=5z?Z@BWpixb$k^UbeDJSS&g6YcO!36R`3Q6fN#6Dosbl$AlvpJBHBR9b z2EHzGYK1-E8cWl?KE(YBL&N*xlQM`CpCQY~7Fh)t<-cBH?576js}hk_nmTEOepX6JOq*FLVkdXY* zVZTyo5juh}^W*DEHP{*vEOtM2PrV+!@-@Gy_yy%z3RC#Z#7ONvD+gaYJozwS?Bg%UadIIM&8#4}VM7k|7l6 zqq)vDbEBhDKDGbee(Ge4z@UBfwlvRLHdo5r3nPxJCg@OJ7_AK}p*wIHw_^z@Tu z_PXS)scutZvWW-3?~dzHxPvB%HYr^1m9K&H1dH^~fxW^t`V7xmQAWHf}{v}*e zT}5A|PDQ%cQXBG%3Y$kEF*tY>zl(JSKhe5%0${6gFQF)I0byR+O4d<5%VRet=2O8) zzbM?KhqJ2p*TuUO^ZCs?IOb5CNY2(cH_kO}KkeE6b*JGc@-|#USJ`UWk+>GMql5}X zpsj=T#yJ-;Z)UT)>ROa>p`!T?0*Va;4ArzT+bFARr%(zH6>wcRr2m}Bo0R>UfpdFD zYeZx7p#s3!X34o-vcqE1sXo!M zoN}{c{tnI=Rh}i7pI&R=zb&=OoQ!vl-HpLWc))@=U@x*7b1!?_6@K&-A-V}7!*ff} znz-EmiAPvY+hh+Z44-fyG+ zjIy@H(RItU;)p_y1P~|^DHf&(S1SE|&cWAJgfOuW*T<#fNP8$K)wFuD%!(s&k6db0 z6lDE__2UE01MmfSBSpP>)3-i&ek8@1<#X@xT=o)UJow!Vhu;MhnN}!{xPo6F_;$3d zG=<5-7VIP<-<+84XbH1~-ePMfr)FdK-`Zoh{yV z5~B>q9g@>28`0`9E15O*H5&U?z?DynMy-)-kOM1;8ncx)zh*{=r`uPKEYnZZ4cU3j zd(GJz8nFf74a=C}cXo)lPU;0w%-As3xc?JHUxO|@Ca*zcJfHt2x6=1`$Qah6D5vZV z)RlhY1g)-^@AI<*#;NqAh8Ri(@rPH7)@u=PrVYT-ZdRV=tu$RLp9iPo-cAroYPuq4 zRQ#qxo4vzeChrB(dsTFsq?cbs*Yo!Iz3*-PYB+-@`B@+5-QlW@rw>t>1Am~Tqx_GO zjw%+j^KO2VkW@Cx;PZOsL(%=OxHJjR@#uZTN8)&MD8#KT8Q3Si(D*__h9i$glI79& z96svafw_|rT|O=}at+xGJ-`8P!0O7?85YWmF{}x_5zH^;lu(fJa7c98Z<)^Wk{_RY zZN$k|{Cg_#Gmj6r zskRRUr~UD*#b6gdU32=~gXVijrs(R++HP-3i?0=###i2(I&_QFC+SD{j|FN);@@nq zS>O$N#0P$u z_#0D(*2+#e@78~#*{HGDmtnXzEZ1>#ERjBP*HoSR;IFd|- zfUN$8rSuOv63r0d-qUAzm%)B!PG}7(_CLa0^-FaAqSxn|Y*Wf62m@V{DgTW~Wgt^^ z={Pkyb{Rd9^pTv%<8keaKe4o-CZ&N}3 zx=$#?8p%0q5qYdW6W7_8t&-x~1v>TpqAU+sZNyp=X~njJ8kw)R)D2gL&pP5#>Lr;$ zPrUHMkzh3@OYG%z;8^MtDdG`(`Be*Be~B<@;vK!4UMBH!vONjDQ0&Bcb*Y)s{it-H ztQ+X@<=jz^^#TdG(!O5t+d_W#2}^r|8M^np^%4>ycKBT-n-F2@3md2!jS5P3we=0( zif^@eWcT2?jlxCT@oLAORttv7g9+`XwTa9Xp?*|3uzDT0rS{rYfrjHIYoXh94Lg$e z0`L2Cm@^We0?ecXQ7XHKlZRA~PthtWZCw1ue7vp&bj zk=@FyS;+93q_A)+*>gEaSGWB8-mAmYnwud*plxn#X5DlR7IK;5dAH&TOBl|uL9(FY z%#IFsSd97Mfa=mJ{>le2aglZd)czb7_`~=Bqr8XSuZEHg^vW$!3isQo1S}Rye-AGe zZG`BWmvOtqJ%|+a`aIFwy3jZBZ>wX(K)m8_i&G!urth}nDw0~KQ7T>BKIIMvXI1;* z6_XM}UdG;TwkLWC?#{E;{o11Il1q@CI4^78)Jf>FLGa0vDql}kuRo8Kjg zqo%jDSX@%l)8a6@4cPQ$i;|Kjg7KF!qFq>0piYof2a2T^ak4R$G=pt4mO*t#uT`3G zUw5?wGm_di4_nz4dhc%hQvV>PxD_TD0nc$3_paIWV#95xS-MLOY@l2!N2)4a;FgM| zGyAE}Mh4tl)ZgNwx1%W_Xutf`oWLy_Bg2|5k0eQuv)TeFKM*e}by6;1c-+$wvTF)| z$5-1vL7P@oe22VT@>XUTE>eJugi<4K8;na!uTv;*E`L^+{NQ(;-r`bIJlw67Fb}Bi z24S%bpJx8KHA!qa13FuE)6be^m&YpyfHihn2wO3Bbz)7ev2gaeiYkwGbQS}CNRP>gXW%;rb1F6 z2}Z+^O?S9_t!awY3r> zN6yigGQ&i_{q^i4mIt&Y7Hp^N(L!lAvlF(Bn;U2NT%oPpv-nn5VQ{bd$a+F}h_38W zq{!LmT7Jwb2hdF&82kT*Z8&k|UuGe`OgwVWf)qgY@VS?ZV0GDd*o%Wwa(p8VU0}lA zWEL3Kj)G$Naf7EuPkz^HkIpPQxH{C$2mLht0c?#va?th6qU?2r9htw9h^%XJxVT(E zb{=ogCkI9zG;LQtAxE~_a1Ox>&DqT{DJEBI6WYMrn<0(2|TG)iJh=LLe z!=fzDz)uEDeVT^j9eswg1R;iQ?ay+lN^#rw6Qqz4W9=sLM;++2gF+{OaxIB2+cwx1 z*zPcL0-B>XyL@Y%71JDZ0_*i(=qOqK7^BFq3p}`L1_SY4tH`30fuFCgn0X&C}Ip$?_+)SHvitP_;XX9l5E8bbn=Zf~^XdC}l~57h-sq8Ne^~ zTq>wYejx;UTE(bT#|#HKp<(o`2k#|ZEnxn)#_*UVFJ5rf!|>C;&d`lL7=blgG7LD# z#vp;=;CxO<&ljt2ADoazrlz@+JS-MnZX9!m2{Y{ysX|BX6YJW!kV`%;b+nj#p1*Y3 z6+winmt91F*l;ywXEgh_eTsz7{%ppXT$%%BDATh35<+&5WsohrPcLY9HgHJQBzqT8 z>~vw*8qDcG!o&wG^6R>x^3PAsm_LrY7EIdsS1xS_}s_#HkKms zx|t~*#?_*b0_N9}ZKd&X>64ao#^vyLGX=SGTwKC;{Q1&=8jSN<%W9_q1I2XqC!gE@ z(d{4PSQ3Qk8NCk}8BUz=LH8Pzu~mxQO{h*A9vOXCuF7>EKV-AjsbZkh&PvT~rXXg{ zUZ=>YcMRgQs}8b8!ciHWB{1}3FdB+I)wFx7M&P$v5yHN|V_wKxjXiGgfYIFvTAqqQ zJ9+GG3Z;)M`sHmmVYX0L-#-u3Vb3~HL+Aax7rzi#=k~KrE$4J0OZUux1K&Z@+O|(9 z$-h3ma?@{2%PLv0zBd)zs|EUJd$8S4G>-H!>PaCe-Li&Bc$;p09MG8bU(2L$FmKm8 z2(gV4q2-=*!5fwGk2VYbwQJkgFNbWzUP9t3xgM2J?46@&)^}kcN+BT#(havU%#Mah zhh1UNRhyg|t{B(UNQ8tXD}AHHh%*XiTZ8^gT|T@CheltmEMfC+#z@|?SkOeuV>tz-OZSp<$)uf*+~rK!No!FL&R=nk&UBT2av(3#xtX)p}- zwZsD6U{mPSajeK=arvTWqxj|6^3kDgGs!d=oo{pr?sspxGw1=KX!vHTJvc|gnArNn zGyBr<%a8RIzt78DIK`sI%-si{3Po(9tp+*la$J||egYRr7O35%T-+%W-5z6)vIlox zh#B)}>P|2P;-1$S9^!7~d;8&nuFoc4MHh+3@RU2touId_7u4$mqh2Qpa=yRh3awBHNesa7hU}t2 zoP<$)GYIRKW!aO9dc61fz5DC#MlJQ*VU0DRL0j~wVdhT%?D^}&k+rwA*azKW3;q>{ zFMySj1qh|dIoTAX>)4vwfU_-l3G2lLY|DoD50?EjB2XHK0rg^W6zPmu7pd**m)i%wq8n8 zz(3v5q+Y1R5oQVb^0d<*ou;;@E9sw;J?+!@KCt;n*?y!Nd-^EM3SO))=7L#0MgsH4 z`LD*-K84eHFXzJi4df3usMkho&G@tC4{!NztP2cDY)nup<3t;!Cb=9{TF1($#*-Y6 z^}>Tw^O*BN^Y9 zm6Xo+a#7l#z&2!Y;Psg0Ww7`4%)@Y7bN3+FV`&-tcv)GKT1dH9lZH`FyWb+X z_^CESu4t>xiq6pH)^cHX}=QoLODhP40N3a^f5~_w$1`Fv0g#H&5_p zua9L9^9OR?w&%LfG%*rwJ_LHGFf%>-VEHjTWHe@FjY)31y)>t6gyx-g+lzoUV%n`1 zFlCXV@(}>CW;m&HBaf9`I`FYBn(n{$P`6@X!gLE3ibFcgW})aWcT@seoxd3WHZjbE zGGWeEB>zh=Pn!?ADRtVbW2A?>w#r7@{`C&64&GKTJyXVHximheEYZ-eYVx-mdd#X%K19pe7Mn1t)0#?y{wQi`)Qmqr8E?X81*+bK{r%*SLxiixOvys&K z6PWbA;0`ML)13+VSzig?jUBf;XiqidArq0BQ8E7hN}zi%a~|;sNMsf} z>Wn4ZyQk9#jphu$z${Q^4GlI<{YhV0xBmqf!+SA*b)0aFTim6;?{PZ|6)hoNstfGg z$T@Xz);W0e9`D)X0oq!>%g@Vr|At%-AhIgrlt5bT_~{MVJx(~{Xj<1e_iIGj9h$yD z8k9}4yC!dDijY2C(i9}V7UD>{W0qM8xjn$t)%k5BlXV2 z0bzT3lz2XUPFdO3rpoy}ZjWw0liG*6SAW|LP5J#FM`VVQXx}xhk;@_zn1L}@R~ga# z@S(~~wjqOTyO7by(1p(8{_b;k)nbmsP1rmp{``bs(0zqTX_g{x|4oMV5F>$?6&!8f7JoFEn21zU#_63P$PU zhmHfxoXg|~EAZeVM1bJGguz?NfG+j%K>g{&#UT{*%V3_z;F;j^g<)A`jfTdb!{I+w zZv#C?0ba^~(-Hm;-zu=-`d!f6z5B$(B>SljExX5r7x5#b2QFov?)$#dU%c|gv8PhR zdkzO*3rPP(7`RO-{1U6VTXwo=|KKAldrju^|6~DNFYvs0!QXA7ZQ!%_l;=1z+v7bT zwWa#c)%`CF-2d`;`2a@<>X&6?Wc;KLu&)nT(2DcZyxgPNJ=1K8{vdCqg3XlQZTZ(P zD{$Qq8nj3Y4!9KYtm;M6C0X#{ZhX>dj}dbysU^R^Z@syZ<`}pM@Ro z5E=7#?KI#+Yn_G0rZx~wlc17x8CL!VU397ZVFlNRz)3;WPT8^D1Jx&4o~-|fD^>8e zhrhYS=)Nd4VXcM+Xt2qgol>G#Uo2kkK1|1kn;6J_c*4ueo8Su>UJh?A55t>j^F=85 zu7~_r0{A~XVGmU>G&CIk6#3&$qFUdkVco|M&vGQ3v$n)OeL!aH8-U|aXKK4ldF=*% zrnHjMJpR*uAFq!0S-|l_-{E@i?EAOMqzSWbmViPhSkhq>*Z*T8R!IugY>mu74szbpFe-lVGbw4g8o-z%oMS0J?jxp@565jEpDV+6^k@329T4Jrj&^8hs!04`sFg z${kQ>SoNtn93VCQEOUPsi0G1xT>9XL=GQbYsmwH&s!F%w6Avu?gN#ZL$NO{Fu3XF0 z!dI4#8r#@0fgF3VGO35Do4UQZv$*YM9(7JL`fkPH)WFcvz8xJOA0J3wPAG7WDv0~H zC?KUtI4anZ5;h({PqB!p+ZJ(nhQJ7#TeQ4Th2Cw*<6$Dvbi4L(a7JW~AH1r9AfEC# zfAn;DSq}GIJVJhj2$e6vW5{TL=mY*!n>Ql_b~)Pm4?}i>?9#qn+r)t+4pN$7wbs-< z=P^5ceb7sOo9U|LW=YTJ+Y9VVDwd;rLr zt)=X5V%@3?%+1F`m2USKU~ed)Uwk-?%ld%6ct$nlp(pU+a$du1WX#GY<9zJBz}*3z zwAQ*rDv~JBucvu0AwFeG@LS9`ANl|KWvhw9_VmeKF4VhQ_plS*REv{rm{XY#2YJ*VyyjV5v8%7dhi-2CjYBr&XW$#9O;9$)Q6MR= ziQq;a#n=Oi{B%2gOTAW$XSEo0OHqsKy0e-xW9CT@G09FNJi30^$`+5o^k0^T4wxNN^ZNLea3H}IZ1GVt7C;5o_6ab`^yz?=@OAE;ZVwlM{n7`ZO0Cu8OB z+~hVEOQNn)?E7t-^|z!t;?`HIbj#nH?2^A#8N;C40mKMr$~cbX7gT?OgT)3k)=tGf zytxEbIag-w?7TS&=vX*e0+ zKeV@^#hS-k*x8v|jh? zS^Sfu|NH9es(kQeH%aEh@p_isd?1|fA>_O>x=UyAz04b2@{|5CSa*ntR64u@jonJn)A4*{1DfOohT^B1z2nagvMd^zvthocZHcYQKx zx7-uu@n-iG)tAbx8b>c`Yi>0`7eFgGo(!l%`%+|XR6BJDW*!MslZ@pgY@p@QxC zyE_!1XB=}pl;pvT&%bK7gE2Hre0GMp+*e%7eX78s^H)$fy^0Mu^zCfdm!_!|Rf3ka zXw#?}T9|=;>AsS9;)S_$q@!PpiI%4Bxe?-Nd~5%vX2?TP#K(PJQQGVM-`DN`8_Ogd z2?F;s7hIq0Fe?qpdZ~d}MnkqDj0YEdv0>p*weQXPeQwgLzUh~;WEk6xZ zIU~||KjrOd;Ga3jJSt^}JpORH`*e;2z=NC{zwyQ4y)uSj-ZL%3dtRB;_Nr$R$ZKX(ty*kLjI_)T#tV?t5toh5q$}p z-hF8IV0cW63Yf*=b_d{Y|A{^M|39I{!&~@dbQ*cY9By-*!jU2X+=f9fF6rwR66ews zU?3|!JVZ#WvC&D)<{SGK<5oJ}56&<1)2rus)@4jTu$T<)KG%Agzd&Wy_6*&%C?tRP z&+vv$ChuJg9{_xsR*ky0!2bux5@tpnJFr4ErFWe#i_KL@_8;1PdVj$xC_Ks2d19-CLO+IFc0jbzU~$f<$JGl%aXey8?y zJ@8;YW!`f*;L+tV=iD@Owm-KXMPrcyNWO>qXe{{~8X9h#k=kn4V{-v3EKDS$x0VrOS(4Uj|X%xp@jjNN!!1Ik>EoBAyein*8=A5R%;8gOI} zQ>1+D;2^R-1;bvuI=p%FE-C4@VU4-ZjGYN1Dx=yNxipdQF{~dGB1FC0A!@JGVB~R( zD#c7N%kt3_NbA=;7ia&3)^0p{D^R^cs%F=n*KhmDZZ6El1op{rRQlED^gDL<$~m{g zkU`7H^^tX8Y%4nqzYHvn9*|sIS>%eD+cRpX|jvQUqnLQgI3(y)wQr zNo)cB{{9N3$YP^&?v0~wdFm=4?9Zq&tVoL8-BA4(mJfis!saIh7Ist*FNiZrSEFzF zWz7V^UR@r*TRbadfoX1J{aJ%t;F**EWbWb(q~U4LT|zwEivb8G^~;aZ-+*dgYBSHq zUW~~;AQ+88WK|RdXxwD@O8k-#|Gm0_S3yjh(b@BSjbr87-Tpx`zc!+P{o;*k`c{x`$1TP+6|Jafz$s4cF#qqug7am&Jfi*hT_ZNK z*E-GKjpAmWVw%FeH(yk41Asti_598qJIXs5d~T<$?e&$RD}qFCi^BQCi|R?jW8a38 za1)WZWW}!#+);I3uIG)6RCuhUbT*TPx+p9V`~9>vpj9lbjPoLYtfHdz6QBZwnNON~ z7i{MBnt{LyRycV1+5(MdLj;XyuGq#*uEMWLG+Cyb!w+ADB2X$?*h1Xp+4B>{>?Z(rBVG60MW$427jn7HYOM%quM4`~wQMrfk$PYjshj)g zXZm>Ywrj|7E_gg{S47pylj{7W0>aq(J_iB~IcI7+9+labL-^}fO6n01zDSq_d0r@Gkk09&{sW($bVnUD&JNY1# z9loA%@7eF*73xBwI3JQ(KD}VT<}eDeuhIBy)Cb_A1jztKh&{w{@2t~_9s&CDw(GMw zXLvA*T&Ut6?xn!f2g`cAmyB0C#xT0Y(6*z+ zaZZYGcHT7n+5?3Uf?jAgcN)J#=1F<>T+H0J&$GS}DfYH#L2J{@EDi#z%XC;@->@IQ#H*59{eG?#M=Sm68=RU;M?BbQC~`uhvC4CT8GAl(^x#q} z<7v=Vam-n^@lF`kye<%4ZntZ{o6G5B-iU0|_i8*}J}}*SZeR?5LsT+6=%M)lf%t2a z+|U;gSKJ+%&Wt9c+Nol^eMQ>4X+T6r?u`H9!I=pdg?ipwdNZ2%&`%2mxs#N++QQMOp%c z7D_^rvwfcTeBXJm^C#E-mFwDTuQk`4bBsC0%Tf^Hexz~NW5)CouNd;k-0iRuQKo|C zQ-UFQGunN?=_NnE{p<&qq@xZ~fW%Ee!|X^WmP>*!KyFqjL1n>AcmJ_C;yWIkl5y{G>0nf|z4s+W)#Xjy4Yyxy4=!Xwz1r?d`)%(A1x}~*rHt?Hb zB-3u9Q+zuYANwu>`YazKfJw(%6II-e?fZFOu3|Rr<_qC=vCkUWEQV=XiLsjnH_Fer zwE+;vFz8&cfcvj#aUD4q3FvUf>&p<)^Q;{WHMJQ1GDi!=5G0}fZ}8?`PuAZ2Hqd0X zo!Dk9qgzHS{9}r0R+OJ#%jNn-_61y#AT>xkd2au|f-n=F#o3~c1sM!%S~Kw(MK~T; zCw{6KAmp8RJ(?2D@PbFK^UkY%vj#z&oy)vx-CB>J#XGL}BnNye{UbXNWWR+#8 zNGe`^q_O`%QcrsSo6(&MFOuX}n;c2V7FDKhYQJI-ermV)@N%I^#st5{l*KkPjc+gX#mCvKw0zq`e3PM;K zEHGdWeUVS~{aQEd^C!!JK%+x^<}-t8K)Qs$?sfW5)8yczY!f^CUYGu{vL=?ZNz0aE z7%P6#cUhJF_-DUL-CEy}@)HyPtimJbDrJt!RnaSN5g($*>y2c{7)w=4S`y2 z>(*}bu*A2)x#^k3Z>mHXzW%2EA6;l+)Q2whnU93(KM9I|lK*8(d(0l8r|ZWl-8 zSMS`pSkWyUm9La%du%xYjxm!XaJwL=H7ylg;xNC0IN~^XX9Ae*&%?KTLon-OB63>l z3frY%beSfi5T~eef1hBOrzv}Fj^6w2kv@glv!Yu~rl|o0S{Md{o!OZ%{tC72?A^ws( z_aN^}1I??KqA#DxOXl2dNW!v+!A3eZJ?^-K-*_GOYo>y0m*61$}=+ z)cl{3)MSMlv`iFEhe;bL&yEakeFHcF{?IerqaW)yASIJ3*)74;?c_f@g=x<%P_I zF4BS~E1qBG76|N1IsOL;zfC;l)rJz4=m8flx&a3{x}OaCD_Qi&Um#(ESH^=pWsTcQ z1AmV|r;W&efdaxB80 zYG{N(onHS!htDfAMhfIYAK-jTf4z)tuo+}nD2DPM%h&v3|9!fMK-|FmRG-|wRq^|3 z5jN{tFEG03&MOu2@TRI={m$}d=13!^Sxq=59tO6kj%EqeO3$Z1Bju8(ZS+bw!o$sb zJWGkz$09zKnVI%rJL3gqwjD~|nS`4gt2eUf6jPmJCGPnA2ae1MkNHE0>a>~bca6$g z0_5;$QydkrY_<-{CFY7YEo0T4iv7sUKeSV)$y{a7A?gS6&+TdSqx{jz$0@rCw;nA7 zcyIgBwN6ZQ+BS!3fEy@~i)}XD>Bk z3`|y0Wrn9;h-?*A99Y(gE#EkQCw*!YFZG9As*Ib7%fc?6_l|JRkrh_1U)yOR(mdq? zr%5+_vy}B7@`4%ZcJ|XqodNuuy1FNQ!mUyqvNRPK&+;^I}Dw{=Ho6C!pGPB}6>!yhLG zqK}<&-RkyR+zwZ^7IRM%_B9gW#-6r&2odYUzQq9*OnaxzRJx$}NK77=g65HJgOat; z$EOA2pOG?@Mtx_tmxt#T+HJNl(~tG7yZ+`HVTZIfj()F54?B49JG|(Kj|vx^ZRHx9 z;tVL|`ReVp39RJDN%4trz1WcIPWGzAxh2*ZX`hx%h-NR6Hviotm6cZT&CTG$@V+ax z^;9x@G3Jr_r)I0YVN%#P7JcLz8?SU+pk{b`a3@iK>M~~}+9Hd|?p+JHuq~JO&kK9_ zE+S#&D_9ys$_{>&=&AHZvh+{%a0NRPwa(|xY`LL6G7)Md)^ccW>-YBOTpc&Ph}rSC zS7xE1>dpCGy6~l>pzWyZa|$oiC6$iJT+IrcYyYKFYQ!xgpB1oYrv7|$<@O};nTGJ8 zKRqU zqffYinmN}^M49Qk2&i_r4K9`rfzYT8=bCs&x8-1KydFJ~NX=#8Z)o4Y|8l^89T2c% zU1^TOnt-As3BzxBiB4g@9*|-uw;SeEh)963zBLeR=8*u+t!4SKqUdVzj=?<+&Hi5+ zs^6~|asYXloJ{4A5Wk)tA`RznRlxKLO;FzQ+uIelbZ=4@#WGl|l*>)th6(al_1joi zbC?M7_Pe^R^$f&|kwA7v4%qkD!IZgs6c+dVPQG296&hEls<=zXq>$i=%Ippy^KBl# zf~P)Dvzpz6NCr)M@i_;1>aGcTL9x>M{( zbr(@<9@wvl#w&R3<#R;-2u!Ri{hkcpzL@HX^}ugCbbme!krZ2&stiym&ubKav!o3z zobvU8NJ2Nl!QHR=Kyy_^^YiI|oP*{YNG_&j2YQs#Yvr91`O)pgKbmObfYbbYAh672 z8iym&4yYw3%wG@DN!H1Smp}W;-$cUdLMqTXr?wF_!)96^VdqbKxzhBQ6Y7H}mu}(B zC`)I|d>asc*5R+}?PGs_@cU#`dVh9GHQW6dGvpH%^r&QFyM|cqq;r-dwpWJp|l1_Y4`~R?44SKSH*<5wrE@ zhutQrx6gHEZ@Dmypzi-pO;4L>Yj)O!HfdJZHSOjwX9fl{Uk0WMT(9ruI(%|$c;*IisM403rg#3P8#Q+){er_1&?77ezoWUn<*)dP>RlY*Ns);f)x29wG5 zx8WFVyTLsdzOL>^TsO?4I3YJ>!1)ws1JrNDGwAzbLKAk}M{bCI=!zTeVF40sJ8yf+ ziSdT_c*S$6+%!M;dpP!jpgUv z_raC;yx@7H4pqR7&vh~WTVZOf9TG<@veAME* zM?zk&;X6MCCV~32o}n62!(<(3jNdZqs{*su%b9EM?i}O8bg;h)upgP@Dkm!TmyoLh zVlAD1dn4Wky)<*^u7yXZQxB_zmy?@OFRnM`cd!kA6h;o6{hF)mGy468vd7VQXh#f)>tMv8ToWDa5b)(K|8@gvCZL&!Xq)_N7@< zISb1;UD}9s&-j$l-?*Zm2aN)!t4*GoiyK#)OvI!Xi}dZUg_O!%;tYTI{YVo$?&(hA zxQp*jr8r|^3+a2$ToE4lIbh?f+D?j|u1l93V=!rM^{cPZBuV@B*2cn+G9P`q^k;a? zGc;6jf7MR$Iybk=XV*(Mv(}4{o-c@O2~X(&_?N?Bpp~Vw=IuNnD}`}A$RsjJf6-|9 z^J9+e?vj?zSyulIGE8jQMFo)+BHeg>B=AE8e&m`{pc>y{#Z%Ta)yy&v==Mk1gg)Q- za_krTkwm=X@8aY$*+>e^s@Fw6IFrv2+48SpPo-vEttu5pY>KvXX*sGGH!g!dyMZ^Q!!O&*GOWXL*IWm8W zI)oq$xhgyUkXS2+x$Y@0{9VVbrHi=wDqg9FLdiF^r|zy`O{n3ht&ubCezmsj>=}vg zUQ|oWBreWtkXD$I6o4zsQ4Y$j3Co>qL62*UbUHoj>8wkXt<>AL*w<+(LPO%pPB9W& zV?_t3#nH}m<$+y{_&stH7{W{@1!03})Wnb-QjcvIWjtd9vNLkEXsyOXlucUm^TZ?> znxN(f8|&!7i26dkgWQ4W;9tQASH(59o;U4WZhf_U;~7yP;aNC80PtA%rMJ=#W)6LO zZa3wYfNmVNo!Q{^rc;D+&ejfKZRz{&dW)~7;eI6DXCF{wzMpV^bl%psJkszE8QoD2 ziu3jPU3({ibyVo`%a@NjEk)L$q?gmI5vZT$0O^y6Qu6pwSo~Us!uN7eup^01sE2{y zD=y`$B&0!*t&P>sV{k-qiukQO>!c)Y~raDO>xe>a38{$^a9o~3j8LFao{$ldAhBWQGRug%n>?{S3 zf`qES^BX;;Po$%M8tWgu%2!)9U!CG-b0#ky)h0)FWXIumpX%Vg?gVOy!1OIo;)lJ1 zp_F77Z`Q?~kB=Cu*T!Q z8TR9>-Hr-tfvOr(tslo+AG4dVo#RvPa;o3D#CTsHsW%rF^T@pRYBq(nG~>R4*KS4p zVNHtGQIU#!>=OJ!@>;gL|IGC!*Whujk(%GJl{fcaWv}Mhy(jV&4gq?Y%)QA%im{J+pcFkcy{jtD)lgTEvT*hal;3tJd!nS^Ydq0BV+q}qRgb_j?efi9mb^D zR1x-v&DWfG73}*rVhHm&&E@6QdXkjNh;#KvxOWoJUAl6@<3K05-%_Ju5Wpm0{$&r} z;2X>I?S8-LBHP3?C>(Ne$X)bmI4c7BQDky|b5OHNHhNoA`|f|k%!k&f0?KOT({`9x zDSchtoNG@YH~8C!uc7l#sGZBhIme%T9oB6gDzgoXtN)!lIq}Gxerw`2II&g-=gyHf z-tM(pGjps}cA;fT%f`MnH-*cAh(5$zwSz=Q4;mNU54wjw98T*C{;5X%&e}kOg8A4;{wPur|80TV5&75l;ye58eGx76$g^y4SHDI@ZkC(x zb)JqurP3B-+O;x0YwrmDt51MtC2IJBUgPvLsv=PRw{`+qO=sB)MGTvXGb1pD*-3&I zglCC>ULzmY%brsvvV?W--U*8Jh(TO@`8GQg7}bb*c^1-MvS^Njs_6sxS8u9b+4Sc} z#z83txx2qBW=x^E4n4fjnN4NWY${)8SYqbY_Pcu+$`Wp;mM-{OxW~7HfdY^L>s&_j z&&K7uNabC`h~n8Bmu_bVKLxLL6N^-!+j9jt)HG{2%whEDM-|hbraQ`bDf}-wb+Tlz zz-JUFIYD%7TXbE0$crlc!94mG!X1oXu#wz8DjQx|uiM-&6asXnZtLTZZWN8Z=5nP= z4m5(czPWjes||gUP1+L02$g&8Joa|1{VKkof_Zez&K|PWi(3<(-==1-xlw&h>czt4 z7K?MX&XH1Wwcxd*rnw&NK~odr!C8LoMW(e1)SS#{4lV;|-elYe#U4lalN z;ODeWT;Rc?)5FN0b;p_6PGo-n5QCC2V&`ubgDPDuFEU_9K;;zVMPgg+ z67(Pgav}Ni*w)wgc=%$ME?jZgKJ>16R-_IqXClh&`2K1U(TwL{7s!+Tc>OwT>ar)( z?K{i|Q{@Y}bcp2@gXmSR-wth&l0({a{^RPVc%|TYFfAl+7fm#AlT`#i82`4+(WJ6A zGT|hq^nBsRQM6?bBVMb}2jLomoR1PbV=NMQ6xDGAqXJUo<>SgMh~%mTz%{CYK1eL2 zY};Z!R>A$oU6)OEP1Y`ZYUnbfRf8pL3+mmM#c|tZu)j4&kk0JnFx_MKn|_@3wZ?7k zp7oz+*V*+~Mg)Hn#vUY3Z+jPZiwUb-g083UTNMhwdBS?h%Aa`4e$L+qy z3ja=A;$Chl*LGs2pF1oH??qdv48beaad8|6F-gOD(aT(i0x!8 zOhGc$k2&3sg(u=E8^OAb**8@yQXdpCG{T2ZCJF)Cep!iaW-+(;J6-TSAi*uOTE+-N+#sd)Q# zpn6%tYr|aAgRI}pH%AZl#ydlwI(lW69kY%|g=OBq%#XbKid=moDB>*Kr27F@rYgmA zwAU43cyFz2QlXAbK4p??{*0{{in5}b6S>L+hTlS-?%k6)O>LsKv^nrUzj1hp{yyaX z@}Bt#t+G`L+p?$ASn2iAZpxOp#MZ5X=BFbsH6)#nq48~(Vf`hbGyFmmqWfc`YJL?{aDl~!7;g^b*G$y8_y@e4u!0|kR4m8 zY4LDT@c4!~zjF`Q1E7QjK@49#{-~~UVhNH7D@o{clv<=97Pg=_+mp4ZjWkqT=vR82 z5370qm!^GSKQKahMyL3$QqhseN92kM9)2W$x-shi9)5k~!xxQpJ$SiV*pn9Dg~#3w z*zaCBP`~y?)tcBs?{e%vU#&y#akDeE*U^)oUC-~44w8#~M#tXSYK3O0{rKe79=z_9 zS(ojFak=|=MW*JI{<&l>h_7UhO~a2*xv&{W?E%uxR05%K>d=8A!Jas=00*zjl2RSF zf^QJOTZFlCUkU$Lw_M-I9!;$7s*+mrSGshulm#J~({thz=V+33SB*)ZULoH^3!1NtI-7GM0*BE*T(d8gnm9wI@azc8 z(s2BT!tyV!_c7(if+3Y1Ry~ZN=fdTdVj#YArJZi2qCdo8`aMTU05)!%v@5zBBhO`E zsh_4s-Pu^%=0*{ky^aKYIuJYVpuo2PVuo(F5zBG*vzZi^4+s4t;Wa~ef%z7|=Q%hM^Q7u?&i06v1CfO#ETMDqhM3mVQR2b}zs2xJqTm=Q5cQC; zGeOPde%1;#FqJVw3QMHeS{0wnwe~90)%{Ov?B;B@+`a}NsN{Q zT0yu()z8>I3)j}AWj%u;@w>(BW79*124-a$*Kv3n|DHebIvf`p^9twIMYMF0<~|6# z&z}~FNJHFBopRk!uAg{+k>rAznGP(mU(W;ibH5-#naK!yck7x&t-ZXxpzForJP(Hu zQX8Ak5UlI0h@EY*{%T1bIg~BCXK0>=&DI^clZ{ZHPd|ZIuh^uFed<{MJk7$7EVC&J zkeSiCHDXA%Kb5h*-FFGt@C zyWc}7Sy%hKw7=>7zzc{stKs1xlMj7$# zGcNS?%jy$ns0!i5Y0ol$w{2M`joq5cff4{U;r4@*d(y-9?$V*XC)C5@q2Iz9$?c4> zjzRuHb&+(nP{pcPYz8>0P>9RT^o%EhMH~vOtNA9xDcq6l+JxdmYq2{qb&KUK20Yr7 zt-B}CYTT@v?K8_bO#AnVUqR|{8yv|AZvMje9mZb*)&f{AMVaat zlxWd`%P`CU|A%L?1zs=`Q3l7mXzXruZa3x41hbas)I^W%Atg*PKu;!9;cItTaH2v6 zB*-Cm&3tZ0eaA+mSDf#*&cc_3h2;nN_9;*pMC#&y!-w(1SlQ=g;oqCcBvKQjU%|28 z2zHHj}%1 zlh7ePo?%`}*WZ3-_G+pUf>IgKHE-H3Ys9*U%o@`1DXX?gE9u#_TMkR$M9^a2s$9K- zEln=nhJ!Zu1_iuwW>P6r&-?f%a~5%VB|r8ze%eyn>(XIP_D_!-4Ele)+I@CfVkNn$yd$-yxjQkP zkbbk5@*%fcm2tA3%x~^mB)gRbpKuQovJ@Tw2xIK^Sm)f`U$Xv3De@(G`I~*W809BD z_E@`U4H<9dEU(74pIyJ!Z#|M*t`_sg@JWEj7mL+DPWLV&8D!5bH9f$`#OE%3@Zb5R z?syqv-}?r@nJ^Q!EwvM6>E+e2+`YbI7tFcrC+mw*N5WjUg{2a0*VGbR=%7s zi-L{38mO*^Q|`?iYPHR$A($giZC9gi^Kl*V(T?&3$H*V7-dlvWV$7z4b;Vwr^FbTSNn{~?NuH%dPSz=?UAAQX- z4@xqmnhYMK zRM~FR^6MZste~QeLtfQ;`6E6_Ty&BpcVt(|4*>X&K-%#KayLBgNrA_;mUP^kBd~W# z(8{``J9!b~go3xwLPBUG11&u0zgL7IR|vqQm9TXHkO3S&O-<+xMcqclZ_=QebDGM4Hp{=yBi1 z;dgiD+E4P-Gd*vUxLF>*On&Zk%R1HC^{E_=#C(0|SiTjlbA3CMa^alA5TqSxCBM^^ z@&G-=z*^Y#3e%*jQoXV1H3XTz&Wc;|x#oF|K(hyzS;BTJ)3G<2;qo(d=f0fOrv^ z6G-03?a*klxXt?ksGoc=J3zPaSI+z9EG6*;i`N1ZxyHtgR7{FyjIk4xQbubg>kaWM z{QXcdcf|p{uP4;Y8VjK~CmWn&smeW^yxf3odC9MM1=e3PE!~8`@TIJ4;|~oL-`AL8 z7C7)b}h83)CvEsCMDOatKe^ zRXRjPa8H7^3sGM<6RnGEU!;jLIXt9=zX1I1E@559N=j%g54&o%MyKS@#9qqE1DR~Cs**a+B?Y?n|yV;z&m!6Y5;i}JlDhF)q|+# zrAUJ-tNUBL8aTLrnK%8S?&NFu`)l2obQRFW{D9L2zFv8$|7?DkRE5l5qhtpT>#eTi ziZi2`K#o;Y_t-Td8}UP`2SjYd6U#S(*0bb;_Os2$K8yU^!Dh>j-;`FLedlgBM%xvx zCh9%;7KH1UxSueSShUtrWeUnKrD}sF$Hi3aB)5!e_xi z23>mXnY9Rr;HfouDz{ zy^#Aam?54u%PWtPc6V-&!u+u9LNc;Njt4I$?EZ%tD&hg+2!6*438;A9&aYaZHG)K9 zYWkExHX8X-IRQGTbHx<}fl@)(`uC%ux6gjPQa;t6`)t=d${2LTlR?(GH`hoxYuHa} z@21;A3>pERmE*naJB{e<2j+`eWnxy|BPJ@1m$IOxoUaxrS=Yy{!d!BQNGIb!i!$E9 zz?XhKpRq!iD>jGH|Jo_!OPLTkH3jT0#K#5>ROR){q(7@KrK_r)v}7i8y`JEuS(I*p z)VM6o9bbS(#GynZ&2A7RW5;f>o$oAp%*vv03b<;AubeS{J8&h3{)2{~0q#z*6yHD$ zV(d{|8Y9J!IJ{WI=Jz!sUL@m8a$m10-qKRn{QUs0N?QLoIbnitlKtUBqkbLD38+B^ zo0DeSFP7ZbTC)c~B}LS2_6tmWxp+cgD+OUoXQRFoq}lfV8qCR@XVp5JGrJB*fm2t1 zP;RLG6J?xcS8GSRWSzFvy#D1Z+j16Cn3d0To{O{wn(!9V<%<#IFR9Igt+^{LVbm>Z zbFjCqoKeFT1&VmsfMzEG+f67aicQUek==b3etgVdJID!xOUDil8{@b7slYoq7H6`7 zYVlm<$0H@IyR;xp>G-1@UKV9rB5DhkFLDPq5m~z)m4Bl zP+`A|EqQs@*8yGU-e?)Z0%{K|*6bnK3#>ynRF+alka)a*6z zQ6u;q{EIv@2iZo-u-6~uKZ5!6pj%tWHCqFp@MDbWKR_qwY4I^A)>>SvM2q?g-?3+m~ckks1g+)AIe#s5er=+0|vq`C7KmTUt0bJ@EV! z0Q7k2#It?zb1SZB!l}`4`j29}{q4(;SCqJWCjiv{4-e#uLfC^eUKqcO{8FnGp(!Ur z2?%?E+{LSY{O3bq`KC)A`HLBUKq;{Go?g)>~HoJLv9^=gjr%QppO~kH9u?G z2wLrix(LCr9d_OaXvwAw$P-GzPNB3IjOW&N+VD!l3-?Jwy|CtBaH7-4@MT^nSLpuo z#h{*kX@TiID!GR%W53x&g<(L2VYgy$S+B0_uJ~z?^!j!8SMm0yfoV8r0A%agK=Uad zFv_!F78JE{Xc*fms0tqoyV|hdvj2$xryZ5U9p|1TV0Wc@*WrcdVAJ48_PdKlvGEEK zi)BkqT5g9aRVt#}(-NurH~i%?#ya254C}`$q@iz6T%tzoV@=SFGC2>lZMb#O{pD%H)utpdNrS46 zH)fIs3Yvb8a~{Z6^a}rnSluf+MtpG~ovHa1Uh{dh589~t=2Dp-Lr=%|%j)vc+^TSo z@2@O=?IGut&#&wCh4JhGf@Tb9#LTCP?_H8_CZ`08(jMT`8Ix#^;#d!>{jqLsk2D4B zLYADBQMNiWjWo=yef%w^swoXpnNnEbFvD`Q()I^V+@ zasRZNw$UaL0t-|A5nd>o4 zM`bg318qda9qD@ArLdu1`_Y(RKPf5fc0z)MoJ zuK90HuFO)!J0F7oUoC*hmfpaGCw9TSXZqpoLu{XjxV9iSbt z6J*hf+SpDcYNqKZy5djPBqC!pvB58YkNu=~E@w*X^IHxJowC6=&eOiMVV$E7>Hm0t zP#+{0?}f~^OQ;W7HTivU2d7XTv;u_{0>9nDkl6c%rr&`K|`(jW!2eH`WxH{wwG8%EtaB|S_ zezU$(opOW^x$izoVHmA3Mn#TM{Fg$PT`1nVu^HE9<&W4X`2n3`pmtYgfSz8{PVgeZBvlU5b|-CXX8=uVF< zw4`1=)-2LqBz{bn$*P_)Y1|r3`4^vSOpkvpG_P$Z(n0^JOat5X1*!kh+*<4x=s6BF zT$Sk8I+cdEhLRKfN`8D~NLy2Bm(eOgcLgoVbQSI>iciYt6Ut_A2aC*F}1r=D>K9R5IWhe5#9r6o(^I}YI|v%}1C7Yn7eVN)CG8z}Z8>*nH`fY5_>>8h(a9tG(Kz3g45 z%*A%A9|sRenwp>iVp#q7MI0tH6~WdH-IP~fERsMGC>puA-^Snt19{=Fz9ZC@mPhme zbc26AV(;`;O+Wo};_{ix1zC%QyO^;#rohdCH9U33IU&Ap_^qF8&bEBh_%7aNAE;4< zT9QRuDz6;)HxI^5%Gu7q#(wSGrl!qhQ^UJ869SdUMc&Im1~!tjMfuaKg8R}BgWjC` z@nkxD;SBUh8^d5`+l{fgzqnhsPW)cxa+AE|vce<%)pcjZA)ohcJ&uWA2_r^m98jGC zX`?$r=ATM0-LxzP=VmXoxyiq!G1Y`;%31w3Hr>UUEqIr8=+)8?@8Jl4<*63q+YC27 zPN+8~!Oo%m0$v&HqJ~;M6xL*Xw}f`JMIz1tlfn`1c-^aE)Jz<_bo3!P1i1yq=~YNx z^?8WytU2|qF|to`!pYIqHz6I&xFCIbF2)tkmZ3bT>H@DaP8v#$5pUGa?6N1L-uop# zs@-Qh_P)0ytleg*Q0wrpS~fM&ua<6kvAiO`#XJ&v)j3l_|l+?yzH{IOndL2~95t+LAg8_=g$BPWiI5?xe;yjmM=UocT1{4bQJl1z9Dmz;(=3O&dK%S#X_^3S05h+sIj)`HfmLqO*OK_%z|`F9pU`bl=Q+ z7Do`8evx|CHgX(C=r?!7i}St+HY911zwOmt8C`!`WrFw_^5>mV`*Sz?`Y{D@>%G97 zoB4js#n9>`@&$#X01Cg(8};+wb5d+gI}4p$V80>Rrw{D7^p79Zn|WZ=sTZ1mh86fh zryrZnREMm7zl_|>Lu3nbq*qkY(|+2na`wi@r-UDbRZMn^2)HCrdW^YB(8t=?UjEDI z$C2U!JH$8Apd&0r&WD@!$l;$&r|rISa7Ue&neC2nPi31cLmv5>91w(NsUc5e0s-v}f0-44p$mTq5ce`H?LNJ9w) z6103bPJ4@W`U_qBC7OrIhtFCBZS!cfeuqp$X0WVVS_hTw*Xn^D zv!1u~rjsCG760bSm+m#)+K#Q6CQ9Ip#0$rT+_1j!Uyb6T3yhgN>;qNkL)q~4LsIX{ zk0y_X^-u(~EBkj4%uLeD^aEr1J`vAyB{h&0v;Yi-e#73Pp5>k$VRo86cUihF3E;$V zN0rawQ=V+Ii&_yrO!VO_!bXe~$3z;(z=)5e)B(^yKqcu|I25$91*vpWaUc@h%wW?z zV#+UZkb;W+dc$;OWH>{t?xe|b*l5sAUvt2bXKqb-T@EoXojIicFULT@(1$x0BI_R( zovXaZEj;GG!%i(!L`}cg|Fyu$+7YvZWgO)E`P|wPO_w?l*kyj$Rng#a%@CQTZqFow zyOS%`sbzQrPB^s)@AXBte~3k(=5vc;^SLStMD)x-1 zHRjZ!%Plli-JWdyw&!b7Q{}hpLi}FLa?wyF?YKlE#VN5?_O!7Y@y+3yt8s1vd9 zu_&F1DKMJ6A+3(FuY~qn*Rv+G5jo2170ledv2N6)5D7z?%-tkV4UXVVwqT?iAu|u3U)#R`-bAT>c`ox~PY5S7}&R z%x^+qcJ!7zgi?`X-nbY91cF-qgTCG_bX^Vzixv7+v-oJlas``li6f+*ZG6~*OTS5$ zqc)UFKS7Wq{7B>{S8tTrJq}&e?#oCo=@f5VdvYyy31+rQWy1nLEee)QARC5s>@+ENj@$_-?%%;2^dijSEx&1j`I)%W@L9*UAl|JW*nWYYgV{DF?oho1)ri{+Ex z`BE@TQ@{pkLbexYs_oKg=-=&EK#>Sufd|2%dF7LbT-9`MJixxuS~@pu%Bh0cvy)#8 zU0yxDUB$HxnfV@^U;njVJP6d*(k+FU>f8DfV<(%URP7W7BnUx0GGcMD?)}O*V{??y;+)94YZv#hCHf$nWq&&YGi28< z%(oC+4|^8R#bD_@Ogr)1vM3}&z;#g9>jNgyyEe$bleByrE-^W#b@=WJUb@I^GP|Z{ z7Jhu4Qg3w|&O|GILkqI0y;F%(=sH8*gXiq!$`}9Z6#w5ZtA8r$)C^bnqW^<4(SImP zwjAFS>osgjH@|Oc3MAQCmFYZL7_3P#f8ECVV?3{|+Q`yhQ#`j07*hSv-TG+lyOGjq zs4;3mdKh>1*{jvLb_p9R;Q+Z&3H_;bt~c}V2WOh@RBa9z2#jY3v%Zp48<)OvVn{+lX3R5G<;k*tVK7on$^V`U=&_3GwqQLvWw$O<~dk@O6N*z_* zy~M$PXh3&;l8 zy@#^ZFh5+f0Y>$seYZOb$Qh~Fl*|RzN#S)Dnc9I}DLKurzNp}OJV9uWZ}{Yo-TmyG z=Fj@@jOcyr{#D2adIUEuLQA47fU!aNrDIZ2Ru6ng(ZH|hVqQ~y5kC|M}|t2>+M*CuiETbTHO zvcF+l!l4smq!A_pu*7>VHz}k#;X_o6R+~E87^saB%tZp7Ye71Ug} zeQ+i(Rp+YA{nX@$9O*HM`G|n9&4-LvJl@jRfB3hf+QI9=iJmtkmb-_a;P;<`7KbN( zNzLxgnQ$^s|4D9(kRdO_57;(O^mz7Ymw&UVcS8*__YC(0raGiKRFnVj9tNI#JOPIc9Cne5vmpMxq z?8g@(r4)h11RDqMRdSGZaX(*Rlyz5a>)BH8L4x`}`OG=iojAO7PUR66?F52O+TJ_H zx2MqRt<3qwoXYWOI92$elnuzawYUDxXZ?_sX;fS(jNf9G1cnmj?&)iA`{FP`m{_qR z)_8O%8KR%8c06kLj{fo~3&GdCq)|{yQoh4P8(aC%6;w}aZW-b&!c(!0!czwVN}Mmz>m_=Ydy0~(W^R9dnX6%DD*K`cz*=zGi@Ii0yYr%4doivt=UA~6H$2B68_K=YMAoUKqIR_I!X6@QUIfB=nPvj( znf=J2#g!f@@zrMpKZq9Nr0D+88 zH8|(s*yzY4Nq}7NKO6G@x8xF5=Xh9iH!gE5sSV>emY!B!iu`coWO26CO_gn)SyYn_ zXWsM+HcH0@nQ4yHUWWqM6n3;?=MslM4Kg|6HY%lGG4bK*L++N*V^OjyHrv%wt%+^) z)0GpCH5&ku`pSjL-^4d|f=MPJ1zVnD_d0}PqjGRcs zhgJWMc4my)jlpa9<(lg*(^j#Tf|@o8+E?5b};x|B>S%HA~(@!_OA)o^4;72r~3HcG4nscbW`Q+ zakDPQ@})=>?l7Hy&lcyW;p@q^t$H6%<}sdza3~L>{-5@~J1UB7>lY9NML{GfSriq? zq9D*D5l|2iQIXgRD3V2zq;3$E45H+$WXU;_qR`2O)3)Ox@Y6_7`9Gj+MUWo)e7#E zjQU_pA=&GRC;K^}M`oH?_Qc+BAHdpIri&$WX8=cs69w$6{H0#Ew=u49U*u+%3HYx~ z-{*$7G%x)_ELIj3dUbk}cM_?Xo}c;p7Ov&z$-YF~%A5%iNTyv#pQMb_S_^)5CHlSDzH3`i}MM zdVw1*?AXWAO`qcyh)$wg$?(~eCd?9*;%%LW+1HOU0G*JBYnXn*(r=hf6pq>M7ptN@ z!SCWbJSipo>w%A&RmVI#<25!$YtB;`d{nqN98<&1NdEQ^SqoN;;`&(%^r0QhnZsw^ zk(boGIWpY^H1L$0tg+=(J4j>N#rh?wJkfqJn`y^xJ1*THds%L<<~YTr+;ykyOM{?C z?+!a+3l17hUV!x-*ffIZD8SSu=_g(I3sB{@^s6pbA0O0F*W8`i z?(8TsJ_<+ID6HC%MQcoR;`#1eGFTU_(fV>-m*m!o?mlsa(sLAdT(2sWcIDeHZ)6>( zl+IPt>FDc7Ql+O@*YR-uD92o7^-&{tt$GQ+@ChSqr=5?TsG1VbAFJ-B-+VpbMh5FN zn8W6ljn?RSo{G-L$|`zNtqtrM_gN;zo})g0i`xLfTFrd`=v*Ki+IL$KKUWxAL)?&= zn|ytoXBzh2Y()4co7){)Ka9ter)MOVozs$QO6PK7@+LxR?t^wH4%2;n1bcyZN8vBX zEGGLLBq|T{QqUX;P>}YE^~fR4-*=Rk3&FhWJFymegS?6B9K6ZQYYsG_Yobu@#%8r?8JE0zSw>h2M9x@-DLs?{bv_<(K?qU#hs^98TWK)dVOv7s_q`1#FDHX zfkT!K(_l(JgQ3J^GY=2dB%b2!Ropv~8QOQEKr4?G_!45K>~l2=;RfEE(b&9;j($r| zPC?k+;TWd<_dLoEO*VX$DxK3MCwCYkW8krzD*+rCR(|W%(akc9{B-@%;%^;KvJVeu zvO`N(&JDv(dKT0~2x;=1^w5}Oy>^748h!#7+BP$bzN3gI*t%&BO6#YadCy6&U-o!> zo#p%)D)cyJXfxQALh)3L0w&^4vJ-O+eferdjwpDHb{#9~xeIw%Cm+IRFKL~>wlLfK z`O?9H**gxVH^<;qy0U{CPu+JRsEYu-H(q)b80yFlRzbi<^Hwnw=)*<6j}8wmvn0Xe z`Y+0!J2>M6{7_&S6+||*D@553dQa+@=Pbex?aj&{Zm!$Ep$^s z2=A0~B(WavQ_f%Z5|Ftg7T%8C68I~W%{r(8Nm0~h;6!Lw&!vO7S$sv^LOu$=Q*&Df z`OJ_D)tt4Eq3&jUYR>4sK&|uWb*m=V7K@F#LH|!~9a_TbT|-6anw(~PNc`EPxcDeY z$K3L{C<}R0|K-N!o4)B}~SO?k+vPagV-p#83c*st*j_{Cp_#gn-o*>2|4lso4NZtDx} z+YAs29phZF*ptoRUfz1U`4;o8Z3cIxW3-D9X)YVOYw#LapwO(tg3-^^*>4Lk=%H*q zbUUgz#QC}D!A)w~%yEpw{rqNU0QN>fFjzJ3lKK9n@}F)XDMiSGp`h$?ATkvC#NA46 z6*s9G!fXLen``Q{=79+B_jUZ)m6F1P)^L8M2O=ZNzISqz- z{4;c=!oHS$o{qwYufSyY+3YZzphnp{mgya}z_sTElvm0Qs=q}tF;4GXNdH{(>Dcrz z+Y#lmFuwZr2)+iK_u;43NAJ9WM;!<5r!^FL(*WaZp>W_U+;paH4D0>@TRbp|f!&GA z+eiM8?ffC7f4nN9q}efV5#{~|4|{Xa9$V{L_ie~K^;b0bjV^Ea6zb*GU+?}Inf~+N z=I(oWK||y@oR9vKX$S(_sCTjbZ#Hp%cX@!*5pWK;Zv|HW`ICRPjb?{{^}T&4_csi@ z|0*dKKo88)`x+_zl#PDe;86$G7kOlJAAR}L!1?=)uY~}cb3m`Gu>F&1=mG01HjDma z)c9v>1HNVpp`zJ=?=(Cd`kNW-GssuKY$owFc>KAh|054Q69l%pnVYV9zrOM}yrB>Y ztWWUd!gqN4UqAfD6xeDh!qef)|704!Soz1s|6=7IEc%O;e@NLIzFPxGKC6e&O~9ay zPhAlFYb|=>GX(%UI{DeXD~}|valT$Ao3qWKC{J;E(3(O8cudAz7PZC)OMD8-P4%L? zACmj_njnW4S@RIt+H=brB4>=8cfv-6y;moR=4J5pf&k5u?!K!vA1B>yM6bC(%V zaZ_EdW&e>1*8x;GF!_I^JM#e*hl5D00J+T{BKfnc7(j)KTipM#M?b$(fRI!;rubpk zKbi&;u)Z4&!1+KwWZ`F58v=j|hb0|#|3@mEnu?^$_rD+9@L!3p z-^5-toGSfh%*Ak*Bu|pvlb|JUT4)aCp28Xf-EqC2!pdAa-6l`U*p9o&y~{ZDM(BHl z*1G+$rJnLi-NA+C;$o^RWkojkFFplKmN$FRs~HLogC?VBnJW|=V4vBq_Xe4#hxLbE zO8K0tU%&i3(mE?2uz6s5u2Ww1-3n|tjUux`^ z6ZOLn{N+Ubi!HnWC+cHWZ4PQx$hhXYXHqNa@y6s)I5MCBLgMs-}VB=^t9CH zfw6)@Vj_QiVZ@Dw7mCNGci1SeNod~z?w_X>r@jG1cW-z`+NA;0xco-+06FDrKd(9% z05Jm`?p@|&^ z|Ml>zYK z6nEtFNREd0PW_&B>5bGPMhZD_LVdAC9EF^L%|_-*s#llpIdU803)97S)lxo?1GkSV zlmyGy9Lk@Mq!snMSprT<@;m%*{{ z!S81QP}2M|?myY?eS!GpA^lXB|MHN2N*YR<{|g>cx9KzCv{K}qAWoUGgv)mrf9}FM zntRuIVx8d&Rx*I2|CrX>uLJ1Ic$vGc`@K@&l)HpT%qOrL1|jZaFdC##S-)dH`?l*| zWoT^Yp&bvW&aT0Ez>>^MSW1gbZw|OBxj!+gH_vurRcAj9j8CY#$K0zox6)@+XFCUs zuc*6Q@&iFiZv&W;#*y252cv$96xJ;rzK#G-IHWuxiP2w3n8N?gi1OH)NSe0d&pmfUn{9;N6JJ4=E$ktNk&qa@< z(7kPWy36gRY`fJo&M0`*uLvD0+)YnUU-03Oc6_q#-nGCdz7q(YGe6e(*yeB}^`LpS z25&^dF(9u+nW!;F%3HlteRPfrNP8W6vUwXwdjUznqrh!TvJXZjK z>cq{{K-%l5KNVnM7=Kxqf0ncSCD*^?`d|LV|I_UGFHZbl!HM2n?;QAZZ8A(&h6|B7 z?_+$9o@B zwo-iwWsmYGYxV4R)U1bRJ3z{WS=UA`lxz37k|*VNvX3e)1My03_5INk!1&9R(Y6v` z+!uSl_X=sOH>#r!SeHmH0mH01bI zQTz+C?BDv8FZ+oZB(YBTVRUr7x`mAlvuSgPjrq}mzepQR=q1}t)lF3Hj%os7$jDV5 zlT8f`jgIDvj*Hzhn8XhC$J0@hweDT@_asQEsPCNlvkPefo;0XAm5tWlJy@jx6v1h` zYx(~AeKf}wbATfFuQ&dT;8hCA=H`tznD#}h9#4TwnAt&t%AVs`aMS$(a$oq!;-S#dh0FVT!wNx3ipNG8r zJHJQ(APEyh4rz+Nc`*6_Pzi?DzgvCt7kCkp0Zzu!((Jep9TsVRVvc`b>;pQcMJZ~o zVx`0;iqongHyeEEwM>nlpVElEWWfT%uHc=4HU}g%{TX(%%$FWF~0dNuHPog!sDd zrm1(RirMWFM0(s{?0f_;_XtKmb1;6hIRBJYe!@Kio6^=&EuY_)diNNBv-qqR9X$4Q zvqLf?syRf%0J0!%FK1te5yeqcx zcsr>2W~YqZs7atuuc1T7JS62Ed^*NF@rITx-qCFiOlUvtF3M9qiSr6w~&Z)^1C;pYRV<~00fRIm2F2?U0%&b zEBZE}+#x%+e0A~h;O}6Ub}D9wne{*u*@VGXJ>g(9Dl>YE z=8`4}zLW*1;^49OvLu{W{gFf8J$cu$_e#Dju&#lcMiELj!DALU$*$8Z;d-Z!4L;1< zw*uZ$iaaXYwA)?d;+&lPpReRFQ%orG^zJl&CFn_ggk6q$PJE(Tl2Q60Avdd}Hbb?F?O#{6*7wSI7qo-r4u*NpDtA2jm!V_AQg^qd~0iy`Tpa z$42hnj5)UO{gLPiDwUuU+NtZ3MbL#LglGsq<7dNDx179tQj)qXaJre-#8WtX5(fjE z%hG!ZMHl-nxPH+NFD-U{DKdw$j~rXF1F$dl%+uQ{zYxL*-j6&AJZ&%E@ca=r@t&@pb?uCR%Pznz)lE~{V5Qk9z{ zwB!7!wuEb<^I&*7sqbXyJ< z67rjZ5FI-dbW;4RP?Rd>e6h-dyvhZ&-^l$RHtUuw34fKqD8&1_mVTo208k11=|$R) zgB9-kR{m>Ezdt}C?G_~fZE?{o@%vpN{l4(u9%{FzFf~qkGo$*O*_9UNnVzy2usZ%T zZ(97ubS!mq>Q0PK+Zdz#&91r)O*djg=)xU}=6#feI8V33!7{|!S6HQLO95NDW{gyrnpZE;AoXlu7uT^<-C&5hXqP1QPBeUNe}wcg;=XH#P-}f zt#qIfLwbVCWMv0_;KCHpx4^NP(kYv7!hEnImEF@;D`j(gTlJ>B@l0y9QG_?1|xio?NnrCbiuJ8m& zPW!nq#9~TlVtOY91@h3%P8UyEw~8K%t{bwz-&9?lR;T6~UK)eUFeY7(B5vhd1YaI} zQd*bID)&}~pJVNnNUC;d~%qMPb`{@VJKn5|azkznsSjn{<$#Xc182f&-Q@OIQw(q* zVP4Qc-)dI)?1w-R+hS8i8*Wb5U_2C4J*6SDQlPZn7iPm&*$D$05tVewVVO8j0s6ii zJ%o$VAqkl*1-B`!)O>03q#C|7hXJg>t-t)SJ=#@4s;j0zDIaC)rDbIFLjdc{|(K`%u<0` zAL^ye*SPMpCmc=K?hdq=)e=gin|QqIIP6NF5xD9hEClPnYJs*wrC+Ar))d$_sQ9e< z!GfeYx+5sWWz&esiaxy+>$8DLLz@lD%Nt)SPxx>5CE?ECS6yJst)`esW+p#{Pui-S z5rbeT9=$UgTK?o)pO$f*nf+a^YX}*XQHXc7}Gl{1M-C9AF?B^Phc1|{~F2!guk-BI?tPE=n zZlkTKP?g$8CH%&_vuVYhAn~K_i|3OpP`fByWmYUn@fXn6<{+lniW)Cf_4(PT{EeDA< zHsGRXGSLGy=0%)nlCT~wU^)#qK^ueGEwXC}qgG9FCe?=+&!)!N)k#gbE$22psVG8z zKuS(NH{FBF6GxVmf=@VYw~Kc%IoS&W`wDJ#zMI2}^*`Qhk1QQZXE?1=QZ+RyEji(+ z6Be+O7G->)%nB)=U$>!FwYQW_8G#Ze6YdU>TBc z^4#9(BCb2h=MnK(j0nYu{4PSPZ`t+C-qvo_ZSV}+(AQ;MLlL0OLLIM@r(wgGfW46% zaISeT{pR&u*!-onK*6Hx?r|lVp(T4mh7KBTV$|KhQTxVc@(dmxM=@f~iXAR4-H2)v z7Rc+;ozqT*wR4WL-JLP!RT)Q>^l2he-)Y%$wQzf9pLoTMF|$t~HgFR}I}BdWD@i>i zhC*?tYqtgs`ueqMd5&7kl?k+^!i9WspD>*SyJ8;k7-p62) z_dW{zFVO1d&8WUnOh=yW(~@9LHug5&dM-YS`D}P1 zs&Iwf6+Qk2B<(Q|!g6YjKd)-|?B zV;Zx4qgpEx#eHQ!nxel9-?QGFp?H9r#U{v;-q3l(v7a8&J@Z}-bR+<;tlwG-oVITA zQXz-^)7F*Fz2xhZ=j0+(B^&KgiQHc3t`vj1=k?@5(l67;q3M167TEjDtm zX!Ykv5ZuA>VsuqJWDlY2HmU9qFpu1D^o?!qq%<(5Fv5uIl6-+M|RXX&L+Jw^ht;PH;$dU zt+KIRK*tzUE*KbOls-D`aCn+oez&D$=|VQs&_D%Uv(sM0c%iVGq3-P7Ub90de_qnM zJFZ5%*WY#vG@wX1Gz&=RGeGI4SA0o)EAzEff>YCsvEr=D1w-=$C;b{5%F3BU^OSK0 zkyX~2#FcuEJAR%K5v0C+8McYXp9ai^%m%Mel=g;sb}r(T%>X_PwZ$pc5&h{{NX9K) zHMtsSJYL1Br}2?xC4Xe-1wjnu;nB>9&TnzVU9LlMxs0z{;Ezh2pLmj;Tc$}3H>dUG zhTKO;fB$%(K3;Hb)Bu%+N|XT!U4eEvc1jLuloOr@I&LWX^ex-Lk|0kB;p8sgHaFY2 zH8f~D;__tN`W*3Maq~_?>Y<^oG70X>;bE9DdSmHY5TW+c#;&CP)5(qeLW`|OtQrSc zmIq8+RSEYYSyVc}=NA!CQ_n^4FeoS{WT&qI@j@n1BTSa3m6{t8&efquxhE-f%V0o& zze5qJ&@NVZWQJ+4Acy2q98Y9yLUrk}gQpCQN3!!sXjQ= z7rI^>Q+LI~6QOdPkuu!Z3``o$avL+tVR)n+fA2(Hv^d!{=_zul9jF=?QWwm7tI$eQ zYFtwO3R|L#T4PN;9FNdae*ALr+AV=^5bZ?ywa*;Kp4{2hfh37{UP<|SP;5)i4u;<@ z0on$!(wA5|fjblAXTBMFoc(xvkgMi#4)UU-qI(Pd_e#J~c z(<#ZlJ?4a!4?5F{?#PJ5mJu(K^AEITuw%wBh*7|D1j}}2l+QI5?F~k%sunNOoIns! z9V3(`OCKs0RLWi~?y#POesE|QlhSzFUd~P)x+4pj!Rrf%-PM@#$nRa;c(D3_0U6Za z|L76Gf|hjs&C+2O2m$b}+9)6O9yhiN8h{P_N_H&9e+Kd)hr<-GIHV`lF(0(_BTMY1Zm=FIA0yKtH>%?DOYLuJd#ZXm22S%!(U;PVtn zft-S?(&s|`z}AILz1$nP673I2V$|44WXN_iO)=ns<B9JHBC@$){g$VMVaAT$ISNG22H22p~ccFkIGM94R)}e;o5DplWa4XcznD zlU15H#Qk>9%c5kcXMy}+uno!UZttWBW5d8J-x&qfgiJpRR{_I5q!<_)XmxUqn3R}O z4nonh+!?{o;4twGr}8j9im4uc)Z2Kc$GN407xH@8;nhQ5@ni%F1s;$nM+btfAWMglwUURJ?+MPQ0p z^X|ZIZp4+r#g`}f@|L%nr+PdkfYZ53ok?HzdB-Z0AKeD`%f7&0hU<6Br~CSAF1{dK zjwu7rjx@JQZBD*bFPl2G_HNhR)6k_Yoz4=Q(7&Gp@XllP(?*7ZhMvuNlw62ZHZoTW z3gRvprr^IlHxzZMZR>{o&UnUc6ct=U7QPxLVeGo@h?wZg;5NE=aUxz78QOM|jHOkN zoWeN51&Iy;$d*qMa#NX4h)53|$qWdd85K4S}nfhosoW4PekN0o8u0{Iv ze&TkynJE-Fbf3j(KXz*;P>ADO9NOV*2$N;?pw()D(8g?8uaI54(;ZU_adyTq{V+G@ z%?=55Ag(o8Z*oeDAC;r1ahVHS?P1_qcoM9QtA=|yQs$F#6eZIyXwzj-RoU0KxMTvU zyTVkiaOuNhRTGfWs&h7xn3LY?A>m#9k52h*nG(fbtb7CQ$*Q%VWIsup*rkYB^aO)V zUV?+3lKf38ChIE41*>H=y>y?PraVI%#kL-?odfg=M zbK?5bt%0G`txd8QS^0tD)v}D|G|wZ=B)8mE&jplkY`)pK**tydyT|M(>Fup999@TL zWPmJ`pwKO0V^ZB~M6Z8hlIef7fd$>y~<; z{6zSaxmr_keofp5#;cxsRu7WU?QN*kqxTGRo-~xZ;LJubsl>-zdEj!EVR>vizuqe< z?6MPXzcvRZ#ki*xpbnZx+4)p_o8zm(7My`pjm=9R%qXb>L`A(Le$A0^HD^HXattrLWPZ%m-cuu?1V_!rd-qi z=IJnvp9El*LS6rvU{^)U0P zSe0#07UtAC3^OBb02_?VbRrWif(dj6V9qE!xOcem3HJQ@8tC@2YQYtiqye?l`QWe( ziPj4H+4M?C({Ozx z6glwu;O#x^x$S$JqL{i)n0P68!0QNm%Yj{=bLh5J-C2xET*PEG9XC!~%0z~__UuB} zJv{SgTxPGZeCNh~T0T zDNeT?28uc`+mOT+<*a8ndjBfkt#nHFzU6yIcuIahLC`b!4iRfxWWCO4v1NB+{WdA7 z(rX_DdJDnK^ZV>XE?3{u7t=a$%$q>apk+!yX9l9K52~?igjB~q`R4R@b(i>LtT`I8pza_~NQ%WQb15c^3O@_9 zAhX&cQ56tv=4b>tk>Sh`szawaLg+q2*Xs`iHkD2jXI+^l`|019nz^PX z=gU41Q6MO~Vo%(K0xdFuyoOBfqD?u{d5rlqLAQf`i<*vOm^bqny`o9y$d13rIs;mM zJKD6DoSNf}3Ej6cwG#eqUGN7Df%$$nZC<2j_~MCd?}0ZJ!F`j_ z;#`X9C+FSgDIBhi3>IdIMLZYJ<|GtTqZh@})E`XoMOoW^(RVeTUVL)QVPfbu|Ih{B z#|?W$omI;$^jPlp6zs=4GoS8Rk%vc5@+^J4-}-{^0)H1|FnY~PA%*h8=16Dt%5Aej z3#+8y8i=y$=f$;h<<3^dlb(;A{jrgISuZV7tTC^7X>_XdvxHEsLlxA1{4_IeLnDdjiZud*oEcVNGXVm17R2L&ZvwdLPiaO3z??X*tdH!Y3uz z-oU+OK>(Z#1Oj&%E7rdiS`b{}whU{b61GXIU8=hXunKd&DymRzv&iIAGtm|ON>XPf zMR1y^-RZ%&Ij%}3uSPkTUxRvuu4l~8aMImDF;zp|7n>Xh4v3@@v7cl4%Ul;*>@9-5 z1JmmQJCj&e2gH%<2=b=u1fUZ#e%2*%<9G^Aqz+qI(mq3*N;95MY#~BmwuL6_&}$0v z7pccZacPd9BZ!*j5YLH{iRrYyTB55E67pHr0y&ROl~JjEE5j|Q<9IMSTt*2eSRL=t z{ifs@@zvOHu>v&i$^{a7j-9rI10qLD=ZNy$=2w~Hb{V(tKp;M_x&=*#Q1@v7bxjkwX*uS?9P%?EHEJ=-+RrKsw{6x)dtqJE|7A)z$s;uGz zPI`K^YI9b~&6F@n$_z%sBnPu1oStQPKTjrJU%hq<%*(my3Qsght;i7)iEAjoH}!RB z=SY7@ZP`>bQEc>BAIF{2{uhMO?L*r`XKIe~h-T1UB1L+WwsnYfmZ2=skmbooQAUHe zR!e=4@UIH+R%?1z;%2eYnfYo`=x(LJ@J9J7OPMB1DOqDqj}srCQb9MPRZEXYmXCWl zh-uui(W`lF>iEGrTS>{x(Jv{hc;jZ5ocTOn8Na@p0AzA?d0r6Wtc&a!)`COk8PMRz zF|-=t0|3;lrm_d>+9G1USF$ttowHpiWh@7}2wXr!*)isDp+ilMUt<4T` zm%`m~S9$qK_bqfFURTTN=t@H8-e{F_D={<3Wl;)aXi;xc8{mZZ-B0b^6cTMaKwtN%Yc-$unC(bxzYAS!o`iy(-_e*)*t@I%@>=IT{dNTdKXP zT@Hj@Ea3Ljr4UbPB8@Y#s~nc#+>vsW7$!s6=(n~=_S9(g`Y}b~wpzOdQJ%OduUyab zf>;e*b+DKO?IG71=S|gfNJ}^5?#z$ycv+`KrqiJ zwk3NX(1DG&%;bo7S=)0I!aOl!B2DLs2`;1j37&&`2;hMJ>riP_P02*rSh_qgiS|m4 zc>q+P9;Q643){Q9j%g=oJy(r`wNm45gI|->gt+ zE|t#HF0@{p5@ax3Z=ZW?@kS}Z4siZ_w zHaE~cgkByes!UC&MpX%TikOq_dQP*cV+e9#X+f;ei{a$s8Wb4Y^a* zvXCttGn0|iSXSDD&CM;YuaR{)=9zbCs5#~-yak2H1_eNCK~=;Y5Dc#iAwH0XM`i?M zQG^h2;TFDdAiA0IbiH0Ug-Vj)MG~Mctej*Y`aWTtV=qn@Y#Z0I#!K2%^qL?_C8xB+ zMq%RfuZGU04n!H13@hI}z4nlctl}9_xz~IMp=+k#S3+24OgL<^t3I1O)>AA_R|Y^o z$hRFE2Rnj=1hfl$3#WhE_6oqRg8w$EJNpxp?z>b9KLwxNVQ zr4t+;=mqhp48BAtJ^pF^sKdw6=nNp%lgt+q?)^6P<^X=`tM@mWz2?ka0N0iQpgQF7 zD{y%`0JoLH{ z4nD4!T(_w>;vOy(YE8*H=ex{MPn-f>wEINGD#!)G3sYmP9B5SP@e%P&(|#PsLPZE+ zuQBlsoj0mcg{pxQU&K^n2{c~g?}85=Iq}B1jeaAgV~a1HzT#qra1c+uSTmPF&}yK? z$?euYX58Ko%+~mF@8%c7rmeoCmvlQ{f@g4vNvfBxiSo&b*bAFnn#oyuD#}RLB22Pl zs{1bdLHv08AzcUedXSyBy!S1^Jp7#1v*^2~^ zQ!eO0t4Bh!ny06V#j&G6liq+%2)H8)lHy>J+xP@ua42O@qPg~(qZw#nF(uIbO+k=0 zzQ>+tb!=2Xb*{caWJC6z*`0tpjqV7{XkEr>fOBop0^k)!$j5Ppd zTuLfq!@%pDrIx(x^PNC!)P6Sz9g3=0=gd9KM}ZI%y6VAgY~Q4H@BNLwl5ari1!u=b z1Qp32WWV#s4C-l`QrwtZK9L9kioEG|kWF@F3D{b&)$)5^$Ypwu=XKBP?3Z)P%}}|0 zTax2%im~C;H#EnVncfF4=pd@%9INh;W zhmI>gsY}aNY(wrgng|4c&z!%78+$I-nwSxr@#L|X&6YG)O>^jJEf?%3RYb=j?o>XO zm{0d_DUgvJdVb@Y;+_0T+a>Ua1B$2Bg64>QHeIRINglGwYtTth05R3)V1^FhXny5FY!e5DapMVg$Am+N+_6M3Mzs(eQGB!}Ox3l~@c;wpNiN4) zlSulGKVGYooVZtMZ?!Bx>djASxjblm+KA`}Ixez68G;F+{D=Wx58uvpi<|L_$H*2sFjsqd9*_;n6D5ZaHQ){^sxB{uL8Dk;ADOW7=9?D98OgK`xxoyYaW*NmT zUDTyA0R{n~$fJ&86713D@U?Dv>1&>ApTe06)={@TsQuf|^)OTgJZd;9pTJ^l_RXZ< z1cj(=Q)TFri$@oM#Rq@jFOpOY4)sFx1zNtnp zFa1i8%e5eKI>3Q+A3aN7;HOBhEwY8=qHU=jbLGYMhkV z3#6%tIF+RHD)91BBfVHW%|8G@$$gnO@S@&m0H3e)}jZrrC`Q;i=_9x6s#yBJWRA%?${y8(#`dN zVnfnN{y<@AzEVy1u`ijp80}VyL!}r?)Ua?Rx|dg(c*u3`A*jSnmilU5?e?h2#}|Xr zT8)s?*N?}LwW{gLt_6f0`JQg$q7a`2T*TSO0}Xo>_sU#?^Lk?f+;jFqpvY1o!HiYw z;Yo?oy&~l3n7Qr*xPKH$LNW^E4PUfbp8n(`5L&cnnSJ+T@Wi*#6A74^jC&@BYgTVB z`QUpNzSg)XcXk%H3Cm)M=b{`%^?Sk;EFD_1>&({;Gw-U#FOvXVjlR<;dy>AnF4?9wqmO){Wje0LN$ zuDW+e@(ZipX1Zh_!;Sq{DT91ql_hYi`Q)WYPjw?)!_4p~wTz zS5b31-WqjB)wb#0Qu+=VYuBcr6)L$^wHIr(kZI~%@Y^K!C(iTctrxT(!M)db=6 zAngx1e*^se2li}$F*@5#qnG#lvPrXB&ICGp|+ zi_Am-*}c{2&+@NB?MhA+v|h+g1yC}j!I$q`zCDe|Nc?Ne-Mf)=WuWMZv33xgm{*8) zi4sps*eCZ%@6thifm$K~i-98XA69E78#Zv5L$G*<5C% zwMH8L(WCUc8$l`Ot<15iPmcar&^~=k0hF@%!sBc@8uaoMEE-&XAX`6;oT7Zf+2F9l z_v}skAqtt53R5;WRP76b__~>DfqYPd{;6(G%7aICuDJm|--Vw3A1F*%&ORoQiG)df zuQ~eHm_#EtM;~f>O@^2E{vA)aKII+y1M?|3)0ZwKr$_trdpZa2j2aK_V^?WN*i{-V zTyvBJ@A?7FEMG7O{}Gy*tV*~l-5^EZ9Ksm~9DdRFER?`|>)bx*7>lYFv8mf#Xwou@ z0JEPtM!#@+7-lwA2O!7Ft-FnpK&vo-y1?NU04!!7D<@|9CDotttYgw^-Ga1D%u>YO z)^gN6y+_OzIDzMoOIe=YJB0Odhpx0>$;m6Z1)mQ~mpq2o(60{A`N>z;SQl^ILja%;)Q0mD(k!HT zpSBk^rvntZ40JEFpKz`Lde-ynnQ=b!*DAZ_Ro!CU9&s8-qwEngw!)L zHo&;CZGNT&@ToV%mu(il;#Js+ug`Ly#_Ae54M_FYEB-iFB$L$-#Z51JNidiT??Kl% zIMhBqm13uu8WqX_=d1*<(c4|k=xFB%bB-o-iTEH4@g=Ue*0nVpfaeO@R3Xv}Gur~Z zpiw>Ura<{#2@s7W;q;Awb{zx2QFXMoxJyCF#s>gI&)>iTNJs@Z04|k*n_K;nkAWLX zlqRXff^o6=SHi0&pv|1+F0L6quf0YO0p}~`Djae4zeG?PG#>j(7}*w-DOlwJUdh?{ zRsu)^oTCY+!nt=IqDGw{CHH=l;_nxC8T=gk2d>}nbi8}L#ODu!p01N@-j#@GpqIbt zpaFNjckYjW`3GI~{Z`ejWZit@rfS+9mv1-L2LI%VKd6%jZd3rn=vr?O$;Wxm7TajW SB$5ID-B8xNmUs2Ri~j@XpcXv< literal 0 HcmV?d00001 diff --git a/docs/mkdocs/docs/assets/images/sfapi_step3.png b/docs/mkdocs/docs/assets/images/sfapi_step3.png new file mode 100644 index 0000000000000000000000000000000000000000..ac152b69574d395d32b3a5a24c321bc8bc1e257e GIT binary patch literal 279194 zcmdRVWn5L=*6$`Hq(M3b38@WinoWs-N~?5>#HPDDrIGHCMoJoK5RlxIba&UL^KPDV z&U4QD9`48c;lkp#7HifTV~#mT{b#VEyfoHRvZnw50PF3W*Y5!Uq(cDU$s`&I{0)^g zqZI(~^gBdCLh-GH1dXDtl@Y|;5CC`+9G8Hq7&ArWeQ=*2gp7!e7lp53;%kqH_TmKa zu8j8;ezf8zOyp-m6?ML5;;)g-H<=~dssR;5V$Wv2O~b(5 z1Tr4vW86r$gQmHGHc-7~mwG;vay;>th17tVeoiYAZKE5yWFsnGSbpw z(s1eDvmaO@n2hPvQVyIRwA`x>KSQg$0EpqC<)c+xB%x7Do(SkNKv4kI7)L4VMtxHC z1p3JoQ#9>PQtI4;e%Yy9T_zkv1O#hA99RHwaCb~7fQA!Slr9MP>Nsi=ZARt^{>yeQ z|0WQK7!6m%znmW5@8V7NrTWW-HN0Ck;}B-XNXGX^&Oig-^MubmT+X5Q(3OjY=%lQ~ zg0|POFXB<9pQ~U9R0?fjyi#VBOCl6$!*5f&AqgdH`$5nTJ(fwTh4p6FP! z=5YsD`pzuL9LlMow|ECB0dfPsWrr!%FcJ0Q5JtRvmHTbAV6A}mBWV%YA8%7hhpf`< z*aGn}X>KApNzhP@iGn?cAkly{@HJ4nOhPr&Y)EcX{=~r^=!$krDABcPV9r5B!p?v| z*S`AcDW_qgN1{PuWnxOAXTK0DkxaLS-duUxM|4L@N1+vKjvk0@0SspFXeIy+$D}Se7sLL^j@7+b3F?EAP16l&TLa3iBqOBw2 zyy6!ZQ^S;27mbv^1?&S}nV4)2Ke2su|$|BK3=x2xvIuC{EKn3s5$euBz# z@@e1M6?L<83g>e?g&$&=?j$Mm=*K5;j(Ob@- zJSq;-0)4PcGfG>3GGMI1&z7g+9WVA>%q()8v_ynC#Pr>1_Hp6&LbpQW!dq_E0pG6N z+~E)iDrBzexdoer@gD!4RBxAc}xI;1VkxKy7Pl%c+s6DI_;TTqjP-&e+dL&)oJ7giHkd`B#Oi1REWz99M-@1m~@3g`NnOSex3ou9>f} z+Rrb!cL!{9jT(l_1oaWC@~PIF`R#b8BxzP?#?4p|g%h@EbPFCt*ql4)E_t@!W(R$j zuIbi~Yyn#cj|%UAGX}~NpQrwClFCR3CWe~TCXA#krR_HLX>J+pcEAeHT>5zpF($7k z6$DgLxd#yjWpv$i>kl{=&ONC-`);Ohll1wHn`jh=iJIty-F!B0F{Ev{y2>$Ro z0h?-@gO%GAndLb2QG$4St#@NGu`g;~EJZIzpV4DMu@jpN9q%a;1On_V2dWbrO!fm@ zTwz@6X{lwL8GQF|xEKUP>e(+rz*N-%a`~makGYsfw-=h1KxA|ALOrofdR9CUxzl%S zZ#@`}@hyn3zIgmHTow<`$a!_l%*JnFJbis~cyhb7z_kn_105yxBz5O%jMzgq5IdfP z5}o`u{~W-z_9`+!lL)&kBO*F4Ml@267y1&~MV;&{N8UCzNVTH-uH1pWB+a^d3hL=CM3gIVr=`P_GQBWA^xC zb$ND4e`%%KU1rI~!K9`>bS^Q3a~iN6aNIs)2A)kV)p)`5Q_Wmg zOXbFe(rd0WgTj;ferx7x>~xM?i*I-G%N*03vsnx@FByLzx-q(W0K~@^$Mr7s-GXIO z%i60XyQ#sFUnL#^jC=%@!x5e-M*syA_8pyZ8JA+!EcV=z0ytudBvEE#Ypev@sk{*auq0FXYG4)L* z;=|$zO^IU*W)tp~h;XWMt)aU9vp;dm7^d(seN3c&>Dkf8wnD3Xm^@Bx&os)6oxplF1rdrRcP`Eq2jsK_}n__^pyH`9}< zv!su5BCuZXY9`XtaLZp&87nwxaarOpTLY6q=^ z0_g&&oBG`Oj+;0A_JfimizBr&mPLgrdv3$7weC8%Wp*Zp%5a(uJ+B#7dRz>$0Z*0q zc?i8Ht`TS$5yhIG0&LDMz`_he7ksTGKKH*?K<9~dh(oO?G>uq*`#8WDZG5mA{Gd3olvM33(E6D7{q%>=e|fK-J|3%4KmHxgaO7YApz3K$>N?~U%CFauF2)_Km1 z;nwRXL$$X?a&iD>_%j*+5dj~71b;$+{{n&_0OY@(0RRU09d7hS2Le#xcYOGtVkY9h zN|6pTk^c4kWb&b+_y>u%Z{hb32DXNVmUbpq_C=Lqng9TzDCDD>y_%e?pn;VIyZ&dZ zPloKy7S<0<0AOc9_@jlPy*`bzg}J4jptCUTUo`~b&kwJGv^0NJu{RT@Rg+Vsk+8Bg zq~T@fVCSF}c}hb=1GfEaB>4Wd)IXcU{}ZM)vA4Gt1OlC$oY{hnMKu!Sx0U!q# zkc*2AUW3if#nN8ina$FU?jN1}tDo0~b_TW(YkP>5CCx*>`k$;E?1gD*9|rpS=O5o` z=nVPyNS1d0BnwU;@Sz0A$<6_MteHK;=>O8}q2wRU{-Wz2!+{?<6I6sa8=9-XhFHM4 z3jZ__PHq8S@L%KnSJA(}^dC)??F?-ttSsOy?M41QFaK=(pN0Rs;a_8_{l}Ob9Q^+| z=6@9ZN7Dx)1Ql!{@Zt0yxG2I22L9K(f7S;BA6Wb!EdR%R{`D3&o&7Ug`5d};QZL>};Z@SU#p_2 zQA~*w?;~a+hxdyf0aSry8-d-qdCU8S+PaDHle7L4^Hd?v)2S)>KY#u#lopk$9{kzC zV@$PvMfB{o81jE#)5)#BYX;M}7!he{{_Da;f8(txj*Eczzb~I^Z*Sy_>c!_TLmz22 zr1V7G*w|RD+EjUSd;6)du&}bc62*UY0`JxbACXQv0yA4W65F6F41J|5oF!PP`Fj>1 za$E1h&~eqPffgVG>6pr4$4)*t>Bu#{|8ie5Ww zD^quOwbSt{n1BWSm~Pu8OZTTeBvHW5h=2JDT|2hWuHyN!;C~Yhe>22~sH)>)$bWwW z>xib8Y0r5ZL&osF$Vb|F;ZrDEyW2cW2QovxJSzSOO`AeK!e^6oyEQZWOXLguFK6wb z!dL+pI`X?5DH&jw3jgzF*hJDJ1Slo}M#q#g4W2~G>Htsh7n5LjBMHqPlh0j$Rp~x5 zb0~>`SKtqmUwd`}$PlRBdgZ4%J}o#T{O>7-zX^;0+-+FX>MCI9(zU`0>-K6FJU4Cg zS%SUMzWmpxE!#^MEAl?iaE2ututRoXCF+_LJ zD?#@El2#f-3o&>ZIT+bc{1JX7<3#}$!lEKV|BK)+V&u5+G87zf^+!I+#|ZgFD<56X z$|G|&gaR+CZIe(WdW1ijqzLGT187@ykMOt(0bXWth#ULl5fS=-MB=9N#ZnN=oK>xy zMt36Qk&pV)Iz*7jXQi{bzrXLW6j7gTydjj}xcXfK@3Dy>K|!R;>wijTR80LUK7ixX z>*iSWTMjU$_WMVePlhk%%=kiPYc@md+x=sVWI&ePiCCXQs%W%G2DBKlvELTNKT?kd zu|uMD)qbMU4aGbmX{h1jfsANT?xVDn<62|6SBz0y=)2(mEO?{}H_am1#pjm@%?M=y z7{r0FK_MWk_RlJDR!V#XJhd4#dk2RA4E8!2@L?+wkGk=F2qDWd5EeM{X)ILNu7Wqv zUktv|4C6-T9453o+EA&rUha!Yxvr2ekPM1k?{XORCt4pgov%YT5)C5I z^D{hM8^=7(=ZJad>__W#fOlf+g`yA}VC|fvCjUmNT{~aP<-lR;1~^~eR!2efvPU5> zp>dU7KT`PY_evxg5A$wCANvw1)LAN#*FvLNRQm`v)qUA;(Cp?~H*y81_24j@`j%yjLX!VdUgMt- zNkz8t1ELo~&<{n<=Mp?k7ULgw>x*BEBE#I`g60`lWBKhu^G6YRHmE}F29d%<>w{_J zKnYX=pU=y`f<2mdd*fJ;C{@~brz%sP)_61zabzTlnE8i%FKZSIqZVzpEX7@HGH={g zF4NdScCM<5RY%VHXwZkHYuiJA)NDMEOz3%5nGvudeBk0BP%MMme3A<75+C!E=xT@CF>{Siy1Ga$W zxMn@C_m;}MZp~)-Jg!a-VHx+}#>FSVg5VEU2AMNEt`mhn?|Vk(RoAkd|GAWg29fXX z&Rc@KZdRxf0tVJotmfaVC?VU3YzSXZK`pmtjlO2vi?rX-{bKBU&X<*M5UA4UMQ&d1 zv&Luy1m9h(ETtW|8$Sz7o*UAmmNp2X@nzV~iILwjM}j z4@=>odylbCbo=x{>)v&Rlk>6iH=k9PNtVEVbW_cI)RaRZ@ATR;(!p6>y)^Gsi;v`dj&JiFO|=H%xya#;KDdS$XkkqmWAGZW7LXn#=s-^S-q-x9N` z;bQYIdjD?f^}w`u(_TvsUj~!TTuXuNW&gQTU{~M!>7^vZMPjq05mB)@|hwn-!(9_0Ur%Fh~dyw1IxHsWo*i` z|Ja0`2b(dkluJEpyKzJuR}y(rU@F+@oB@})Z%N?mBu$0eDfo_J_RVJRyy?JW*rkB_ zV&AD{=etJ8-wMCgn_$u%HjsVU$T%ze1xPUe)?m@EXTxOvw$@0#5&e*%r+cx_r^Wuy zGwl@#_jUYdNU*0Z@ZJJE&ZZbM=OhjnEF1Pd1>r=UwjZzcAw^VMGfQ>bR1WZl+Lc%} z9(6hQ!cD{U0{vHg@hGX~SuU=9n)A{75j!T?vk1@DQ&xin2C1)xf<75fDCp@hLUPUn z^LhrhhR2DJu++VbCR#V%##euYzgTuxF#jUj1eg;i8?y+NW(@B@!54idw_WQQ1cHVU<(+@l3 zdNv`T+c0Sj8RDqHn3&jX6X9e<}c$gpL;OhZ&o9g?NTH$puc?c^PF{S5F zpuo9A?){oTUo2`}G`VnC>eR%d@J%#Ne@xN|`%5R13X+0L?}y{6P?mC!@yc{UClHWI zYT#A&S+bobMEc8p8;&Q37rXgNYP63LD{4V^|MmSzzGxbZQC@f2ZSce2W*~_UqH@4R z7Qy_)VY%lA%&p@9X)h+&LUT|{&y_k7b$=Mispn0S_s4U@alCtvgIs%MP17)fiE1%Vpi3xf1%zaNRjc)f^q&F3pEdqcCz2d^^6mSwt1xbV!%W{vDYTHN8`Z<@lMQYQtv4 z{fDL9uG!$_WVc9scQv-#04MpE-*7A7D>dIz{yIOo4w3m`&cu&p?9tK;7uvq1-Z}+gxnCo5zbM^fGc_KHzIHcQe|fmY$z5&& z81DV%BjKkrHj*WIba`U3pbJZ$cwNH1o@a6q0Y3hHfu60xq=#=o`O|v>LO_x|TJT^w z+N9N|xV!E#_ z&kC^CNT5+rQDvFh1-3nZs_NvRsx>RoQ|qcd#Y0m-HUdx%Q@nKP@h~YJsm9vC>-S7O1 zp#lOXQ4{o0Z^@_AkipK+q>ScjzKP|}TnLC+@OqkC?&kBuvo9i zr@s6<45*}sau-+TPS3y5u>X@q5KFeKBNeh8mB1n*+GUSYOgUsaBPKl-l|5ki%bo_w zB^&#CWAI|ycfxE!t7Aw)aIO9P$I0!i6P=ugEB4M9&N*)+0Y9S4Q5mITid&mUKw~;$#blz6N{EVUb{qhcMW%-xkqX~{Vi9>KB>*jZSh37ORU()z{ z)hTlGE9>5u`M*w8rmvlS@BVhEs|XQRv+PA0Qbuv=*M;A6?g=HIFCW0I#9ApNFD>3w zGG~zJKyh}u8Go@|Dvn;FUdjPq*#c?u821f)BN~V#KfH-b3fQm}UKi;64Eu3NI z_EFOupIUi0FisyDLDy|;ofbTBVq~;)#9ut{i+5j|psUGRnEZ92zy_Qf?z0CPI^Q#S zx(8PIsW!C8DT)MCYQzPqd@^q+TBK7vqmnq%)TKpDU-u0fE{K^d)-$Y)A(tfgWul)L z8za3`Hx1Wow@Ein(Rgo1Ma~em(GBHa3A=GF^5N45HEezhJvP`1nIPxS<}5ph8F$sI zh48m<^`U|Gppi2Gb~24KwOgHx4app^^R{?KTirgD;z>zpV)6z& z2gTea25}9PCoLF*sgF&NUUrU>Tjq~56gn=*{(-r531+kAoK}5Bna}&P3dtQ8g>M{Y zGta(&DSMT?d|{N>_6Q=Q4&2I`fKiiu6rt?aa?CJ*w0{bhg0^6Vb&<S7IMengyP$=7h-uXg=eRh%{LA$V34VCa82 z>R-iLe!C%hpCnloDCXAnhU|Q;jpxkz^~yLaE{+9-3WiCmFo$k}ZOD35{Zjvi!wy~~ zC`yK)B0E4TnZa)Aav7tJy3R_f-C(fM&{nfO<@0N&mTPTG#D30ZGuhUcMC z5Xt(D?tE;K#@34tiD}@D;-he`#t-p(p4!*?>zicgGLVvK6Kb4u5m5s%@1S&=#iRV9 zf7aMdm)`ZbiG??An1(B*ovR;f1~HUbt#5|Tyx$vQeFFm1^}?Et}qOO3KS1cG(-%f*P96)qO&Y^1h$!o24ygW*(AktoC-H z(7A*&V@A%*gxJhp8>^eU!%)I4M0%uuX||Fzkx+~^WeXtrKf;YZB6w=qo1EcLg8UiV z-%p$cqYD`=xm)aq(8Ej?3;79|hmo`DX@`=!`y4C1hs06i1)#`P!JPu9=_q!ekJ{55 zPj1wv@Sk_|zh9yFgp9tPbWE3~?Knm6iYOhULZ1P-+B*GUp+Ux|%_x#w!~Ky^aMVss zIy;gmq5d>S^Mp<@Y$3rS9YLob@23BU-m{vcZ7pC9_;7QeOzF)^K5PQ77tdPleSD%< zs0zh%YJ_x`Llelb$(3=%2#j1CBy6<4tu(qlt%4GLchM~>)fR50O;Cp_c+1Mf9o4E8q)-_?WE$?C9m zlUkE~D}K|&Z}5a~>}-Lo)_2;Vq!ILfZdIsbl^?zHg<;UA%QL>l$~-yA4*n~&#c1)1c_`GH2csmvL@OM^=DV+7RCToce(gLc z%PFQAPLznvlYz>D@e-#OX)vo75lX4UdmtNf}?B`lrh-_kaM018vjbF`*PRmW3VY=(XjJ#jwc z!_aCd%A>O2!k(Rd?_Gr_OL~~idc{aRYL=%0^61e`9@i@cZ3a1@8BJp1D+LYLWmD#B z)7G87iE|%HUpgE?+LO!nE?O!jzN?Re=G-#j2W; zv;fRR`R{;ZsF;$ov>s!fn5g}$#-6QG7bpLkK4fP;&C^_J%4B+@G0TyD3bXYhn1v~h z{o=63NSk!bnw`$Ab2fe6kbNSa9xh;#9R-=3uY--stSa0)aEn`XP+DxtxQ0*02}2uLks$o97nd zB0W#CmA**!g(zx5>3G24O!`v3C+#cG5=@3$uICQcjbI}8b4_=!ChIEGL?h7VG7_N) z1!p%^fj1Ajp9*{Hwbm`S2a4Brz{fXip%V3Zjf4Zx;J#EV(Tr>Q94%7WCAZBz+{6zD z?Lx4n;Wu>PD-Zp(_uPaE2f4DXRzcUtU_Gfw08|feZ7Q{6ZGPi&`E~Zzi}XO?a8151 zL>L2;nj*8GGf^@{ndp%ID|S(Q=hlG;N~g)pEE)2SpZ!n?s|U)>qAfocb~ZikJe$1ROL(sWH(e>i`a3&F9%z-K%szu^8F`c zFySFzJSb^+<~7D*vSFKxH6}vCpG6IO|5qqpxI@FO^>pmW5`1yeb;f`|ZzM2x_kgdL zZ@7IjWDHLQ%#kta2bD1zC)|!WZ~e>=ij~RJ(yM>nx*U8~p|rWX8+J!KD11IECbYd)J1k(>eKivi zn_KAdcRYWG!AGiH$SEv!b=tWGZ(YDx(GMO{Gt@*oOgwT8!uG)@*x{}V7Vh45;ppak zmgWgIYiRv-9yJ%1J}5L_Jx4*El{To)Vzj>+>;v!@^V}a08Zg^-5pkG}H682o$7_)u zX-CYG&Fl~-JO2$l{gtNE5_dXAnu(K{v$67Iej_0N`nn;4;Se6VG)=@bZ1w&2b%Da50BZ;^{}iFUZZIC!25i7Nqxw+(a*@BVS4(&B4U zJ>J%bM@PcH$4XN05h3}Br!KHg=0lGmbX3G33+^eZ_XxkAyv-xLpfHc@#$Vd$aB)?@wr?ybRaBXaq zks7(~gHdCesie1YauCLy+6DLVrswbHLCDk`WR?i&*IoFXw<)Pk^DZ`u0k@s4-q^Y& z*74S6c_qNSrI}j`X>giKf(`?JS#gsUk8mgPzw$XGT8IPwPAykMPml6E_zd-rj1e0# z(3($nvG$JPf!cnd6qC**za50Vav{f#-2|6WRV^FI%E=#KVfU3PY>PUlv%?Yj z?u8BO<{j;W5Ia-O?sQ={NA9_Ng4Cyco&&Bj{rq^Qco&;$w7=*Mah(t0&c2gL7w3B8 z&v8;5sPhZy(491AZb^5?uvRdnKVTTld7tRqU$`MsFD-ahJv#KI+IBA4iZOc6F;3r7~4>oWf|sS)L|5s*2whT6(LGG@o4@W-L`W^n`$^5IgF&Q_Xsi0N+Q}6)(G>g1zq>*FV zl-A`9g^H{hDU=h-Zs3lY9B#~-51h*C4b~1MKL_nuHd`R!$i>KJR6DejckzuL^{;1b zY^8XQvNR`s*y#R%-7N1Zcev+3B_z9|YFCQx(6p?C0%n6%5A8T-Q|1XkWl09jy_pm` zMw!_yI9`H+!g+dqh}%s19Z zP5e1`N)sl0>h6lwE0<&ar6Bq!mhG>qir=hcsY~>^PZ=isNs~x5opl-H$6!I3P^=IQ z*9}vx33mYGO_^XP{Z#4-v2}x|yrtP(1gR5Fn1He?i_TY#F5}KbKoB~x4|OeRV>8)g z8SYy#9b?YX(hC#sJdOCliqc>4!;yb2_CrVziUs)|cd{Dmr`c`qir?HM7oXHW05hjL zv)zU@ykD}ylayBAz|#S}CwZjqYb?6K79T6k>PUr;OuFA691IM~Rd2fUmdz6*kJy(t zAKC_=&FACh+63CB*}l~rfVKMu?s1ogw*Xj`%k8S*A) zUVVLi52iAb&D?Zl9IPmhdL_$)@kJk=$wI~?dHcv^WPpsP`u^^~N>xX6{UO?F;?1Y# zkf^ZyEo%hggmfE~FJR##&@5=#mdAoMtSDJF#g+7K`A| z+RWfR6sGR{r=sVUGm;zuMV}|_yKX!%W5J57brbxRKgS1s=;jhemtn7%$%?fgJ(XXK z=X_@9r$^<)6hCGryNNeH8zlK>=e;L{95;d#vp|qZmFL#zh~t_O2U9){E0yCs1t^N! zKh1FddVzJ$;gnyODr-=&41$rwtlVXhx$nh4jaNEV+(cb){Xy{GrUtf8$Go$BqKI45 z{T%N5hAk0U?{13O@&0y?ts57!0Kk1qgGpoHC^JRCu+Jx|y#QjJM6>K0*{UrQC3NQ& zc}g!A3O32iIkZ^x!@YRvwfuO1-2&;@(dCUApq`+4I&m5f7Wg||;9Nn#zZ-6aHEK+c zctb`*sepE1{`^;&rq3W3Lr&uQGCjo}HkpjCQLOeOyZE+YiFKN=kPxqcAikpFBs z=CwNuxv`pi>6gpu{+HcL6bxtl5@xlRW1YtN7I<&a8)0b&VV=aj6tLW*Dv=vx+(}5# zA<$bHH)8i@G`=H4XzgG5`G;g-rZ76kQd4c*ji4}UvoN*4E&U^4om2G2{`FQUQxcL@ z##g(-`Giw~VZ48{@^7Bv`n1vG=GslPGRhDvJ`Qi14iV!z7hdnHnYa`|P0K!W+bd%&2qUOSj4thWkbvi%J{_&XE&I#V01 z4n;)W6x+Q4OtB{MKUn*ZTB+V}n>H8T9xgxs$((;YXR@I^TW>AviCJ&0?d(uu_;=5L z*Ih(^Ksso_(#3`;E|3FN9_dA|4t-PTu;Uil-s2I93*NQz9QITgz{)v9Jv`N-%gp|# z2>f$Qy-oBmjb;ve*1bKuW`m~Faq_F#&5%bKlZy<`65;iJKAora*xH(Y`N*`=a-_mB zQ8a#gD_ zSO33PhL}I-Oq5uaksvdzU`tUN)AT)wU9q1WN1O%ESfdr{#0LXVOh;9I|e4Slr}r!l5ip1`JhsvwOsuvYwy4j z)q0l)2mDaAz#$YekvQjrLi=@e>sCF{^ttNl&F(H>&rU+sSSsqGG;8iyf$?YbOujysuVE_Y)Bd)U{37oa-Kr=(8dAEmfSUVi|*71P#TKYKd zZxf!IYM=aJ(DD20pYKO3-@uuba?J~eUZ7J9`4w$J?j zn%NtU4D*Uio0_UL#u?8hi?Z+AK6Yw`r5#>Pw1|wuB(McugpqK55-Y8)dGTK5VEXw1 zTIo(ntxh^&@8n3XOtxP10YzS1(LvMQ=Q213FaB+ONuZDW!nhgTmhn6sq^;kuS7XNG zmV30LozH#Dad5Mvdrt<3pAma0SG-!-e*j%xv=cWgk~K9n2sQ2+0z+?KU0vK@3!e;? z_b{aV67T%DTbB&4%fOgh=yHFgg~d`};F4^Af_n+xZg=TGlu+TRd4udjaJ zp&3UAy@A6;A;BLSUf;FcU$;CUJ_A+!W3mbq_S%gzkCjy7?`}3kaf(phLAQeQgKB~h zmPtWICtEhhyI05aTce#PV+ZA*AE>Vo_L8bo%eA4EGC zS}p1mfge68Nvs{q=vPRR7moG==+Yx|d}~tR@>a{b?TP-~&E@5t{gVN_JgrXa(?6;~ z*9R>vbqy1|6LlBMp-3)j<$B^^Y@Gf`l3Q7%by<6R(#u~jibyzqFo+@_%QSEKKSMfS zrNomh@`Ph40{?th4MHbi_+>o&h4cJ$(GN!oq%aO_JZvwBW76@nkaBnnKa~Il8m?}9 z44-()qCnn8oZcAbTv)EdjbYaHXY^fybrT%QoIQR3-%g}k8}UcEa1ycEjPq8wmyK z(KyADAwJ;uL@vSkS)^vY@d-;O;9g#rZ^gI;cHj@&2&X=Nsgi#jIXAn* zAuMzT+qgv1_VXcW{YK~gfERmVwUG2P_-O^IQ7NEH6bn~6KB}=!Yfp_BBSnhX+<9vlmDz0v5vZK>`_k-eOh!K zlji5e2;UaicrOoZIvMf4di3Tw2IDwHfRcs|8K%eG-l~{#KK)F;|c=zUji( z5V6n#Olfi}Ix}__sRDxUU2kPddtj^D2T##RD$2IK*Mb?-N%G`Uh)BRWayeSWNUZ3x zhFM-vs|0OF^ao^S2mJU2*)^Ar^cNws@#1ngn3yQ0r`F{lpX`eH;PsV4j>mRkewT4C z&!jwTW>-7PIP-I^y4#=K`uG>5Yj`O91u}z-C5&4FV5_V4D!wgYynOYhx1;)LuzGoQ z>Y!RN=@sx3bmemWFaz%9Biu*==T<)G8}uOCv(RD3Zv$+zRbe_ac2<;0kt z5%YP%fz8GY+}+%DqA6DKx9}~zx45mp-~DBzB|~6RxbKFGPYn7+65$mXo8>_pf_9LN z$2X-^R#WPX(v#FX`&W|Bgh8QxVH*Zm4q+?1>KYwU-Eyk3nZf6VNR&xQC2P>JJc`e} zu4z)?=bXlasflNBDD%pQlnuyi_3Bv}^Y`Rm7AsGH&f;a)*KnjYb++=FhHT8<0PhTI zya0J|$W{si4OdMUNC>X&%=TxvHa5NlSCRrp;A4d9p_GuGvA1#y(dJq z764XD7$&yhW@cTat!$ciCDyZFD@aNiOwic02bFpx7#j@q*plJ7JjWU(F-oy&uD4p3 z*~Zzpg=2HVu8S&aAgo_4(ua}>r@wBE=5I7SoFgz2+N=6tl;`O}rLPPeGMc2=BAM8E zVv+0c8TP^a&ZY*AeA}3r1sNt7Z)d%8`n86htg8Z%37+Y#{66i;!(x}-&1?=X z1wLRW)}hdu%!#Ibha>AhZh5CP6shguBLH;d}ad->Ae67Bc- zB~X*!1UEE>O=T8KztmPKL>do<9(+AVu@H56IByX@C8j9%Q+dIS!P(FVF^v*#>~XSL zt!xtK9(dxHjnpnt97_qDE3UbWyDgmM4sx5)$)Ww0Re!b5_gZCsDaEwG$t|XaBd}da=k@%sxX4+1 z`USJ4nNqH=eB22AR=KdfeBT)U3 z_SAQs(;|3K=T`()KuA|OF+C*{|Kf!|_bec`0eKMkp{n1JgM4S#W zJ>EI2$UWpNSEP@UCUZsh5_|$T#5ASugcEz-$Udx5U89JDiU-u6+XGVk{5gdihcvYf zHQ{qCURvVw88ok-4Mw-9q8H*}2CzGj;0TAL9+#YKKl%A<8eKH18}lmO>?1x4gJ-tp zo;tH<{^xT}3;pYohO-&r&Rt#pK2IkT{h~?2r6#M^lfsv~ESzDfi>-n!u0GsvLhHaX z%$zRY=z6P#Crv&#qM|%E8QLsPcbta&Km-9+vrHR{iotXCJ-0)JvqP|IPoe=?$5)ewLb4?!o7e6W9c8V2_R)cXsQqrAaEFOQdU()Y5 zd3=Yqj@stalHUVTA6D4OC86fr!k|6vbRs_??q9PLVq=WGFV2IkWf^^b%G9%9JRBS` z@K~h+ms`SqSE@EH6WXY<6k8^{e-L9lu0`L*ZkmSKgR(!{A%v6{T#+8|n*!j;QL@5ur!HNE~*+;1~iWd0=u zWtwhUzn2`TVNCUaNu|jc<=vK;l5{+qCQ%=yU)k789OZ3#xI!(^Me{|oLYu=J89rL( zZ{s%NY!Qi)tp{}n9W_aLlo7)%)C#r@q8QI0Jow-y%&npPC$`EJ#$}$ ztF&=+TR(SVhWGCYR<@Bjo*m_F0SD@~*cx>GB==*AmtIxqRL<;}xUQLDc==3v*EaFW zJAiX3jo7y+@Uv2I134r3dgQwFYCfCdIp94iZY(Z|^MuEtA9WZc88iz{QNHHQV+-9# zrdI{mZo`VtoA+eWAY)SRvGq(;LP-6X8|~PCZhm6|gUSc${h6T+!w#Ql1H4KbnVKII zxe1Hn?2tlENjByr_@EBDkbOewoIgr@0YAWld(9GSb{6X{X6@k2jhrz1>_@XJH`3N!Aa49?~SjS zN>PpK>#}pLP)LOw`CE@`{~n##AMjHdjAuZTK8S<|M}jATA1+v?ay$6k53^G^@7+(= z)4bhj<@s*qX+^ss^6R`t9JGfcTg9Tk1^uO5OxHYmTT%AVpuRrpWwkltLY+b}72S7I z2hhaA>WhW)r-$q(u1J0~18=g;%H5yE$UkrEZ(l(=!mhA$Vtak{ZTb1}VgGQTbSUth z;Cr3K8^A<9$Pk9&p}JdlgfI2G3?KJ*p*xx$+IwzuD(9q;3!26ppZ2gn;>-o@{@jfn zTSvppggj23Vf1myJ0nblNNFf)b-cmELjQ-kw+ySY>-N5-L8QC8yOA#GQY0lsVo6JP zmy}9NOG$Sxx|CXUcXus1pOd}!v)}vj^7;NA$N3S~ajv=Mm~+H`{08hNA>;O!mPp}S z0WZ-gI)rHqKoY|=hJ7reLqdUwk(K9{LwMOW>O@9ArMFZ9}QRb+|1f`n^0R|Fy7uFMa~vw4QQcvSky zkcLmOI41l~^o`g3WyKfF#S=K4U{hUv_i*$Cq4sk{`kv&YbZPXb%ApXrK>au*ik-5) z?-qm0`m-yvD!b%l<1s-0)CsojO2A~ME~UyYqZVWL@d}2oU>)o7Mz?nkBsw|*BW%>6 zY?Qp_4Mu;u23L%A-DpDX>3(6Y(O{pxCcoKbk63kNc`tLfG|Oe=Ij(cSY0tB1b91sa zIQ<;$8bfYI1+f+&vEjkLI+_>JeQ)SX6tsR1w13F_hdwR1(2<84X5rf$N7Fyi%sRJ2!HT>Hne2s8I0J;;vu@w^x3#MqC(sEJ@C z`26Z4Nuq3(VgkJaI329v_KOx=(5OolWGSWud}+=6&MRG~KG&bOVplqhrh2q4i$j)3 z5=-LejY#;riD><4@Z;0wi$Jf;-J%$rA=@;|mB`4>!U-|wjT z5&z)gZU~7JyI4j;&l1oi3Nh^hYfu z@sC@+*-pG^9M7z?f-#v@sL)x>!skd|%>9a^h3_k|^>G6g%d&%^ z=v0MbVy`(l9#rTmNSWCBE!%Vi=WziV2Iq=T@z*m0hu`;l?-nlqFAE^2_#^r(B6KVX z=L0=$xAe#wgLTr}E5Eg>jQf3KGU)eu^A23HNX#i^18R)To#Y)=6%idQ>+)RlRy&t)52hBL3OM(cW_qRoPU3O2%MJtvmJh&ycR~RlgymhD0G*b`>nBtji(B7-AA`V318EHi}EC8;n z^La#kQ;HDRI26IOE2n=Kn9%u81coeVNI*(0g#Pv@-q+&;6`vh4c8V+)0`OlE9 zrEI{u?i!N$hhrm@M%P8#w;WMQ&TQmkFXoz$wY}5zLmkIlF>Y@(ECgGV$fx}9?jw2C zwRei#TZ%;Nh8RSpPJ|pRPRkYn8}kjxQFaTz=i5{rLsWmTF4P^UtD_o#tMk7ze8{;P zprv9_JwV5?k&Mt1jXHTm- z6RAB_4dS4(!FlISfD3(7(_9GLKsyzKj@fFGk1S}%8`slqui8xb{arE6k!wMjs;BcR zI?<3IQ?qk=(cY<)=9qBpT(n$_zWkedf)Dji$)^#uIHOQA-E=+oj8ieLS7zTo7~8h! zN=#JyiWw~xds?lE5%D7$_~mO1D)nx1uZw^zul9qIM|x!gKHFqZHmFO{NK&ysugfcm zMwr?h+yuRr$CMa__$Tqxxb$JZxXNuG&?9l-9<4v0BqCjg;)0h)C0Yu?3}MDhah)In z%9dBwxa?8bQznudKMCyXrI7lB%|If=s)C%1CNzl^0i6wmzfjwuYiL#{KEG(FY%tfN z`5x~??5xL3yU#r?yZJ=C(@FfFE9BFo@vjmLyk$8oBnG0qGoc-!C|XtNHzs9xr69c) zbR?k$@cd|-XflzD_43$#ampl}kT;i#fma^L;Iy8^XTb_t+@DjWwN=}FVcBNpB zRVM%x{64Us0N2t_^ekU&#?sq~TWaAa_BP{5iLr0Gc`nwrOme)WNHg9fSZ2owA1MqR z>9O_M2A+S(Hcm|kbBeb*>c=Z8%u*+N3FgwZnrPciW+m(p$tl=RaTD2X&9fxn7 z0~zUXGJ>@6@>rr^ICCvZx~i@L`(Clf-Rh^zJ)-CWMhNKmc56WcU$3A)U5h87OKFd6 zYNOdL4-J=ce>N`BIB%z&7xem=sh{AsaMBU+Kt94Hsok=wFarI6@b2^?NCXS4>_Noh zGHsf}_gFXyGmEhp)qUZADF%o1KyI_qEa!85Mt5KK-Y-L!plbne*<|QdwpQjecV%a8 z6QH+mECkC8Nq+k*#-2@#N*LtEtOU|%rn_eE<$$Wt6DwUYAQ{8OOQy3sSBn=GvX4!|4Am!NRq;DyI>^2A|Pl8t+?UuuVIF;8x+3P@z z@Z6hue+=`Dg7u;grI=!C$r0R)Ds}vjf;c9Z9xiPx9#GZVl`(zW`bf%v zap->QEBA3~fR?RAWYj|WMLG^zu}VQ0L%B-k*sC=oj<-?T@tVzh#{xqslfhRh(|nOYQh=1_>>w=D%ZZBaVcST&jIH- z_pae5nzNJaVZgF*OnAVUGg;KSpIl|WpSTEv7Eb<`5w!ld5lmbZy-qIb%iCYv{;*O~ zZy7b-CmPGHM~!G?iDAR;7=`fS+DeN7pxW=0 zw{MsG_DYqT?2_4pa*rrM7{sYD)CS1ii3fRAS%A1jX*Ksede0#PdEBzwJwZp4_ZDmz z@1fc>(Dozwd2iW}+hY)azt`#dHKkg$B!33qqS*Oobyhfst?#}-^a8^948kiL>GOxtIulVcu z4J&t{rLJj7@+%a@{fnX^zEV{_7|K|?$G;9XB3y9-BBtPYDq_6x7XfK#)0bt3h~AmJ z#bNT~&L;ia!xPNyo$Ne+PboL<4Ah;&QRh|uk2Hee7GE}F4hoNCW@T`l3Ru<84b%th zSq?Otw}5n(&7y8~;&|EHWtjq5;Z<+x5A>_aj$r+)*9qje#1ko%j>N!4V3dpYs)aj) zXylt82Nn298yct-r}iP*!+jTWi#rrl?NvVb@IK|&1A{c-%V_}x1X^4eDCYL@G&uXJ z;p|=n3>3om>sj#%U1D(^VqBBQU|}UV;*uguWZQtd^-BS zN6?YXQ&6!13R{(LpNo(SMQ*t#=W)PHb)Zbv5ee*V!ciDw3JzEd%QHg0t=vO0rKB+~ zD0<7aJF>-h^FBB|Gg&nh5hN)f+_;vrR8cuK?Ld3@icvrOQda$a(6ls5@k(Ee&;g7j z&I)UZ+ImG~Yep=4IFf|`+L`G%Ul45RhSZ1e^!sBT8a1MYdKi163=pHn;@STRFEpRR ziyhHxp)tPYj7ha<&1Q=-nk4ogHIhmSslZt?9xVo(}ZmMWofFyX{=xdjCmxGTQmcyxW3Bb zMF+CFg!cWo3-qj?Gq(kbqFYrCV5H=SEgRPRXO5qU`KGd)Uz4cdurOOkEUUq#qSRniieqs|a_SJ>4FHWbeuQh@HuyCIpKwLm$XGGV8cr z=y2%_<~NV}ZDQ|X^1W07^mvNK+e&P4Pv! zSJgXZ4}VA_d2(%Cmj_Ugi6>zu&(sa`fc3F;R+oE&?zeUNm?gn-o&~ZN80pW3M2ahR zH*mXR64@NSy-L5=FaVLVfJau6`7((X$!)s0g&-9KzIU|gy*1GWPQ}4;7eIVr!9}k6 zS=ovpZ9&&`{k12X1GYaL3d#9EF%Mxg@fs!^$MkwXv#Jj|di!*5zKim9|1mYHM(T3% zFz%YJ>HP3@BB<{4>@$m(c~3FS4;m(`%!p8^4M~02r52cj^unPlq2*`9k0}V{R4tUxCn-4GOcHZ&PFUK4=)gwnC&{HVTQ-EJz&J?30;U5~RN1Bb3 z%~E_aL7k(~y;GkYl`M!l-P?k#>ZbW@P}izs-VrS7thKA2QZw(0SZ%+LL0=Nc-@6LP z8KWO-qV{=k9sj0psb0z2^^6hf6g^?*+l!qx|Nfq`bBFwZ>mC6EaJTr4vB?rw+|$+1 z-LdQTz2~NcG|Hl!knHkS*1x%D8lj-N{?yxWcl^}6w~kV#3%WQg7|m#p5b;(b1+Q#J zdY&})3NHkzq(oc>n7$&#gC72Pl$l{=5%)<)OXz}b=f?&AkTz)PgRPP^Nu;_M=%@uoXZ<@yV`9hF4X&ir z9?RF2F+OAvM1}_XNUBTgYI^9J8TYgvg zbO%p9#aFeclN52Xb38_f6AIa&id>NtIS|2zoY)A3@GIg&_)4scmwEZ~97-C*SI6qP zliHobBJ$nGIy~PrB$^4Cf{uH}MZ@xXy~q}wm@Ol>DJM~Rb<8eb^^R8~@a2!Rt(>PI zc0eP0y-CatHNaXr6Z&oZggTj#!wO2xO*mylQ}6Yx=NQd8D?1}1ww|kT2){VSg-i49 zqjC;tCmH+Eb>%j7-ge3D{?${We#LABPIR1!q@?tuUS+;zl0m3_JsBoZE5O(Ai>hg} zdN3d4IY!Zo(XSY@=wsAf&Kz@#3W3@aZ5Er1t>dpp&D4(p{>_j$UzgDB&NB9$?o&4k zU!vtCIhUV;P`b8wgP~~gW$GQ+X{MO%puehU)JK!ead0~4@{PWQ{TJ>*4z$6;g_Jeq_^w zx?vpKY||W&9OtP-TyKHTcs{4w$G4y@5nD*iNKb4j+<5?()i}{8ixmx#h-vu`!mC2r zLd(KauxM1}-W8V-zvt!Koy%ieNf_ztY=bN#nPg=s(BcqFCSDZTrm7#r5~sshL}<@~ znksFS`apVVh|idv)9d%l|ERMnjd`hW$dm$ZDbDDy$}sb&%{!T1NEWn_CjU;}@UWt` zg0K*b`g!ZhrRt(^#_`-~l-czlUsosqtriCQdjozRx@|tJRZ%)uX$OiA1blv*zBPjqmq4}^D5;{aevg%4y3}r)FSEN zlrQQ%i6&o+=I*D7>QT39D1CX?%OU#p{7~kn4Hn09*gWG|Fxst1eO`keM~mmm=W(>c zIu%rD>7o2#7Q56g5dHjH#BI1nuH6{tS45<}X1iNG3q24`#LUMH_scCIp9SW7qCN_5 zgN(kFE&Rmwg`n;{NR``SW8*@GYr90X_Be?zImqbSC+U=64(dFbcmz_#Hyk9tQO7!w6M1n z(kxk%)%m&guzL^XIPkEm)Hd1)P&{YS!I}@@iI%Lfk*Elxli3)m>-Sdtp2Z#rV)pqS zQO3PoT{^;$Fqz=_M(jz9FOs~QJlG#e{5;k8gv|`)C`yTjSg_jSkhjRenhXyHYnuZm zNs-!!sc7 z;Rzt)UN}xR&z;Cm82j?~h}|s=Qw|l}@%&wAjz1Jy=uoUAr`y4&hzGfN;@Tuqfd5_k6{F4%rDSjTB1!Kz&|X!^3Z<#`YF)uYWeA`OLIH~o$g?=R8tL(1Ez&9`t#q^ zxq*tA>CZTDZX~-)yZ)kLHa13<_oE<;R{WcO8!;Zxj7oMF{XhQE=@?B4!z*9?Z%9lT z<11v!l>ExZtsK+&dWRPBo=BqD|F=r~zyDN%{_OuJD%4>u*t>XbL!xK^3ULfbZ_ZI> z7{pG(?QeGSFL(6QX#XOEK2XjqfxiNTivlafRYT8t%m*x!bpedy)%=zi!IRiC0WyK| z7ZL67A-qrsbD)8N!AcK_-3!+xZ`Wy;Ih!XT8-;8XF&@X`)mlvGQ;`}FpgbZ2I>=W~ zP{s~G>}E;|Spo!`IDncagcCHHQ{K&{dW#sE!f7rO!u{%-J8H9q66(9}5gYo-iWo+xp+(YkUesyYA=#tl%{Q2dwfH4z{R zt++!x4X54KqB@@%zuic)ya9b7x}94m$?ApY((PhWLdr_(IvocJKoBBWtvFj^0TdC( zCo#Svhr2sF;d>-D)vuqxbY$o+*S|0(Wq9ls>@cmYMsT2~*wpIc{xUlGtli~hd>Yes zXtL4~d~)|h(NL&*6fubnTz;p?=i zTfjdA@J?;1&K)Pc6cm`JC{Gki*x4J3hvQI(TcFNjEqFQa*a@(Hw#q+k^x5AY)FDwI zO9U>Cj9}q&7?J`q2klpbVq2$kpZeK4!R}Dl@3+8j%#*kAs0ApBhi7@zB_yDaz^l<4 zI*$B!Wdi7sSDvH{Zh)c!>ktr15h9g5IHp5W-E2;W+B!h_Y516@ttU}-U4nPFfS{Ee z!J8G9P^PW%nN=XrD_$u(3{e~Y8?*H-B_}PcNXIDvro%S}$dpypfYeBa=Q_q~oyNg) z{nytKr?jXieBJYneVV6wGx8Ja$PMULq?iunPl55@=uZrN(n|M8hCM=jA8-&pO+g|-Rmhs^9M7HcW128NtK*Sz+GW#S~ z9`7(Ue@v(Y)BTrPn!wO4i-An#nk!`q_=2lqi0p)Z^~(XMu-|?DhP~ZJ^`}aP^PM&c zMHpBgiEG_av3m~jK6LB^B%emHOHa(pfpk&fA%8Ut$}^rsQZy0t-C?iCC!G_(jSzuh z6L6dz`JN9`at&yU9ZhCq9M}-yhSzbP6uS0+1-+8YfC&VQNW@t6Xy>Vq`1@t+_A8F( z3~5f^7;q9Tp(CfBZ37gJ4*>s001Sw-(UB;;>{B6-)W5$ji$B#!TgIn8rWGrrTfM8f zr3d_)dE}u-K=x&I8mWFj>>%#fDO|l}S%p>Z3vBIRCH4l|kCQV3w)6Fbc#>)AA{1yM zL))@3eGRZtYV;@zWcwU%+xMJp|#XA|r3X4_u%mO+5m|<)7CoDg10x^Ejc` zHz%(TxJ$Y|Kh|LttQQ8t!`cZ0V~?a%rx8yjsLStaQDQ2TMB(Egxi>H@pX z=yrChr{UDCto5knc2(b5j4w3Yi@BKQE*sQ4=Sh6bc$Bx_LXqz};I0PQng+yMmH zdtbFlm(qakbixbo3LA*|ME~-*{|tKrDDerA;FG2x#@(;w$ncscdu7hl{?ci3BVnwL z;FkK^8vwaQes?_$nGtfF0l=}-+ob5BF7TVbXu4=*JSB!@1EcFMpJ2Bw!gqi|H!|<6 z;MvgKWl8O9VPYOxI|=lLcbOK0YGJGUNfnM^1ju;T7&gKHq~PAG6|yn!3~311s{;`2 z;wC=Cs4Y>gH}!gotYx@wCi@5qm90IXq~k-E4XQHa#{Xtiyi}Y&9hXPIYs+YAr%N}J zb*l^$oDt$ce`+@3{8l@(*(LQC5F6W-9RVQM*Zt8KK&AT8eSUN91iT!$6h z6a3s+CED?_gP;Bwe)SI66&O#Noop9!n0e8j+h@-LpFhDMVfDN=TOmnYG^x&$x+8(2 z8L>29)WqamSHl^4Bwd4K9ySTKlt>y69sSM*?3zT4u>V=C1O*TYCR5U%G)Cko)UxrwCxG{DdGz7cSj`2;9xBe zP;}5J$9=bimrzjgS;KaCkbw4xM<3fhNb_1;jOY7hslRRNH?Y5k*)8&JkqrDUBb9pmlNk zaIb3x06Tp&nm3x4mFGa!gs8E>_KliQT+abF;2zn~CC%~_D)!>u*Abt8wd?qMz#~qC z8#G6JZ2e11RhWbo(63Y1Q)E}cc`9KO%P?#3QmtNlB%s_`B0=KezJFn(k@1nIKxrDFZM}>&?Nd&HnP&F z_b2E3fo;zB)}i;Zp~G5vpydjI!o6>$f8 zqtw#9YiCop+4PCE2H)W}lCRZq7W;U;-sW#7Gr32BSqM5~#~E#Elr|sQ4oK^CpCR_Y z_mvXXb5#QWbnk6Xzn4wov|GGe#qbxC$yy9@hrb=ECADdJw$I~5gLT*&gOZ)6`XW0| z4L#dJt!inU(_?a+(|j|IvoGi`XR^O@Mq8|wpWO1HUOjDbsxU{f$MMP4p2r8tZ9_2Y zhqS!cbw;)Cb3v0$0S!GdQa66Mp?sTX$xNHl+tCV*0WG#S$al!+roBN}mb+_q)`Tw| z^tU;we9l4unp%{3NQI+y!$TxzUo^|V*P{3xd^#_zh~BAY32{o|P?vgd{#ci(v%Mu?|xo3YB9O4T|9F!AT6$ z0|DD1>#@<8bDqC6m+2^>6gRTnI?k8US^b;km(wleK`y-0dme2L>1e9OwMK5^wik!%A_&3ZQ!vvVV&&AByj&YjIsdNbPG+7Ka2mEWlOWfkEnhzHjkm<2GYcpT5k9a-x$qU zsqO?YHS$z1FoLH9t4Z1`8Bl4oE~g=pD+dBUiWnk!kIredO3`Vqon3;_Z=>`~#8U#r zv60-p(Oe#17#_d79|o77mNuAlpSuQk^H?7OC}13yd1~c(lPXS^K%$WbLnDrC=)e7E z#Kbl|mbS~JYtQ_bneRN#)3TtGw^CD`Q0dEnZu+04+XvUp2qB{MUS;XL{7y4l^IB@; z1daLAkXjy``9PFI!B|xKoS1E$HLjMmaJ7!8u86zuYF#tWeZwX8>G$ZHp%w|SZuz}m z$up88l-?b2nY7ILj1-9)9&()sw2Mt!?8>qm5XrLbydXQNS(F_^K3? z(e&<~uob~1UWH{Fyt7#DHd)PS4n^zeGN^rlQ@6zHG`+R=mx&;=5pA<{ZKl+Aus4+{ zN2p3=4V~4$F+yw|WkR>EhSQo{ij)Zt+@`uJOeW64v#f4{%4dCM#!slGE#{7|#75y= z#MbzGe6sgrZM_a{#(%!Pw#z};-?4*N-S|d9WFm})d}P*2&bwjdv*5yUx5~dqKfZ>R zVK-P-ZI+L6=+=i}?#}t~;dHpd*qyLy*e60O=r!^k9-PK=EpZ|7=TH`vBqN7~G?S53 zT@x^9gW;f3ld$e&pFzx3n@jZ7{~}udw{@IQ1QzJ<&q)65K6Te^@>{bm)J%H`tC5@& z;}lBJ4FT5#&_CbL(=~eO`ALV*^UZcL7uN@ko|H?MTiUs+3X{ltp`IwT>;c@CPTu_; z2!boT%cL%KDnTN9TWr~nTkpLay~OM| zotP{kJ{oi#?XJ{0-J1}?1x*Zotx0;$^|Sm3kJDhyQfEwP{HC<=POdSpQq<*w9`6iN{45W^mA ze(8YjqnqbNLF`LHY9b^xb-&rZboV@R?o&1=sEn*5hfE*_AB9}=-yN&|Sv125z=9n{ za5OWq%B@8&+p6vq3gzrQeSvfO(4nZj;6t>$ps!=}A+Pz%JX5{(Y|iSrOd?SZOnakC zux1VB(mzfasJpUx-stEXt6gdu*twLVLet6TfS<{8YKZ2S9L!<@GW21OJN8zyUpt8X z)@iZtRll^`eD$d#L5Kqe+jCRpdJ2=BNi-d}X^FPNrmgSh4;AGdA~PH_J-ny4A`|e} zY_z|o5c1*vOb2iZ!p|!D^}h9q_r9<>X|;RaUs^u_F9+{>+u}|m>4=s{&U12@%E#nn z%T42yvMkmyuqK?R{9=tk?S<(8E4hCfMMsm>oI!uIMAR2U-0>8Y*+8iLL;l3&*_c;O zM>FbqX_iH}dJHE7f^Z$rGT61~!-BP0r-zTvp^q;sV*dU5b}RG*gY89jV#%4(xgodP za0BKw0oDq&VU?L9(1>TV@}4Kn82O(L2;mFZ1FE(ZYPXt~Ki*k8&L`@7 zN$WTtb*d%x31H1%SI>QPl&bR@Bg(R>N~Rh}#}>*{d%l*h+!uV|u-5Y`YXRYMoKy^N zT7$}+Qv&1dcaWHO+U>C)CPX3_^}K*HhH~j;HWbB#{NN@*siW^&spBV-3XAX2+Q>l5 z2*cj7@sjhS^vDTWM)teLF^!IKiv!#qy9yJr_%>+PzvpJ&53=E(fIh2vC2VPFigwoNMEsx_JQ5f0|!$sZSfb^Gsh9W%Cst&*9u zc!??C?Y597uvI}S&|WX!Zm}H99iVbF@_+9y^RYBst}>63s`K{s?hYC2=qoV~wO%7K zIU6W2Sshm}nZG(Q0pIJ}_sWpBi{i~{P}Pm_O$M9#l(huGi7a=t^#q|@{1(C4%$8k5 zv=_=(MoHt*4#IOx2-Qp(L`Nm(s=D!1{vuE!;<%v{h=zW~YCtYz*E}UM1wl=#6UEFE z@!aAxn=0x2#dTmZ{rbDz!Jj?s{`?GnUMWCl{DR5X>c#^{3#I*(HPpv53de63>f5^3 z=)1&0+X_BI)x!-3H&g@3L_GuFiF;<|UEp8aMpAq?&lrnt`+U3$YaN)Afg73{Xtfp5 zA!2?Jd04DtB6T;;WYT@ZBem7T)46JcAA-(%>MCl!lqA65X6EmHc@f@<_t8}f^SXGeOM2)-@MtW9?bej3%Xo2P<#?f}r zI;($~a)^fN>#D2c-V2vyq2TR!FWBonxnAxSiY7h%zM0eKeuutZ?K3^R<-!72npGTK z{bdPd910Zq)V8#u1OD19mOf;IUz6(jX$s_dwwJbZAWT+Tr$LLl+%|IFFL;uxGl}<(b zS@|&hwS#~tA-x!jw`YjLoXspty34;E<+lu+KVe1a zp=lRvm)CjDMm`%h{Mxf315?+$`j^wu<@4~J@xr#GvUc1o%KXp+h1m4rMj9DK`?3f1 zTb!{HBK=G?1b4*O2`Z_W391>SLQ@&US_`0AH=Om}A&!bX36fSz1s#`}!7QuA)$-kI zHH@+J22(=N(AmE(QwexI22RV-w=p1-#3e3)0rAJx^3@(ssao)SiaY-&86D7)tx*fK z$0P`}z)&X6qs;5*9^?uytK7Jd4Nmc@^GOlP)`UHWzrIQqBT?7=n0MZ9u8g8IRaXW( ze@@+;cmLw`F3VvE@&1SY!v~#ph>B&F^#F zlUrE^#E({o&6ZHhF5T-7E3!L3Iew}An5fb@tzsUkz>^pJJC;E%)*b%UB{N66_f;Q= z`Pc2%HsRaw0I+CwZmcqkxZ@`s%OOMt!%IKVnozk6qu3iBQQyy~+4_}^({+i-N*PBuG&3-Hgg}~~I;fP1{_3Dofv}>J-lQ+Y zoLk)NQ>k+=i%Y zV>F@^MYLZ#Bsuhuq{j|yXhUlAZ@9)d9cL!2S7NdP$K4c~$p}JX$OMMCq0kZ;1~iRC z6_w>;o;q@2RC&{4Fig)?dDCF;P;aTRVbb4(MqTX-1&@oe4)-D{rfM1{KBSxpb4uJe zeOG&8GuHh?OE$salj(MaDG!FXY*PJ1>a5^D5KjN?JTlU88UvQ|BiE#TsVt(y^U@Sm zkz-P{hjhZG!BD4pD#2tlZ!eT-EI*h@Bd-gRkmW4b8R7SH_yJ!3iyYjM;huYlskLXt zpo0;&nOBniM6=0$z#{tLacKE!-@8tc6rK#SivdnhipuO`NWAo)hD&G;3rF3y)p2Xb zfwlnc3%;Ddv1vIQ+YG`|TcTD@Gwrw?XL=1i)N}R5w@p%#qF_Ig0;{>US-=?X$;A+n z;YU%BB+M|tG?jrFl*=K5cyL1pSB7_MUh&Htw+dk#u%rJXw%{V4z4Pm}_VM>#^L`Dt z;vJu?$uZ{*<@DGZ?2JP*4=vCbTuJuUCS5VmyX?TxMHRqWzgL10QJWN2{$lij_}YyA z*QLI@c9Ppxwu#INPQdH1*25vrr6CH3uBwhMHZ#>$d@&z0-G62Tjl+=7l>A&nozH_V z2jHtT#~4EJZmscp4(oN38W#uj-CX6I&y_c;_&U-ljDRNkCqP{1Q=)l%*bH@ znLlW;Ft4yMF<;RlBg^w<%3xuBVO02}?_z>uf|Ihgp1yOHal~F%P+EFenqFO1wd9?C zs90yK*^JlfIV9>j^a;&2k#HQ7T8vYJ?F7O7+ilv;3+pWcLFm_KC|~~Lv37tLVp|^3 z)E`62Q0&YulmAc?8o}G;X%Gy5kxJP;7x``X28O=x4xplKOlTX$=PD)kRfEPp0L>+d z00Ds3uNBC!uLJYJc%YNTx-`zt&YuA2NB>5m5j*BXkygpK-QC?FXa0#UpiJ|DzZoKS z`>V}A{PjdOvlfNVl}VLXUmTM{Z$rUF14A9^3A%s>%eu)|$XHvj%C-CG_Y4*@3ikN< zFr}`&8eF$WQ(bL+#&fx&B7=d6vvu}LYivrGe>~HyC{BuPa?&Os09>6`X=E82Mj6BC@{l-10s5+X1ezv zm@Z+~V?sTujO>n<*S>5*WnPn8zo`wID6BoLN{h#yxt%Y~$~|}bB?C(<@z3X&&c9e# zF9!6u=$~6;CVJy;R4r0aBp7!2#f1ic+UiG~jezV&#L3^YHC7uy9g%O941X3V=V+)3 z?a&g5`+#R+i%J_6RYgv^I4=84U)Vy+ds@#Y-+_W#dJA1pSv9gcj+huy>DV!lNf6uvm#Kyz(K1O1U;MCnVjT-f0yH6rX-k$#zK( zivo*X3FPsKx7&@?TY&fIhri~Z;>IH?^JTwUW_;7m)6f9HB@>4%4283k@ z*rkI3LPgKV7NBuXT`4hTx;#d2WVMNa^HDiZ&=4R4POG_5o2aN{ptlL;~Hwe zfmv%&{ntRrl;LS^jLHX}ChiNOTQTctPM|Tm<$W{^9w82cDlZ; zCp#B>_YN^6`AQJ#vQ=Nxl075+>eZ{eR6a{O^&%IlQ2rAX|AX;ts><5edWLrr&vy%R z%r9+L_nfwTDp>!qe}U~hO^Wh?ZdrQJX7p^x3*>n~^VOJf$4z?n()G5w;*F45aRs>Z zbar-@8K|ssxkE&t1MvPRCa!~gQ^>_rmK^BqKv6QEK0bm-P3$^H-|M8?eCDYWu)h3h z?$$a`ZI#zSnhVZca_A)+^r}C4nkdA#Mn*>V3D^;e@7|>(1I62-C@W$8c`7FxG1ef# z9$qlg9kN^f00mIthkVd~6Ja(v}^F=Md-tgiq$!cg}+o0|Bi^1@dx)1IUm05+Tk zzB}!&1n`Z&fjX6i5C0=jAEX9`_BE0MikMpi%b{O*@3mWgt-f2Bj7#W6n!|x%#`bqVVBkW5Xz9QvJ%u9r@ zuv$;M`ws=YqUW_;dioa*Xv zR#_qz?NVO$d&>y%55Htb1VN*;WBiZ6U_u9iSY9-$sA}W(fFQ+ZuCj06X1;kg2(!#~ z4Ah^y;kQ_rnZGqSZartlokuCX18Nxs1#aaqkLm84L}M-F@u)z3GI}QgV-Gls%uXhR zI~iI&2}IYw@^ZF`dK*o`{?(vI>2{bylx{U;+^WQM5gX*TX&{-4h4YUy{po!73*$f) zVa+3BnNkRB917JgR#?R!Deai;MT@&zpjVj4Y^-;aqvX-awwd+VD><;NdvVmb$F4C% zWRXhsgsK_5G6^N42ND_acr04TtP7aCLuV02cTzsLb4%ZKrzw;>?@3pQ5^UkirEgt} zWB6~9E@GGYeYdkh`+(XyO#?t@*Z{aN_#7(~&x2PJoUE7ha!n585W0ZF_FMH!nR(xx z+(hMIIOeAbl1Q;zKw+|W9@s19q$-H&;r^ff>68+F{rXCi%j5Npe^G^Jo|+tw=&J%2 z2()JxRI==qmS6lFj>=;`b8_6ok z&Y-}OGWM}5$1|sUa}86|Y_fy1V*rDc)y5sozq5^Kx0U9q%8Z)eA~P-*40F0dSZ)*M znv;-_kmFoCW1do9_l%jrS}M*f1wb`f$mKRSFSt=Tg{XnQkP}u=lfM|IUkr0QcwgSk zMR{E|5UX!PK%r)d(W*MvY2_I84W%>3D;2SAtj$gN&pV?$7yB3EG8P)xW<(L>^WYs` z<4WuBTsu+8a_Z!bs7umYO>Lqq&@GIdd4AZdcW!g+3%bKVjmx%+RK}o0mgO=Qa~=&A z6NMZW`EXO&e$dX=)@6u$E$p*xtmFL9-tIAQ5DTgkja`h+W8%Wr2z{coR#|tKMEd~TS zqx7c3d4_~n|Gf4jU{m1yt8JE}_U>j8K_2JQ;l91iyM+(I zHFHwo8ykvlmL=;)LXyl#Y?s6L#`s`}2^@v(SP)0v&QO z_-pCe*Kq$lEvzNsu_tQR+DyD{jhaVp$f203D}`^u_eFYi)K+c|=@xAq%6VxdNU^C~ zI-AK7KI&|ZUI%h?pp)d){QosFIRj0DhFo*8q8V?&9( z1n%w4%}t!aoO|hAx3ZIa-B!9Mz1a*PP#P40ACl%2a#9m;KV4VV#HGbM{63=LrJu5M zM$7Dp$pS;&qR8U^xt5seV0YweTs3vUTC5pys0n$=v4Xu|;2++I64jSCZE`}!-bVE8 z$*6I!ez3S#a~gG+0vL$NYMc53sBVtNrgoa8->qtGY;p~!rb&J%r?y4ymZQ3w-#3n;4@Q52`jaW zt99K}>J2$W4N9uU8v4VDrSun5rrgqQz0uT9%ND-?NO|4x!B+2TWwn-W!A{_Z*Q19> zNZLwgVW2m4X&2X}2~Ild3#Cv6mY3$N`t`$3&WNbhJc`XUN|V`uqosZrhDEcAHCM&n zWr_fwCcPn}Mk~`#!D;M=RWACvrD`C3b!}eOqW5)H;Ndx@ll|aUf9`p3@|(ifJC05* zJR9i6@w?fV@s0Pc1G^rM3GEJl9w$`378JuJNim$^@M&3xRIF%}R~ZOL&+M)c-m^!X>`>VboPC z&E@ygHqwuH6RE$C&;8akh|E;PC~#0~;%!0ge@N}uNBQ=7d|ZO!&ag+g@|orPVe+`) zj(BXH#-6T^+od*Dr2kruJ$P(*k@>Aav_b5oDjiyEmFY`MpL7{MH-my&@6dSM!y%_M zHHofpTgqoxF&8(v&J*9nO~QhdBCwSvtavL+74){)vPitCiT?Dq&KzM;V99%lOUBI> z|1S&Rtpo0)Slt4Xo)+tm_C3WS=A55D!&Vp2!p0V)p2bT|J~!h1;1?t}Nf%FI^D#d9 zT8Zw5D17E_7kW9Wx)X zf7#jC@W~-j#f{~`bl{h%3eD;$WUWX3j9neYURxZfJyHl0%qBtUDmk8gn{6o(&x=v; z^)64M1OEpb3<|p;-H2#*hw*0=OF_wh-h}1JILPp@?`s;N>Ze#=(3yy%6isaXyR=Fj z*_Q-M50+46Cwpli8f^?TtEq={&p0sP6?HP*kpYJeAuOJF0FxvmVXl z$;$^1rn8c@4rlYmI2QJA0pAtFwXDx@9S+T^OAG>v1k~Gvv+Ps%l*zuSY;@vq25@mT*{84S!Y0T0M)7^7{vWxg~V~EB4J3G2=dt5Ya;veCYQ?P`BLih_3 zer3Q_B}oPpsM?=*&l<1G^;Z*d#{8h)V&AHx!-HjEj|v6#8b0MdP1vRJu)qj+c?pp zP)p`gm`Z{1;v%`2^LsVkUKZWLvtR;T+~L#36pI=P8`P<^6VwqaP9_t7_9=c!g@`{L z7C(58Z@}c(YIa^byIgn}iVJKh=G0*WZ3`UONnL@!p@jDHT6@@fcG1%Iq|cX0dYep3 zb;o;wp=$p)I8wrZ&v^QTdcT-LDxP(^;OZ!cYSHN(+eS>Abl8?URg1^@=$#``Ol2`a zyyG|J74CAUQbSaKUPkE*&j#n-F|jYx6bAnv&b~6L>h4=tkQ8YFsZFCah;(-&Al-s= z_a+6TyAeq#>F(~5M(J*(yYVj0xp$oN`u;!NPwLoXZ1*qLT5~@0iCL)D9>t%La;;ZtQj_jF~zjiS}*k++8nzbB_5-vzrU#i@Y zpgvF~7@NlAT~wrb1Y~ts@ivQ3hhw?x(%D~}?Oq${fC++isK`Gb7Yx?+4>)@(+lC-% zS0)&eT_h=}j>C7I?c0-fhb4A&SWK%9Or52arpn*^OdH+QZJEsxv8;Az|6IUNkk8Oz zdtlLzu_w!OQ*2KZg?NjtV)8DkheTnU<=WJjpAutDfTx3lE2(QG_T61lk~Z@11uU6? zjvdrE8hC1--7eEs2+!Y;8x7lnp5FD)WPx!MUE>^Fe2Q_@O}o5HOU1#FNZul#2gl3) z*Tsl{%_uU@+h%Zr)Y&}zon&-S7l9^aT}||V?W$uFaA8oJ!>G{$)R?R^(t|iHO-lcB z74f#lM7yv5+#kV)Lio6$NECl5u2+0Kt%wb?X-M~FRKckby%Ls2jbV)@*}JNeiKo|* zYXEoaKt)fG#n zn(IoM#?~$PMuqo z4GjsQWO~=gbQ&FnevN{H;<>))0m@=NE#O?4-Ur;Y=dBPw6d(G$O9-x;ZS6hN9Li^m z&BQ$4)GkA*x9%MF{d?1AhKb)JjKviCEw0C0rU`ovad!XmHG5+Do=-84>uS4Q2`KKs#di5e(`V z?1{wNN^N2y(DA6_@#s`9uRl~PX)A6vzM;y;_;(*M6ej2k4_UConN*f`RQ4SI(DgH% zKYMc2x$dpW3>t5Knf#!sxp@qTahHO8y0g=$_$bu`xA$Ytrj_9lUBp2FoAaBAm7_$@ z*;37FX<+26;R>FdQFjQ=vD1DZ=w3grbc73IS|6ed+?4@u|JZrTBj!U zT(DbClLET+m%(HXrf*lkYxot5TwezE&Xo_Im!!kiFxy;b{crPU7E6aPT{`Mt`_MLe z212P!_Vj5KMi8J}qp(XVv)bwZTsry=q*0{HWPTy$bd)xC`oXw8;irU=B4WnS4L7r- z>Wm5B%1Xh}3y;1iGW$mm3&5kVvc$tj^eHqfI46iV6CYtDuZ%$YQn#qoLV+c-JYKk~N`<5pMlH>A|P1-^{ z0400?`Be(((9lp%%pE{I#jNW$Z~^KoAxr3hq=_V_Zpad$r=yb!kZV?Z;CKJh4h`ik z{Z9Z(wP-sGvGtH2&a1?o1Gu$6ki^Th<`kIFBX?Z@3C!W(IAJN}syQ(btPgb3fqfd}w=$em0NZ%1Y551(#qt+BFjzqZ({})UZ7* zq@<)nb4}orHhD=Y`uCYP`VN^A8LG|}ia050d7==!Q5T_E{B1>yF2{A#q?vk|EN>;m z3;a|6f9-K;V_1pU=|02j*|EwSRzSYf!BmV=AwW z6Wm*PTo0ltfS_&w&wKfqhr8c}rR*I|EiJkO0|Rh!2x-t&TyK}of0fLw z)KSZ?&{yCwRh2Zz;D2Wl6%@vGum$fYQS1`r>=(O^vlyksnYv%6$z#-Ex3~MNU?;Lo;`ekLFK^>MxUd!MhNUF|<)s<@I>zF# z-?Qg?rHQvUV1k*Vv1YaTq5hH3579%Yb)&2r351RW=ru(#Mzq~TETYXWsa?9Ng_s^S z^liK4zf^s9>54}vAM?LvrvPU^`}Eii2U;}ngB}M9M=D$k9naNOo*cfq15rSSddOxZ zKC@IdRI3G>)oJO4=(0;Tp9sU}5-g&dMU=tx4|)>0Rv?YbXa zWMyX;{Y<$mQNM|Ow)I!Z3~I17xEMIzF39}0j74F#(<(EvXPxM?Nt>#{S;Js!)0jz) zP3f5$!)!-W&&XO42){?2S#3nkE|5wvAFN^=mVME4+sSdRjR|<1YS?K=P(tC{Dt${E z8n|gn)krqeJx+rA1KXT+&48D#jJeEV{0J91`7MCe*aWJ$B)s|St|&N{WC)B%koO1P z=Yag^F(iBS*_H5R*}UzeSP=JR5sNw}WZR}pL+F)1&}ZzW?#^vG=6ooG8n-P!mgyTl zdC4LYcFUVJ(un+bDV5QREou5R@^g6Om<7*KO!QN#s(VQ*AiRA}GP!+-k|47d%1X5w zg0;BNHfntLI5>@nKzn}%L|bxY?COIHI+^vgs4fno9|BS8aSp+7SU%>Kck**GZ$qX7 zwIv?%!TCb;sP&(g+XL*gK;3!-r&j`YsANBrL14%A0Y)dzt;*CPgWtam3O{^;Xi=9}7W#0Ci$r8>GcH+wj|#X_NUnZV0u zNK7HF?#RnbIXKmbHZuNa;py`rOxh0{`Lt5?sse^>oZYn!ZmTpZ6NeBhf@^Y~+UjsP zF*mp~$9Q(VPEfw@Cn~mSVmSMXsT$DT{h1d{ln`|a(ifF|mk83(nJ6S52er?76!C3A zd_+=))xr5PW8}d)W8E_BLolPi2xv*XzzX{T6s8*> z?16c%+6wpoJkd}r1RG6ai}N+Pny1AAyBz1@Qry$9F-4 zy{RwKOj$Ji{Y3rDI@O!^rW?}s4e2#5hJh5=!XFf|RU z$^YgzJpNr!fC}KkB9s6@licYO8s{{Mdr@Q^r(GS>6Dy7qNv}sY#5Bm9a1+xW470p? z-N5M^v|r{YA}R8kT*xwO$pz`w5qY8Do1O!b3K)Vgw*n0+pZ_2J+dV@#miZnsV?=a_ zC3&-+>qh(zd-Sh2qP3quWgvHURqDBKk;}tTcbXN-t4x0-r;ZrCj>!VCKFz}r9ObPl zzD$0sbl*qAgz|v7iVw^MdT+#fBFR_yHi{?vS7YCK8)ARt(s8YE|5*`6Zpy_${JpRu zd-Q-br6#o~ZU4{`o^rcnVV*%m87~gk&;lDvOXM)PJGcip6TgVb&v>xu zPlD=fbe2FDU>@EmlnD`wQz#W*~@-YdhqpU!tzTrjSBN{?(z9gEj9$ zm!H(~`FUO;>uu1*;lGAmqj*S;-#4p!9R_H=Q=Y2^7+vr0>(y`s=+`1!pXJJMRd=X8 zC!yS}zPFQH*hcw^MNuSCU3xIdu_f6a1@mUPEgWXj5`TeD7DtGG!kP5-`(T&L22Sf% z*M(hU*3=>X`X6FEZf7fKiB9SbD=d#v?IiYNq@YAAX}mr(BcsaFnIIR@R2#i-I=5iH z?1x(_JG*Zx)Z$k)SGP*sBUR5Xka5n3jQRK5cP0Wmx+!05jn2|qR5TbNV_{)1r2~Gg zlLljHL0p^PhjDJF=JjT7fth&S#s_n+2R@pY3N$J_^7^;yoMUPDK8@V+uMJ?^C}5Cr z&ww~Hd#;Jt&3FV`My&A?0abH~UnJ}PlN}K|qV|@^ISzTFd{oGd=9-=Nvz)ZEkg~bW zsDIYM-f-+xFfYYZH5U#-O$ecl7zX|7D^42&K2Ng^Z^y=43KXTc>a}-NJp6G>q5@tm z#lY?tHd@a2JWN<|d3s&}{zcN--tMj0CAtAI%M$eQz@;a0BTInXJ-ZQ<9_ z)cv}G@5HaIK~83Mshcx}1-ni*&eufoA8JafC`HU}it)mr0wOk11D?p(h?@aQ-Ew_N z?oIl*^T=j0A0vyHgv$+lq-TiHb@fzraal}mRVzy=UG8<&I$uXW`>0-t5^BYiNra$K za=PX<3`fQf{H9&&81Y()Wwt~3=dx(S6(`N|4rKO&`XaXq>+JlX4v$*4UZd1jjq{_$ z-HvK&JpH)Ps+SnS7Q8S_WwaT_5F;pB8l8*IAB~FMO1`498~al9^pujbK-bkrEnxbYevmx}U|Gg~%vRLJYyTxUrO2f+ao#^@B&ObpiO9ldK{Q8T}?&9kUM+FSb? zHrlzmY2-?IQs8R5C+O7rIE@0UfFW@if}L%CXPd8VgAVm%wq3w=N;=a23-}txI5P6- zy_t-M?3d8d(uMiIm7wseHP_ns5>o1yy-GhjPg`6%7_&dQKJfi;SXO;7m?2cKu2_d) zDT!&91ghA5`z+ejK1aT;Kz7!Jkjd!xf}+ZMVwcpVl-Ua#4U>2YnSZwQWu{}~hR{6* ztQLl&X|f<1by3W7-U4#^H`Mf~#AId@dNU@4*q8X^x)kA&^nI~M6Rg=*ZGtq!tduP1 z-*5PQ6daO+*_W=acCk)ulM%fY#Dc5j2q~xts!AHHpJWYR-WK{IO8(=&y#M?J?yl@Z z^Aw7fDl8<(${iYXB`Fs!+9Q!oLln)b?l=z8N<{ z`0KcqlY0W|1^95yHG~={(7mKeK3@9caw!(GapNPahC!LGt-!4Dn?>H>8aPVFqhY1y zUiO&61drSYDbn02(c2TJiU$E3>D^=p%7$3)lnP2~oX>w@>R1SMw}$GLn>;?FqDDkNiT{B3@zp68?&M-xdsw@fTl zy}76lt+Mh+_M2jt8uHA1YteBpUiJW7i|!) zdLAve8|&KKJ88BVA})|G1~F9+bL4y5Fb>9^%>1C_hwEjrf}s>=O~29W!ciFb{zZ_{NC=BMCs z1pWQ}d$K;SE<7bL`3sjnLx$bnB!Jl+-E8Jq*i1!SqJoL^ohM%$e9y9|oJZbLr7&RZ z#dz(L;gBpokCRtIf;AY6sRX8CQaw2>AEBPY{PJg>upgYFZM^8zfCUV2P_1hTdav#g0!M16V@#M zb;7s{p43W!>SYZZoU3}rY_+Ot?f%i=lo#1xvW9P=*HeDK=f5L{ub zro!y9U(c+g{Z&C)io-R?wXJ{&#TQr;+JCPhDm|d--$*)Zs&NLDi@h(C7qi2@niw`?7 z?`7%qW_RhRnB^LSlR0H+5vTAA-CbSlcbdI*x~W*#{x#Z_l*rJ6=Vgb(wCUE+&`84X zb@ck+^HwcRdd=IO=9lwvw?%U7rA%>SxnLt^;3R<_yH$+xeogg@NfoNCtrT4{3@dc= zDckfy+jJ^o2Rs?*5JIq-i(S4_cAcPRZ{d$Sw5)8$GyQz!#Or|p>GpGdIv3ki=D*%e zaDT2|jqbxPS`*xN&0P#RHwk8ibHwM%REJzGOUUHh#j3;*o|tkJFARwhYGvPN!DDGQ zXwDjY&|;j#^SsN9s`YmNVCvVXIh(AT%&_l(o8){@ei0F;74WM$t44d)`6XFBcA)gH zQMde`H3zmy5?$ij=ykaNDj>Y!Dqf8m!a^Eh>F8#;M5@}Kp~E1eSiUz4t|}O{c&SZA zMZyrf`b#ctA%g6+=0}|@xT+y1+hb`nPm`9oeNG`-=dZ8DK3h}@XqK#m3Q~@e6+0wK zo@zPdcrM0-cj)j@jOQ>H9bh9!LFZPKcK=+GH!$;ki;G4KZ1O7G1K%Tz1dg zPspQQfPfxlUtQPvJA!d_`%QWTQ6!$zo{^WP~!HMavb=Kg- z?=)NryuVd_g4$)!q~hzGuf2hxcI+dvWe0L zJ$cfocuT$t_t!bfx{vW@bW0ftOEJabkI+NT$3gx6UkAHinvlE56_N8LZB*J9FUH7s z6fAhY(u&6AXB#|-DYEbdu2hhlrRE)+#0E${@<>&+id8S|eHxprb|X&1PQgIv?`Jv6 z2j`~JW|&wU6ljDiFvke#7*StCG@UXOv7ju27RL?h(R3FQsPwtJs08q@V=6OLS%zPb zwaRnz>rgYy3b?jMg&VXuo=DO_SfUJc>j}q7bgGnQH{oCZ!xt9XXT<(mfD)*hU&ZwmFmCi(nb!Ap|T!8S93yC>AARc@m5 z`$KqOl#Pmo^=_NF86lS-tc-HsrzUBCkJ$Bs4NjwRqcQP)5r}~L#uh^Yb@lHp^rGlh zWRaYY9Y{3zIUSW1Z;deTt%Q^t-FDUMbzwEjF%?!I+7+6~ByVMuKf{X8VZQPu_{m!q~1 zr+6}Rji*$#naU?`w{?6p<4^*pWX8(4X_STZ)U{~12sD~UURm*Dm|3YocB$$-QsJuc zg=}{;kqf;eYpSF~*Pe=M#YaLuvD0WqG5H9^kF(^;XHuBwWVA@rT2wU(?B9$$bFr}N ztUPP&83o_M2U$}f!=$s z52FMHZzD>JMq7GS`XWuEFM>Z)mDOmR@ar*A$#6)H1+e$`V@UC}py(mp1U0 z6diICBiujM^4mdWIoXVOQzcGP*Qq|_?hPwidwQt=#Y4HgV$1pqTuJM1Xlw&s5zh?o z-~_C!(O4J>qR?SBM)~;)bgH|#9rJ|NGkr2dm4hTSVS6gjP~1f43M8s*oRO|f12=zp zdyZXXpdIf z@xJcW>H5D8^KCO&Po=iG&+=42_F`kBc4?=+jtH0SsoqvM#ZkCyr8`FsKMSvO9D+cj z%#zN2l)4b*kVnpwo3MveG&2kr%Gx0QnNuygBt(_rggcx}``}dhYIo%M7=#GOL(C1c z**_I5t2FJRi_oo)SrNkV&6ak$p(#cjt4qQ+UW{3bHY`0u-gqN4~w4+M;CQbS9FlL9y&sJRNxnzQbn@T4*oprO(7>S^zBlW zr;8gU?|1Kh8-rV`H$6)2`a5B9Nl6x-XSUMi`RiB(oqAE_GMQ?GJvydnVVnbhg0K>W zh5KTCM1k+7-;+H)HIhSl|N05{{%yuo%M6aY16IXdwlcM<*JYZ0WSje?S*q~OA=jYk zM~8>SQYBj|uo}?P{3+HVRD80ll_RLLk3#TNX>>lV`A73Yz8rJa{`6;+-{A`HX+PYH zS^N;KDsDmU`@Vk~s$7TXK{j6OPn=1t>U^hw=_`4{{#hMhiw!)V zX!E_$&rb8mowATKX7Y|*rlL%~k3GMJbU*h8H_UBSAyAbvysn2Kp4#CJCymsLaQ+Zz zy=VXSO52*VK=H_M)5Q1J4!;LxhrGedF;Bk@qDsmeu+~R{17I)KA&;l>M8ocR0zr1C@)C`;Ila zyd_V3O;0cmH6MCRPa7n-BN*4iLey5QW6;K$iTpewLaxFw_t-M|$Y&@rn;-1Y6HD)h zvt$~0pZ{uXb9*Tm*Z7H(zj2;sGBx2MoN!CvpO_ZFgW5JFGxVN8FN4d37oTa-gI^*{ zC=leh>l{fY+KhAUY?^2|o3vDNNMu0xW%2dQap~OHr$(livm~N5IJ3?qsUl(DZ)TkK zZ)#*Mm#k6N{sp`Oiav+W-K6Ti^?vA!QBs?GJ}xl^Q5Gz}8sdgtvY*AwwV4ZCRX>;= zA1qyS^?TDDZdEM}P5tYDhM-c!#A}N)r1$N=6)J6;@1D%g=l|Nr+@N?sf#Fv~u~Wu$WV z3V+s$JMh4cC;TL3&1>LYVlPC`WMFTgVdo;BiD3>aqMYe?v*quWj`E4s!O!0bGLem= zB28e{@`;`B4+zp_$dZVxnUm+zt}=~Y4m=0qx@`I2t9JtT%DNuMVo5B90goZ)6~F?s z|7ZUDa5}dE)J9xR|6+%E<4V{Od6(W%3!a&{M(DADD;V*w18dCaa>L#Md7gbrkj?8p z-Y)<$RX2cVb~A^LztAW=JluX%2(cd!KT@j}AN6m4fW70#4nJh6 zA2*$0pTG#m{^5Gdgg5Td!|rit|8cPQ`M-V<9`gFdnS8qy#Gr{4Ig@1fR?f=G>I5)8 z_!t*o$$h01xLZLFhbJ)N1-6wvSdkvdw0yw%*8b1^<&Q@{LhCvIiwkEi;3BuW`WS`X z;{X#6m-o@z$qlG$*m1}gYR34LN0-|_hku7@LOl|}ex4l@6XSQ?8GyPg;y$GFFGAZM z(peom#A=a;Zn-=yo&EG8>cf~9FpV2r2W$Q%c ztv;+6%woST**NWR_36*$r<0Q(Ovy%x+V;WPk1{ZiGWk!B?2p`B>iS2qEl98S^w21X z24AeM3kEJpJT4`Bb!!oiL-Zej?5||R(d#FTOvDjkG+Sjx{hCBo@W>xnrrH=U08_-? zwis~IzBH!Ku<^Vwr zr^|ugY1rE5+Kw+QhJ(i|9!4j5+uPgBy}qOYd2gSnQ3^~UT|5=AT#I`z&w|65ryxw) z{pe%4e_Vf=KJnMPqDAck!8>XW#_7OAR-Hy+Y8nljYkdrMum`%@K3{m0u+aeTK!wL> zKJyAekB)8vov!u}bQp9QzWD>aPxnq*Vfe!8BFs}ie}38VN5LBa0#dnapf|%hX;=B< zg9wQNjF@RBKY(M_oa$>sWmCcBnDU6_DNAQ4OzJuN3-Qt{WqFAYWG`-})!~b!2 zKBdPLgsh;c`L%4&0ydshP0!P5JQFMloyP!Zsv*R8M2KG*Nqxw^@@BkwI(C|zoV*tH zD4*@=Dv)3Aw4U2B%`Q>zQIhSVKkFPnzBwIh1>s5>e7 z1~?c{|KKGHI9`Qv-;?5?+PPjkvV^v135c1A-9IsDaOfVG=ZUIFWPuG{Pd?W1%!#xga%3=(t3x4goGjX0*g=mlbdF7%XUs&Xuxs%9 zUJDxqOlWm&Zv1Q3l>&DY4(|p0hV6c^We%GpFAJm7uFu+m zmazIr&;yRln5^i&BdPl5floy3AspJ9&7x$qsNfmTcYiSxy_XOcd(s4>PZRLVbY%+1!=?vi&qN%_ zxEdqePlq_dy+MidIuOSbX#6X@?r)&C)h(P~~{h%{6 z69RAY;J8+HkyR=}gh)#%$F9v^L;wD8H1W}BDfH3>QK7OEVoLcIZY z6QQlua?_f6B=_&vjjszWC|hD8GJLzJiCE!y)mONIJ3>w9zg2s1j~AR3X@m4Wi1h6J zK|Hu9!`W&P+m3~WBujT^9-E!<9Os`%S{6yQtbynx5SvR+Z~_egO`CO5qk6=>Hyc~> z`g|JBS7%T4u+Qxosj@FSa+av++E~J!FpAKT)8NaF@NjW+Uj!j-o6IfpbTdrWP9QyO zTJ!Lmj*@;1{_gV>@vB{sZPlj{>TeqDfYN5BeUJHZ_yGfo?sHIP+2##1^CBr~>P=0r zAs+Yl=w2ES&8r%gAtRzKa(;eptfB>>ZsT8IGf*8BfUUM=?7qeY}~~& zzg@1Q-nwF)#>mBR-Dgtka(CR@Tl?9Hl!Sb$6ZRmUmX>z)zOl2JaQ`5h^!W$WUi3_C zWLPeNrN&`|gWol`)j)gRwO;$lnc!}Im*QnQ>x@}lW9QdB%h*WCJ9ew-vi1n-=RghY zEvSr3YTkM-?k&-!{$5_ri!iS ztm8Tq$2gqAe?xXUq@y-)ye~6~-(vaU&Zmz3;ldJSgYEjjA2Vb^t#l{Ury-L+fkV9C zh}jO=$XejbEp{S07%dy_xQTGy(?z!4lvJNG`xco; z`Og?pI|96@7_9FMS$lH(en-T?28Qi$51rr!M#GxOiF^2$!MnePXrw-jSleo_5SS^; z<+qN-HDDJYJVxTMfm0Fjjgt7YmxR^Db3+rTg6rAWDfTDhGuj&?;S`GPm$POf%Nlrl zBzMb*xy_u~WeYkZGy7!ExUq`HFm90?1N{V{W!ss=>|KZ8P&FNp_~;m9KnDz~rbvo@ zviCu{X#8GlZVQ-62ygb({Afza0kBT z8hhi38s%wDyASr(H!O}E0gSEs#Qs;-rl!nvf_dK<*ytPNYY!ZA$e{SMwIXNVMy`N7 zby$6lg+y_1ECG^W5!T5^ER5|rM)8MjN_$P?EiK0KBpM3q{LPp#39`_Y0$R~JMy#n# zIMT&hPdF>gBt|s9j@#8rpxh`ok_)AJ0-5s4Nrd&f!nY@wDU9D@p6Im6fr7mVF*@X) z_iR*T=m9F`6IkXy8UPPzTFFII^RgJ~y;#19(05diaK12o73q9^;J4rL6z1U0^nFC$ z6~_H*>|6969Vz*qJjZtSG6DC!^1ytBwu|>vwxUEZT^vA0)`RkNYh;8ira*T1?+(_- z5^3F3&5)%LR(t4!)4?#D5Ym*Bfc5UdzVLGq`gYCPBsdH9293{0B6=G-QnnHjS|?-X zLe8~MYd12Bxr-{i=!o@;FKAIZ2(8aB)>NbBtgi4oa#N%RgCiyk0C6zTWX;gHH z&Ro7<@fko-7SBd`6DSRT*T(!l=`~8PPt&CTqJ;nt4|>=KSH!FX+NBY;@tOM#x4(-- zG?=#xH835i5({gvko|-YQmLTV!8FA9n1n@th;fMFk3=wu5A)TGP+R`*%>aCrXS5(K z%pq}16Jg|&RYGzi#uaqFv#8+Fw0x*~*;vgT=qWRxBXFTV{>H6=3=!$Ew-Ul6hgct_ zI{7b=55u9y{0oKmh8xG&J|=ZGs~Hme>!<(sKlzar+HT>xE&s!ZWi;S38|j;Q_J4Kr zV6;Xl2y=-2qVnHEx2U#U@L4O(j>Fu4_|oOrC-6=06b8lr!`(=Zc>;fzn>}Ln7lGt) z;dsA;eR0Gu#6tKVev73v_$)bwzgF%)eChxH1(s=`T&dGjDe8V%&Oo@Dw~y1JN9UR!pfpO4+21W~WdznIvI85sDF0hS zq8sY&x%jv*a{(77ccaEfelghE1h(U?oTkKey{?UBK%qIv1xzd+CUv@V8MXi|Gk&+E zQGT*i?=T7d$D^&!HJ}7DKE5m0&A@nOd~z}&j>I+k6$TO|!)_rfd2|pU8>9Jxp{BGt=d4QI4<6O= z@pNhzH!ap7r~6y;XVteCpvg|JPvgHk>V)t}B(G{aUg+PYbPgvBons*nrVHpHpb@=( zjN|a|xxGHvnmvwv)#Ag?{>EaG-r>&*zvzjN{kaS1L{cXK$^0wSA+&ufhP;)SH5V)XjPhAaJky+$PO zNl?C}x3_IB$X<|Qgj8`6dRBB1smUI(;+j^S18>ghr4=z8p22f7Mp@D zXH3%N6XF)kK^yzWMAU_oC;=EUFqv(DEmtZf376@`}J6{(gElsX;t!isfz0ArU$Vt4WGO&P%eJ~f1eKrjq~%`_-y8J^R}I+#=j0dws`D8 zNha;;>e`eyycn9dtx;vF*pwp2We?b?$R#4~=_Zv-Wy3%ZuRuUZSgBF0c9Pq$Rdy_b z#w7}YF2m#(745})&b-)~JFK5kzAT<{3Y3=7e#dw1HTPmGvl1_9$zgitq<%3G?#MIn zkar%Py*~ZWtAtp<)$-cYD(DvNGMx%b^X_0Vg2dWCV#>VAqKX{Usj|VCEvJTTSEu1YlZyK}$<(1Be>% zlsOkXPKO?Wn(tc0BTSv45Y^v+FkU-yXi-4ig>niElHwk5mRwj{0BTg|x5GLdE`ex} z6OgDep^TQJ`N<(Q5`IoG9& z%iWSzJ)ZmhD}|j_#~mfE^Plf;=e{}JK-sBc<*u8N*N|J96@4aj%w+Ccwi&(XO(w9m zCtx8Qzp>x(k})+M2`lXrnO81)-CCW(Ao&671lEbN3*n%Z&m<&%Uk>B-`+GW(Hs*Li zK1-Ub26UsBH;C-^sIBI>^J!bEe)s2iq7r%CAJHV%v!?8i88dON^BV>P*2>-8-4$Rt z=Pr=c>3o~So$EzB0WH!O1tPyuz=)@N^hClGK}K=vD2cp&mg&IJM_!t&;i{a9O;tOvZM>gbuqi%qUl*Ym9U{UX9% zd)OCDDzr2-n0L}LGDZN*;=t9ZQ+ibKNLGhzc?jHoJy5s$b}@SgZnNQ+5By&o}Et`t{;e+bJR0nn;!HHu;TF$Q^(NVnM#mEbnf4u5V}a|MLQX z6p>G>F-^JgHkJ)WioKlTPwURhmT_sGISn`Qc>UsJ-bvsRLDetk`60=?i1lO|{I2X} zz1>nRG+|B0$M4jCi(nBx|D`_3k(W>`srDyM7yZo^V>zFjt;>c^wuoEQ-oyilw2Rz~ zRW8A-!-~@(*OUhEsCBhjCSH@}e8;KUR6CD6B3F8S;8E9VyWnk?>kwqZdSdWNDaUaH zqAZzfZV+_eUGI`>*+R_iK$7EL^AX(oTdT{bTa!t)dJX9|b>empWem`f54^-$CazvR z_quvBx}C%BFP6NoHtkzcY@3EJ8>Sv+-SI}yOHHEjVh^CDhI-?Cjdvb@Zj1ulG2;D~ z*I%t@5eQIfGu2VgUs9Fe@g^QIEO!rC@GM+=Mh{&~=zN}8G+g$!fJdM!wJ|MS+ida@ zUd{|<44Fx9K;7k#zkKBs9JbTzGT{4*k3E-><#^-9J0w5(d7NDlKmild(@Ea@;JjCx zfkLXAk~U~J2YIV2zMA16vN(MY&o3-aDk&+60BDfI@i}uQ348hMtY)QlF>!f8LBSV( zWkcF!16m}k6Yf`0c(?0!JH65Uk9c->%SCl*ca>*}hIMVED|;jZitE`LTTA3@58cH8 zUFmeUS>UYv>)c6~*jEb=ri9a9G-C_{A#QmyBmgAbBHS%-ue)A>RxvSZI&WU3 z*b47~2oGyv{uH|Vm%&>T=#kSv0Vn60umj)|!uN+rv549!P0SGvf!QS9EI5O?9zSH} zEPKzMF3|U;^OerZ^W`#j=h8c!hB3Wks{pL~-U!|kNe5X9@vxx$TX&^NLYtre*PsiwG5~oU1to81R=my3R?PSFYX4t=Y=c|e)S)8Rf43{UrnW|$)33@jd z4YYLK@&fmcJos~9!eG!#1F3*(9%`I%YMfe>^D5rRraq| z(mL5~56%sTqUl^;Y$-6H^`o?W`RpzO?QmGVl9AS1XSvnY5F3oG4^gzt}GUih@RbdaWnx-S7Hh%6;pq7_t z;>B@hu@cr#n7jS-`)5A#WmpJ(r8XVk>TW&;^A_%96`!%dRCaUjJ+Imtp$u!xZ~hKe zqm%Xia)2h!6@~A7=nUwt3Kmr|gO;EYbH82TS|0|-Hc2-~PBY}3J_T4YPtV?c*xiN4 z+N>xf8GScDL;m&nNV5-)*F|MI74O{f!T2WRxZ*C2@Y1u;?%kk+DuV)YZ`ChVt?v1e zQV_8)mJgGim0ff>jR$3D14{4G)IB{tQ({#qdw&|zeB%_DQto)B zCp+yBy|XCde6JtALj2hzt{L%Y-l0Y0c(KF9Th+LS16ToFCasMc6CJxAVB%<(+W@^I zbvgs#q518wu$Mbp`%H`8{`u*?%_knOPcs#)eC}0TX)|dd1dSp1vs=NZjj!fx0xb_H z1V$1p(>#Yiw>vasW|9Y77O<|&2^r#3&Ht#=`)S0V8!yd*a}a>~a%SDDWzM3P@sePc z*Q=y-Shw>6(tI*NH^je|rbrPzL(E40#fH}X-Rdkifc>!}2=S}A@~M z7S9dFe_e9dr0&cZ9U|_9-p|-*wGJNX1eqlB@yA^k>fH~_OK_IvkHa7mVJzh?)DUE> z&Wv!bJqKZ;CY+0F2R4aN<6g;{CA^D8mliF>l=S`EW`xT6-k*IRTPBV?Be#gB^J=BX zS^Mnzh!&nB3npu&9uS2%WR@a9tLc+aFiGSjp`0lo*GarBzq@}>hHAMtoDgrK&3Tim zQ1#jAx4^(;IgGnJg;8&FmTJ>49vD|ph1Czf$@D7YntT+FcCpLt{$|NXN9}Ui?Hggx zZZBbdveKHInwi0;1n5Zs^;b(4 z4rpO3hN=u^#>BTQ7a(+pIv3aObfvUp)mUI31s1=YKLQbl#$u`@+-$#s3y~9oDr_>) zLDKcv?P*t)_}E})C!+Aqqjl&628t$~`9L622Y*)P6B686#zhZ|f*c^2>2k*Xw zeKN(|g%dGFy@zrjXNgU6y^Idi0&3pmQ-f|ncBW}d*OkRKzD&8#VPCrIni1O3GI4J| z-*@n3PAagXPd%?AuNQdplde-aZ5iL3Zs$%n_TuaCnRfMaR@8k@zfu!J^pG@!ZM;oL_v$jf__L&DX@ib-2}&bu?vFkq-C3*4OVIu zOXON=Pq7PqG<2^kA=@L9B=fS&$vDNgnQCWVa57oK&wJZmN!05dX`=i*Lp%9S_dR>7 z!}jH3C!52v58_vgw#6)H>jwz!tsNxVcD^o>tStJ-%eU{HiKHO^9wIF@>6vZHmG0>D z^Gyw(-KYI&A|4Xn&6=1)7o@!Hi>_8Tt{3`x$Fc!O4%FK3ponwQ@_pa{8fb?JkKp4eA#lGM+VUM5d@A{gBfHDxWRmRa#o2-HCBjCjrjD zkuRD)Lu`RtIM1=5fm!6Z!jKWxbBwnmJRW*8eimpW6jpB%Y~^M`OxzFMSftLIr&32s zl$Vmdl2&d}o*&)STr##e9PxKH`<;HJU1>oinb)iPzUM-D@+fXJ=3iu}8v!|I#w}Xy zSn(t~O8>p0pFvyaF5OXYntP9Qj9NdmAz@U~e>R|`Nh3S|n0 za7l!l^0dr72q>Ci~ zlUXC@FXnpt7p%mE4VcN1_#PPCvDIl#iQbnDj7N6x<-v*NtnG78GhzAn zcRbj`i2QHtyrS+}^m%@u%n9a9x0o9_QtvCa?*&eTgl*LL4r=12lHf1K7(w08}imK9kLJ;6xz4@jcsW* z-!Bmc9!-J6t|~Wf>p6}rdjo53i9>FYFN>=)Z@4(7OA@^iQ9M$(u}`=TO|)6_vTc`o zG2+#iLY|o558IXQ4nKp2!yOEo7v;akjvtluD$Vy3ZUF%e@lSP+)+Tc@_^W5MBy7WY@28nv^ZUt1FSaIb`O)C3ICSTg*QOKkLx0)PR z30tz%59R4+UgkM(tY*i0aI7ZAF8UYvl?33hClLii|G>#m?)`Z5LA_D5p3U3o4FN<8 zzF;e1GCkQ0D-x_(i`}R9q4k7w1_rP{QhS3>4`1ytSPZ;BwwkRrBzDZnWfGr&$G;d$ zwGB6wD+0O6&bg*cA*UB$7&nvpYpTi}oKA%;NMa1BQbG?u3EQTe?~P9y7)R4I8PqdL zdXT#_ioaR~&d;0{Pf|G?{+1E@IcsjfxcqhE8L>kXGNGfaJ|AUqlf&Qwlw=#xri0f> zhYe;Q%h-$M8e3dxfK^)y5c2ZZWQc_$c@s@{%CZI?I4uw_vOBw=gmar7D%yV zZs)|TgrOf>#M}QmCW5X6mJ)^*`GjZOer31opjR(&8`XD!*nTRoiQhTan z@$#Zd#Q{|)XSNx_Ws7+pE&o)rDR*tk3YxV*bVR~**a-i9X#SU4&Ay0rUhP~eT$=_ z#0rvjw%Y4%G#Xp|HpSk_LjCxCw3=A?vZ?THME!dg@**X@6^a(pEz&a5aDy@H{8U-9 z58Z?C7_H>c)2&dCR^i>P!%4ViPPUe;=1s!w z{DbC;pW7!-pbC9VmOtRgleR-3Oed@DxD2l|boWq8D{t`7Cp5jJk`CnDFAta5 zIzPAd#_7t0h4cH+N4q!B<1%U!lrDeQ9-ln@ZD1FYQ|cu(x|=V9J3)OL+P_2E^_|6Vzsb{W#LJ7D^<%G$g%rO0X1V4ykhB^hr9*4-KBVeDOEC@J!{qUEsbSM=Rw`zC)a zh7fxKZu&zx-VDWKqc+z?Ypc^=moiK-pX54vADdiF3RS*#EIULdl{IQ-Jg0j3mX+ga zRp3huC3XTd4w_~9$FK8Fd$CEi%5^LfFYOBR27YPp2MqHh?6=xynFoGsU!nu3>T|HZ z8nw#$mo~=Zpl?v};(ZS%5Po=o7rb?UcP6~Fm)H2Z5YYc9d6=faF817n4!7Tv8C`wa zEL5fX@R;}g`+I?F!?E(O_y=IbJYx-&FAoJV*5P|TL&{&WwD)JXh0tHCpMjC>CpOuu za(u7C6<#v?0N;8c;HjB`jpjCp&=lF`Q`#L3-`we>ACCw!%_Rw^PC~>b;fYNTZuxAS z_SRrhDg@+6c-x@T?pyurw+r_UoQj7He62wezuziw$a!QU&NqZbLUUusF=CqrUiNeS zJKW}{Y?pS5*}XW!W)eQi)I&GPiPYtdA#(1lq&e_1B7Gt}eFty`IR|+3fAV z*8&=3x)=$Bq2G5kTY!wXD#c6xa$u(P>$QQBTy__3-hL?dm&l{w3D5_Aib!{P=>Nn1 zMv}^}q?E;OT#o0uiH+{*-KdY9$3IB;JxcI(NVPo)g{FOfQXG!rpF27|{;d5~Pp4A9r+OW8Ra!)kR%t^w~k|eDm!Zbno1uZ{-)- z%D&fJKSH6GA0Lxo&XtvXaCfVduZ}v}xUNk0df3fmA(>86y6c$4*+-d-oR6}W25kx| z#n7j;+aGLF{id%=aps@R(Uu1v-?=*vNH}hL%|w5_J-33PY-G9Cve|eBOV~WRGz$GS zV=4UD+j{|*o8J4bEPU~Nb(Dy5HibMQgFAR;ZqgIa3D4VuaPiq{&n+(RXBribL#VB% z^=V%mW2*NojO)L8GD!=}n}2M#LCwy$iQ?$OB=J!geFir!%PsfkU&wAUHjcQl;3?xB zXybp$tRk4aLOQlM9OFWIPt?OR>R{?^MMInSYzNeY&uCBETF(NKd*vV0r8_z?Ui4rG##kOC<#1jy(y1UtIwp=I`VXjD-^V?qsW`^+Y2e`y` zdfvrPrdM#fI$8B99^R9ot*@L#(9q!?bv7PMIp#Gzh$P|kVy=%~e2>K&J+?wIOel)GV~CVNK4NOWGv-lnxhLikVuP?k;r7W)e9$XaM zLeI6i)0Z<;xfoBqJ|}bPm@1wa=Iiu8Gc#DGkdx-FzIr}A+tWO!o^W=(L8q%|GLcnD zDX8UTr1gGF!Y_8(CUI7%o?-D$-2g9y^6MRzPVK3VXslL`ZOWO-QoPWCCdzHrUKqXk zz1!?#nFtD_^GR-hal>fHQSWvQuG0a}!(1bH39q9TNbl-Ti_h`S*U-OEV9)A!TB6JS z4noWSc){c=@_v1<>x_5t(Ytf;BPNj6=_jhr(=?d79#*K?gxGf1#lg_1%Xs_lQ~wUZ zFz51N=FAC|?L&ubW0UK)@s|?M@$NkAtL;dBOvLIJM{l0C8&=5RucX?=i+4Jkx=buz zy?3Adxg9<1AUP(udVH_*5PR0_R($DqI1pMHs$m&Bu1mt`6|%&{Te1q2pbRs4qNwr+ z&0IwjlC&DpgfAD|#&jm_oUbm5)w|kzvjtPtLlN6d1A|odE%m3OI4{Bih&ywmM_s-Z;eEr&8Ez zV}E5B+4wP;{M9J-%Z{TeXow2~DNf^)(TEuQQ8L8poe|t-e6u1MF>}R_m(uFw4xL$~ zgSgZ3JZ#w5G6GJYqy^C!#^#gnUcxBFc1!2a-U>*c)?TLk^7;~$79<^ca|52@5@WVP zWD&7x<|m9B`L&GOll%iM-H2y;Kv0J9NW5=N#W{TH(t~)U=;l`}W)u8&t12VY%on^g zW(JZ*@XsuR`$?u6-9wr0?a`>NK>N&b#(<{-4& z2y(VSzWqw!lh-cZzQF**bfUa8a1|eGS)^%v*Il)^v3M5ooKliGGVPYPZQc64sdGHC zI#=U)kZ0BC4BlH{dM!X*m_5uRV7(+5^!!Y8k|Eq5ZQt0rlaQ0$Nm%%KexIRHe7H_` zhvO3k-}L9{U3cgE-1!d~CmH6RuYRPE=6GI}cAR41s_Ghmh;8&|@%R&YL%%>U@r&5P zoD|~W;{{mZxK7OOs+FTu}-V}2AFBU(rdAi&& zd!wBbG_FmM+I~HM_vd^l@<7;d@rUT~DGNpv#D=BPg=|(fzC!}l-2p*GSV{6M* zea9iY(4uhgP=7lomRCqBb9UtRXPdvz2$9`#(^+=zR@`e}n zYQ=ZEx@{D^2hF6?JrXExmW3thQY>Q)r5|^Z!y$p*FSx`}6&H!(En?OYG*TaqBijvY zmL9$C-XJz!hV7=WG5I{OhPH6bsClUL^+C7dG4KmG1k4OY1G$gx_44SYfQ{;m@aX1( z8UD}@yYy(k&f}0hx@n&HxL*|Bx(%&X=~2k!lPxL>&2sp7h7a{Fr_S79e~kWJZ?DDU zWH(}0H2q?WvPk89Vt=!qHb>s9E;}h&&U8M(Q!XTfUXQcQ2bTdh(=M+ZXB`jyO4Da| z2KVB2FIid_{5l(TpM?ik{X$dr@%kEYVPSvQ(kNj`z2u9fyI`HyP_V2uzfB_vtx}mt zqAH5%P<%*RO&@6WdrpAo;phpc?}K^#&=27!-+g1B4`)0MGkZhAtC;Y#dIcTqJEk1aub#F+xec6Z2yL_)2WzxiUa0;In6>YC{ODoF{FJ zt0$ygO3D8Bm36&W3YqFw!X-3R4kCgOZk2NV8PZ`Rwq*P}X4(eZTclkYnr)q_Yp+;| z7Dy5l8AdbmJVi+S888rE-UlQc+^HO@S&~(rDHzy;7a;4GeAMxS^d3}bzdNPDU+7GF z(ne0aO?2FIx*H)ZeWTbo&xVAYkqi5ldUJi-8SED_w2kVys%e$gX&mte(KU3x=ENXJY_D>N=ZegC@)wl9F{-&08C)Sx_EuywvD0ZaW zmF}2VMdpbSU$gp1(4PYj$A!Y8Kl4xLNXJU*-r;Q%PTOH1YMly`(@gj1ZA_=E`}|NM zF*gxkywwOAw%*ZO2pc!c>+ex@^rs(puNS9Aa`Z#y&?1461la78qXpsAx69TpB#N{A z&2V(`b2!(^n(XS+f>=anaAl&E+_Jfk11HTRmY>+h@F7WCn3m`-zO63~ito-o>MGRp ziFNVg*nYq{_eo|MSw83uS)dTvxpllN)tHH!ZNFIhwkVM>{GC^R0kG=f-ZN$of#1~% z7>`^n`N3y@ny+`2wS2Z**j~O(Gq%Lr%_n}M!A4;t&gR)F`nVR4f_$fwwEt>8bbrmW zaRck_ZeLraC0aci>q*o4L}}jdxR*6eV!3*vE*<;n&k9U$FFcj!mrWNOg@?7+ z>eHz{wagjiyc!Ew<=>*UXpwY2^LCr&U;QqSb@iy*NMyI28y0aH#!Y3Uf9`kZo~j9L zWO~ni zc)Bq!7m)Rx}%mK;1R)IG)y_tOG*t}I}gOucK2lz zBjzvnEd@@CPA#45Iqg1|t$%N=ZbE#~o;0!LtAj=)tgbEU#)Xpm1bnD9dgiS^cdLWZ z*xOn$U?$wHVuZZae_DY$ra#l)Yu14N#lzg_XDQ;XdW{?`P7;b*`g0Pfj^89JpG8DX@`CKsMUrd+QaM;{{lHIW;dGrk5b ztmrfSQL}Ncn2R9s47j?}FUB?%_OM3cqoyE;ZLL){#TBgpB4Z-V^X##r~neCb>SKS2}EAxD+G~Vg&O-R2 z8V&n8gt@h)8kUD=e$&0M&;3(9v-d=5dt~nr?pX(HQU|DlEihaBUQ{s*-Ym;Heoizt z!X%&kB8dIwue%N?`X59aA%aW;t39$OBdou#ZB^g01Wi6fbbgFqHbzOr7q1e8PL3!= zi+q7ygwK;iQ()Ej@Q>u9*V;ZSgR^`-ckr~iJ_~p7@-SwO{||wMJf9+x+oNTw$%zxI zv7YIKy91+{l|mIIJU1a2jbh%W15QG>?)rNb(!7zf!-SqBo!(b5-&&n>P%=3CIMAb! zUg|5U6v{9rl>D*+h;>zO<Q9kx z=LB~yV+alff0wPZ&9uu)B@2aazh`|4Qw~}|y8g~A8gG`L6sWJX;X$fpKSEc5G zOe}vfmZ}*3i9mwM%1|ok6uysS^XZwnV9uq*9sHI+440xCPq$Ha7Grfd4&-XgFjq2Vh z1N%?b(bU|;8j46|cX#)bIirBnKB_QS^K8I5=(4v1k=KrcO+ZiE5jYQA`jpK!jswQlhFaj6)(8?95;Rge zRVSH%MKN{t)}0Q*&{U6me5BUm{Z;VTd#55*JP)NbG^XJ z!~FmeXF!lU-L!^h}`477qn- zxG3f2PJmpgGrUf0qbk%KU!@6bMkB6;`kIy!l|||9@V|(;L{Ya^U6b=#A+2f>hwqw! z<=-jHfDF8bd)KdlMfMc1HXY-kXST;%^{EJUsvLy>3jbpOvpI;xHq$Tih+pK+m08TJ3}x8loOH32S67c* zZ-u^&dJnX;d#ic^*OW~^1M?+8z0{~INs41?znikVCjGJdh=3#h&O17#Nc{%$zCa)B z(jSG(o!9h~2>Mq*{eC5V8RtGS3y*@G+hgyCycqPj0TC9()z9pYF1-N*lqJV3&vAT| z&{wy$f`MbDDgd{9|H!DmYAoJa##xm8(@Dj5%+j|U+CA$7U_+D)MehZGqm)salS0=0 z6a~pRk})KKC!G)na6?}U;zZeN;jF&-#2}h_WJtB*pFD}C`UQZ6G%tSx24=E)OCoI|AZHBdV%h{CMDmV;_hjSYGI4kM{~Z6hGOcHZ#1%$E zWR6^K1^oDdT9r{RRph4S5WnZU%23erc=Pj&^I2v+pwz12+EYx6PXoREJ|Oe_#=Cwj zv3|!q$T>zR`-Ie$`*p&rDho-7-$m7H!u4w`=9FLYre=%}*XyC02PWDy8g@5IEaFZU zYz&?bh+qao;{4`)*KCG(M{lpaLz}GrTENw1m*CzV*$wn}5)Wm`@gKbc+OV&7+F*ac zjI0Kny%Jl|Fmgn=&9nBV=H_Nu8oGkS2b&5I3KT(u+g#Z(GEmZm@g9IIGQ^I_#b1We z?H2GTyAQ8M2y5E1FSnd@bBBkp94qGC;<)Wp5c`nY&ojwc6>sQk}<~_hmhJ5?M?vdglj*uR9v}Hb6brMxng*W zydM*khNoJ(KF08;)!T;RCNCAQo@JgwE#9k%W|M6zOf8T+^DiT4&|D5H4BWfZPv@UB z>Z5d^>vYx?*?!o1vCNGtP2H&vJN&r`tiX~TKld@8qHhUpG0uJr)$U%U>!UQ&+zFb%Ex4%!#x5c;*R^Sb61pH{hNb|L!Vz+D!H1~??irbtxODnxD z^!yiM1OEdiG(kIYpjBrDQ`eD22hm#ra#n=Tn;Aindifwip z0D(*pJ;b-a%UE&MK7790p){P45$4Xw;jEd=2wz2AeG4kGDPO1xBz4dBd0J_{W=!lh zx4|B$TF1UT)A6HSE#-XEMaBXvMV`)8ZVPDo;~)22l9d2Y$~r8qXCeS}?LMRC4~6o` zZh`oUhYd(9gLA zxxjZtMjQ~StfFR<>H2gy%)|-A$?bWehH5aJ!|We!qXFmDZwEy+ONV&l6~VK3*lg-F z-2H+t%_!wyX5n?wU2!4jIVzMQ-Nji&NB!ssn9erMjjyw0ia zJCSiVYpMf_F9Jk>vEIxCz;<3d&b-?GbH(kDx^Lox!PqsDu@mM&E)ologdPK?C?f>Q zVZu7xn_5gafqUf5*htKT^f>;2Ju-C42+8|r}LkuMVbpg z9FxpuBpfBZLP;bLH%q0EATk)k{fMY6oza#+TfqmXmekBHY*fTs6o#cBt zM5|CUoZCiccO1@zi5PC}U+uNH($~#<;u`SZE+6X*vzx-Fz0% z=|7X486C1@$3SwwxyO$gX4OZoc%GSb(d}%Sa?G+aEl#}^;Q|E=YOhE`H%9xbcTAOx zGZih)efGVFXYA^O~T`$#dnymwD# z&^}V=+|oqWgzmDdtuYDOyeCpfIcljp{KPS`^BQGIY|_AuT7ivpUY*1=`@Gs>gq$h} zKlOcRx<9T?KWxpY43CBp8c#;-^ld9ORLoU+JMnQPe5vE>^I=liL3KzN#UB$>@4vMY zRY=*5S{{36Jj23kmWC?1eK00Y>R^qxsUj=Qqyw;xGi zE1a0wDsSS2Y4LXtr+)i0Q%HtbsW{mY(?TpFN%D`s^pEA{2J^@tXt)&a3`CFzD>Dd* z62#2)iq-+z!O~2A79;;Y6uK}q$I0r=L77tiazO zbpBL|Z@1Pc<$TER*{8SCTHX(hvg-^7IwIbom#|2eVL2dm$zlTb*UR@+j*hmD2BAzw`aY&GMHfevA++ukPra77j;T7rqf zT&eW7_MF<+W`!ZaHa+f#E0zoTT2kyQkwOUCXOXH~OjNmV7!lvjvzA%gb~eb$3Kb#V ztPrd5gG@<=>JIk;J{P~zQ1EmEcX;~6?sKN+@1ck)}X z;PA-tqF5TsahCMnF2O-+Wu0e}I=LYd+-hhkgTbWlGB1JBZ!8i=r#7)tn%BvO!tP7X6p@N*&@^4>Gks*k1-`QGH)lq# zQ5qk-OngZKBq!2LjvT#1%tW6YQyub7pxrB7$;P>xY>r{C3-B=cwMDy?60=v{__w|p z03~}Cg1n|hk{8L0%u7snY?sTSF)}skazbJE(!WKDduRy6Cvsg1c8tYCWw{S7z>6vt(3RF4d4s~@2Ko! zu;}zt?(B#HrCOQy;d@$~>u;kc z?2R?T4afh3e>z&>mzaM0fLOoR!}rH>I%`bJ0w^kuB}Ge`)X|DGR&=9yffdwn&{PGc<+Je_7`8@Q-R2cE5FWFCNG8Y8N6saXc6w3&DVL!&^b7X|$ z;ze=p@%0y;&mFf=L#-$%TqtN7h(mphmFS7TT8miNgi2*q)*)N_@zi}8ND z;-ZfuejN9O<$a}kDT^s{AlFKi1Qw*0tuVtQ`&kkD#GCDxi5U4={SVlclep~9Hl42I zT0)uVl_}!{RKpllNV#ShiK|1thZ9x^;fO!_Hd^j4aP)S&+lzBRjQT;!nS6_Mx`Iym zLYT+1ZUU|Wp|tILBQ_o-zkk_)!E)v(lb0iz-`CB`P%AAt0D;Phkabg=xksaE4bJ6 zxucp7(Q5u{j=V- z=+ znQikr&-8b3hSQxmGUvKguN`^s1I0%ZBP2v&cllFd%sbWaJ!_2?)z0P)uh8)eV4@zP02kp&q18P%V;T`iKea^3{!}h z^i_3Sm8bV={K+yX&^B)-PJYJYtLi2`vuG_6Q#bXpjCdk?jcrbMgGD(rDGnL{1O>O) z{bq4&N(&$fo>c~2)*p2IgdR{RIalZ}=Kwe3cgpzH3 z9^k@3e^iM;Qcs6vk8uZ{4!pfmh=vE6H2h=>fa7v$Dm{IJ4sv&$XooOvxWyJ)=1^}W zHgz;oKkMYXu`P1zf-(#UiI|mu}Y)csI%7|%bw_-(H?1~8D3^{ z8~qC+vq8)j*|~mpx(Me%!X(#?wMJcmR~Fr0%w%#_ZVAnNziy3C(v4p27EYf8vmYJ* zPOhOJ^+|E&*$(nDC0I1`?pCLP_u65(+UgF3#LArxL zi9YfHSfB#3dHKJ_byRmWEmvaP`W~m^-TIRl_-!7Q_2p#iQ_DFV9-o)`>r0ZaDYNQs z2#h{=GkUvKtk&Vt?qm{74d4uLcN4OEJ?i)W98%q!J!?s7@or6ugB?U1N5%3eD~RlJ z9XyIL?|gl=t=J1&_zX8aS-dOPLfgu+(a8&yG8L}OBv~YGOJBouE%#gYFL8Tvc2;Q9 zzdk3y82Bw?arzBQHqBXwACqQ)Bu7jW#lHWZZqJWhB?fma71jaIo$^~`q`@W(-_e-a z!F=Ao`tZJCvpLi=J(j!2Fgrc`iJd+~g_>{nB~_eXPCoG!d`*vZJu+c6(s`<=QyIej zb?62!1#z>8H!rujB-UkuMx*uI%;-y%FQ}ByiD(M3lb%tG2|Sif8o_rlDOCmPt9_TR zJe}zD;$;{p3GGrjrCbIdHJQ8c$Wf1_9u#v#xSq9)B62!75tOb8uUv{o^PA%Ai@L4P zzy=9gW>>)-4eM@TnjPK7LuH{CQY75%t^*Ytxj8RawWmTmn<0)Nt<0n^5_ce-t--0rz_}$zwKDm%6h#V~)2<6md&F`{ z zXcFeWA<~>;Pd{=46B_q2H`0UhPL_3{lWU*uF0oKk4QV-ZiUpM%bp3U(t6DPD4bs`xvgnqfZdk07Q>>=m2mH%7`&kxPs5PGV*+AuMg zQ=X%9Bke28WksseUvC6|W^5fzi+rbNO`h5ABrjJ!?E;2%oeZ?g914cD*5;{F%i|BBawhd_mC=#oP7>IokFtZ+) zmx0X_MW{aBa*jbQWoc%8v+kEPgr(gbLAcvd@wDc12O`_Cm=>H5`FtO9eVyy`GidB+ z(&Nn9;p*`c%+|H7wHu`|@A6_Z$oVFebga#ac$jr!W|QYhl8h6>9E=1Ha~hSqYT-TG{D@s$ATu@+B`61ozt82*%G(M77H^la(Ew<{4FCISK4&- zB72LYJIY3XSWK6dD-u(VUb5vwErB$g@i%L^&E?JgnW|w10>%B>gZ6YiVK%o1vV>{`ChL<;Xe@@k!>aLc{T$XS>F4l69B3cWJA)W`|^2v{~VE zQxi!CeMAp0NZk~JRl1YBG6<8KV=Cg6S8V#~srwPaCacXBkO>CUbwzm6#P&-GkZ+Ax zkGly5DmpekEx!9wl|F`gC(u=wU$#eW{csB(jdkBElEB)By{w*OZh!R~cOyHuTfY&WflKOh-+{Q>+!o zM4?e@AHqltYj5vBf9Z$xo%h|znJnZUcj&oMs?gT4fh9{TxcOsrbdWs?C;%lqYnmh~ zqrffCrKnB27o#E~3XN1@dBWdtG7{5!_EM37oG?G$VruO+9wVpOi21^9KzE**%qDN^ zL2vpwx92u*`N3zpo;7zu$KEK)6$w?*MhTUS$B!Dy=6zGpdOKZ3Vt!&0C*QhXvUtf4(OsERON_YV zCtk$q^j^!}PqkHc?w#3oX2Hi^_MYd4dp))bQ}B%$;%992CR$K4p@{FjjPy)TWdWI! zV&1i?yXbUOPH-*KYlE-r{F(V3`v+X7<%#GjlMPx@-3}GCM?K9N3Mz!&>3tTP#cjhs zG!zM)Obd|*ueuT?n&CuB`W`M}l+v_V?^X+ht<_hQ>-M;b2?c`ODcw>o=1>D4q z78zf@t+qF#xvsxg!s%lyf~>RHmA+tU9rH(eBT(n}5Cxb~Hm|?rj8o~&H@Qh%4tof} z5}Ba~c_f~tcu}rJZ(5G`>(VMngX=5yYP_M_?hc3T&_>~e$7I{pK7L+{^py$L?+-6g zP@n8Is$?1tThCv{x|})8;(Jhbb~Y?cUioY^|DSho3TyY4*puUGbI}Xd_@tE4<;g^h zSFWTIZ&i+sy(i2;_AA0ZGinY zs`K?+R9M^hUF_A!6)3idX|%rgqc zj~{vowN*M8w^L-2(do&0##Mz?-En!6tJnVVjSKoTVK~xDHx61r(P68pu>bgMD|r!w zKX=FS8j5}&d4-6r-QjsGL2#+zYtP>_aD$dBI5a zZfwCEmS^}4SjNqM-^f0Oj9_u>g?l8)jS{OqR#ZM&e*d;d!CvFO*8Q%%Ei9MtD$)xm zp8l%+dpH!v&DJ$CljFTd3DRg1k7EH8UDFHww?R9*5p7Os4|n7-t7 z5^H?(?7<~f<6|7%Io{X4{@i4x;#!;&blV)gO)^ECI2HsuOWoS)4{8C(W*E@ZtZPf# z6VPhQ0!W$`wq1!qFs6-&ZoMr2Z!Lg5i$-VjhLTZwS5o>8yl6VRm}=s0>nGVs=so&; z!&?>qHO}?7&XdIJe)y(bu-X5oUuWmio=SMEz*tBLIpEZO+~Xv=Ym`m)=xM3HRm-M@ z&$9yrpS=%*&oj5?rnMwj-c?IAlOCbh&No6*;fX1{k&1g{bf;BniZljm3#oe#RQj+8 zMfUEG3L{n7S^F|zK)gk@rRnar%ml|*?)CF0)bw*KTd7Jyd#ZCKC%4$B0(8$od$TLv z|IGimq8=6Hzh2;7{you6&<+YsErc{mt8?X!^1rH9xX104R*Y~%qz01dRp9fu zxm+A}3%JGH{!R6RCo3$Jn33x|*RcVMM4Sh$+GgZ+h2XUiXG~s15Eh%ivtFq_z-<2` zetM_ib5kTqff9c+YXqWY!jc9#e3^ zH(~4Xy8FjVr*`}+6l<>&#)bfiIHOb(4tosgklfO2QQWM@N+AC7) zzYSMVK4Ef9=t*=CiO>kMycZI!$0HQGJ9y5AAv93oRZ0R+5e!a4CMF(wM#UChTb*%LXT?Qv5+RzU2F#MKJmLBCtBW&O8BnKd_EJvWMKZ-wGuqL-Y=vHrYhh*;$8@p7d>-8`=meDi^ z%UJtL-GEQ)%p~axw>*F4`lao@_em!yHAs#}GAD&b>U#C z{yc%iV=c}M?%sl{3Lj{md&6gQH!A5(?@5ew!6^)#t!jnxQ^s(qpd z!Y@0|K0*k`wO$_CjmIvRr0uWbuJgbmnm+0m|GNJrzOUtZcgLezNfW{{2i`J`y#1(6 z4i>0DatXe5_4?>i;z~t~=4^5SzUF5V2i`wKr8mbW;pJ(qSCeWvc7JvbZFlHVijAPc zG=J3dK$`puCk!!Qo*9OJ-1$XuQM}2`cpRtsx_>6EC;s}X)|-^rd#;_ zgHCe!GaI^@{T1xyheDeC_K7v~j14BlAz$d4_Ia;C1t zDiYdEkxYwgq2iMBwW0EjoiAnV<22){>i@Xy2U#G3F7<-@27G^mSbZe$tl0;u>hFW= zovI{~_Udq$QZz?A8UEJe`8Vs!o7AFp-=U*)I>*Aqatx9ZL*#^EeJ!A`$B^d!^Rm3|H$a`62cs^}d1OcY;qw6Jy&dVC|F2y;0E(cD9|$UZbwF0Lx2OnHp() zts`JUcGqBq4C1X7$FbaacU#k(IX-zmfQ#sAo$eV2IfcbQI!s=K} zu@!zz<>gy0CA*N8mX;3c;Pz)h;}v0hQ32#Tq4aN2LI?QYO44< zX`Dk|Y?P0c})G zp{SG&V@<~PXwT(fN*?a3P7X&&unj+`62A>_VbK{Ksq#vVXa?0F>5SD9Fox_fiTIub z8lNG@j&I8=-?$l>AK%*Lm!9?&oaW|T41-CtHEYh8il^id2UlJHv(y`wCEq0>>yJ~s z-W*>oSC+kzG9`Z-v~Ax((2B(4p?R*W8hQSRukm{>W1EzS-n8ABvX8S*}zwszF8*h5c|o0;Q#u zNq9Rqrz#WA<4KT5e(fkF6FtBz(KukI37Y$O#o%bZ^bcCjQiBKf6h~@__Q0;&08TON zGygjc*0{-Aa{q>ziUU}yw(5!Gf5Tew1FY3x>5|6(|2=?8`=8JK|5u)i)Xz3!16_c8 z0-&?MeH6&Ofn9nzFez`IemF12FhczI{{g;0BaCV2_$662UEqTj_K(*sYrD?^guv0!wDUCBHM6|NTmexmUkQ92G?M*{l>1$GA0jq4S6^IeBU(33z^{6j?IeHS2@ zHw8q_odN8_)S^=7pZA@ji+O1gj12s9={TGzqNA^GcWndU0KP{I2Wsb=JDntTcb)3n zD;ierobSd5I+T6-YH+0w2g*#ZfbQ8eW4yS9{iiP}uq8%Fwd&(vg`q`PK;7;O&{vy$ z$EM{FFLA{46%ue_ZV1qKvP{yRo?j%ul^+4gvOe36Yg=V>B_hn%Hy~!CT)$=`+iwd% zG5rP~&w6N01A!-pSi$kg2wA!YDkq>8^aaSiPi@vAon{szBrnX*mslie>{kj%s^t*|&}pe+J43fZqMzKUIUIG_QJS zZI)28gV_?(Tn$T!^}jfjU!N-hB2CBj#@|K=A=fJDme@Aj($J})=u2JQG)ce?mKe|p z>f8nY(oHNpf5YCFXbyx8JfOUmV*BQa)<-hqFcWJ zuyTSS)ARyGee|<3K=5RJ?G*c3o&A>$bpO2z$!YX?|8lCS)3rv`WLP~*T>Z6TKCqM! zD66a_nZ7s&hzdJ&PGCUkd~PDcCNbcuH*Gbphg3-dykMs=rlO=}PfT@LMPono@PitY zm%Z^nhP7EFnH06!bu1}AhvQ!YBZTx;Rtu5Ms(LNgpzrah%XVByEr8_JNP94Dk-_zT zy8?OqFW0zJC0z3K^i}{Ml&0yG-n~X+(ppgO`k~&i7y$PN;ccKY8Bm$^=esP)FNO#@ zAVd7%3Mk~)f|tSd`cDxztk0Gp=T#S^Ri57SUF;n}cDfee%Rs~doh6_lF#~8w{Bxm6 zv(PFSz#)Zga#Ykmh&%@V)4za&2HFLAEpPb9pq5oz$t)`spp*U<0rigX4t&UcpJv57 z0TgLm8B?lWdPaJ>12DD-$IB6Rwb(&UPTB?nIm%mP(O*Q@dg39F^eVkK!>G)Np2=SB6|h2kS-Q*2&tAPqCagk zptBc9v9Kx{e}6Q0kuqStKlOnq7txye*T4nhSleg$Tm+8z_`+&-;MHmUl=+)4>Frab zE5c{pi}}kE4uxL}a=gpaeu}LB@R*g5q$TO{zaZl_Amvdvorf{GVz?)o;f95GOJhD}Hj8vzQj;n$&yG22xOG-g_v@s5@HQ^ z7jMqSLKX^VQ(G^r{eb<(^;09nI&-WUdfZMJy28{;r;t8j~}m@SIeE7HotPp?C%TX&A@&3JIr2f*hC4Rm6P0*=MS8@ z#`4@Rwai>CK+O9fFSooFD?X=K_Qu`1de0t*@jG8^e~>@6eV67St^$*2E0yk7>)US0 z?6QAi>f63EeMg5*2Cvk$UVh?jJq5rA`q$`gYEy@U*c{2gfcx_Mhu3)Dd|T^AkrkB^ z)6-sH_3&NX9{`>6kfM~BgV9jEncZH9vx<+rk)2~uZb?W8#+of#Tdv=%tG4e92>!u$ zfdCnRsgh`2kgb5YOBGU>BXN0V$%Olf<`n?9Z5cg7|0S#aC&>}HVZxk((O1`k`8Peq z0CN3Sf?|zJmzc%7mg#G7TmXT{1>+Vi>}o9M%3~Gx4=S2Fy!bd$OU~YnRQQ6VP`M|$ zKMY{MO~ERBIoURm?F4=A4-b$OoJ)V0$z3%zHj)xiUAiMACP)vjgDH55t^5Wo^Ql+I zF(KUl63I8n{wtD;>GMO*nNp2L(O!zJF2$||5I?tfXEM}JHx_fqCI;wtKB;@d8sh6m zxwG2WPYzQ+?Edoc<8Um~H3XMkX57I3kt+R!VHIWE-{%AoPcU#x8D~~a|G`F;_1v6= zySWvsy^lM(Qz<8tifcdwYdNC60>t9xfKwS%Ry2=Npl{)oNozgkKMN8X3hW7W=s|ybgpKA@0`R9uTd&A7?T3VJ`#VXfDQ5PucqD_b;3|%gn z=WPHrp>Fn7#&bYx0A4(|ErdLO|8S^&$Neu4mPQJ5tXP#l;2N+m0jAei1oV6o2y@O< z^8TjAi1%&4P;O)Tj8S9cmJMqC`O}A#t z-zH*;7pAOCnRT{sAQ@K?6pR2v|f9L(O&>f;jojs+FFmvoVn7{(yHITfA1(m)}uE81po;+A*m#y zCVrDu=BKdc2F(~XXA@cTH-BxJI3d{KuY=GU_pz_3Ftj11LbIUwHPgQF&GVsDch9xW zldDber=z2zwr#)yVt1)sHwH5epAzRfb4+BPxAg+}{PQMIpgJynOMsC!>Z_d|C>Bzb z{&fVmy?w}Ig)NR5y)t=il{J7V0XsXmFINpOo&}nhwyu)v7o(SXXhaSq9gWDA5QF&! zWbjl;U3-3AD?MCXRp>-cgLQs00invuO768wK%v49c%#M}p(m#xx%Z6`r!zy$%W1@K zX>NLIN=D0V{O+AQGBo9_K^$K*j zt9%vy)?cDEKgIC($&G`jO*tvV>a__BkD{(ugKR|AEn$%Wc z?CDGe=xOu*?4Rr|b~fml{)#{E*KN6-0w__ER;U8eq2wse~p{K zZpFE}1}LHR_JQwln-DHhLO}eUAjuU9Btazr@I5TtMcZLmJfqulrU_7tK-8KN3x^}b z%Dn)GYzla-m?D&bb08oAHx>a*gPmA)D|Ee-Q^~t=h>h0kAXK|e^K2DCEdbzaFT%Q3@3Ps~+yj<0Uk-5R^gGn4Ao4pUl-pUht$UOrEEV8twIiSBuG|~lfNdwi>71kdQ|6G88Ug6t(^FW7?{BG+a z$flnKZGgH!T{JBZkx2ee5c8H&m9y6EJVV$Q5VSFSrpfcIKFIXu-UG1PI!_?oi?xlW z^N|{vbP*w5%yv{GnQSZELb%QUn1y?}lV3*U0FlU80RzLKFUcXOZ8qgQfz9u4HnhWI zE`bLghTi#)yh04$SaWV}Zm0dQ=43Wq-;xz#?F-1n4w#q~pwu7;#p&g~T}HKop2UJQ zvktZueS|~KoOWysj>p1npVj~pD-#6%q}8p24O{4Fh!TPh2YE227hscZV8{7PBf6N) zk+vAPBY^h^7dVx&0xJD%0PgC!qyPL?r+bE-kI_{-X?@K)1Oj=J`-1k)#Bu`6qm(~? zQ-Qbn5@w^6Lk$c%g*n3T`De;^3h8De+35a{CtX3R7L7q!31oj&hS7>bEU zKM`%y%0=8Q6Vsqh$5T~F3atUIjMFA8*M1b2lqx!WIP4*cF?idi17Ugj`F-jR2d&)U z*~6T{h;{B)+`%87E>0`Lq?}V5GT{&1 z0i~hxZKed#L-bF25gYQ#7&NW&MfM^#Iz$tpsttVDJfvPruH9@zi=Kc8cn@BKNFmAH zckuEL+DC|IX1D=rUa{jE@S z!!yr>l+HIydcu}H5@SZcP;4{GH6uLf?q-Axb-JndQ06RP?RE}qsGj*;=Cb5`a^V8kOu7J%aM~<-_ zWSL6RzoQ#svR>0}g_-hNrIuRQzRh9eCnfiU{2puNfP=pR<^F3LvgL~mw0$pp>C{tl&jDBpzu=_>JK`ex5e4+eQ-6Lvte4{w9e^vhV&)5(RNF{5;t*w-ZIA_4{wmGSD(76vt zd8{;s>EN5vXf2^|i95G_W5!W!wGZ8FDc~gj_J=Os`SceM`-aJ$jL8a9AyIF4wI`9> zn_oqHQSJ`%(8&LHQaDR2v%5V+JMOn^q z4gD~Kg=RZrfhQL)_@DptE6}|0IVB0f)|I0;20Ka9na{5!&!M}Crt>D)DGv9x0*2OH z9ME($MoL41TG{#CN7%J)`6Y_$_AS_!;=E8RQt=@gf-I`kMPQV5eqNvxMV81QOg zSb4M)zK1o8=Y8%@ZKU6R2i|%n<$Y?*bSu9N0LYE|9gL;o=GGo*M9D^7M{gf_uE(`o zAJ*$_AA#(5$-#oH<#3>ZKwnx#kOQ?;#@>KU0QI_;SfnMx_oE99*~6!T?jxdng;i%? z>@Cm!N|ID6(Go?)mwQ;99BF0!vsMbzn2I1FZi_7(EmYW}T)B z#O&?PRrI-x`FX=Ffm!TNw*ehn;2J6we>+)t9jzIJ0Nh6c^rWP)xKRu8Jhs~@?BAyV za{@mxxKj!1r~VSfRW{zpJr)G4`3~L-mY?Uh+1bhU1pO}=gyu~_mEQ^_PTYBVsp zanqkv*FM>6p2V{J?Z(KP@7C@a_L#ewj<+%oTBF^w-H~j+y~uup>Hi3Ikd!;Q7nTkc zddNGo4SY!nTY|l0n5E>=98sABfgNm1{#{wASG?tk>DpfZ?~%Kg;JOg1+qo2Whi~d0 z>E|8=drm`17pg|nSUC~1k5;84zCSmF&${J_-?vbmFAMIUwqk_EQ{N+<#bT-7Z@Hv& z{6r`3?Q(8Ty@gI_&O^N-x$TnS!qLK)nbo=Z3%40fLgKT$z*=;U6c4=hzU?KVbP4pj zit3daP4j%ml0O`jO7T^X`QI#yTY5Sz5IiIe0#ZX7Ks+ed+MFR zC&P@xgMma8BOt+{)-!g@eYQt3+ptmTl*K+POmb z=K)(4z7l?Twiw$X$rH(F`@{lgu4jS=VxQX~Ylv$%w6iwi$?gw%VY$H)y|~1VaM55^K0?|O{qozb^N*1!2E&TVILo}2~BAjVSdd&ZP`Ql{I6Vx6cueSP61muro zRmbE$s6r(U%9Z?^D60&Bb{eZOjz$&ZKi6dAL{}Ia8(Y<%8ybq#o(TEpfxxdKdV~L; zipZzuilC*S!MZ`^{f6=1kph45e4U=1-4UfGr1eH_gBtz+QW1Emg9}P&6m2+?Z~yHV z;QG80MHY@4`oFdPP}vbhuS=T;T>p$U5?&A}ApHNg`@5X*{~pX|_g&6b<2cTDgskuH zH*~(E`a84)K!&?0xX=j=d%33hTojKcY6=6wzSX|de}YiJO_ibyfdgBq5}Brk-Habq zw`BvBLA?zW%gO)%B^$?7fZ9hxy{>|bQ&n9vvto=#)t$P@=%zxYFnD!DJKDOc{H$oq zu0STP(J~(<_Pwg=g0;$^!Ow-lh}$9_D^I1AQ#;@|1cO9rxBHt}chqsnkDg-dNgHhn z_8A5Vr3>q7ESPqdb)aEb(M0&YrjW%kBB;XLABzLA=JxNu3PE(P?i7Ts?XrSmnip^# zzH($0+v9&8Kf$B6?aObju3%k&c~#|*vU|c$xaCzp+wNZIn)}3=iX5dT^jssUh(0dJ ze!k;ts{Lp3HaFZ|G^Z1R$Cb9WH3r+>H=1pCs~q?~8EXA<_Y0%k!S9tOJc)@hB8zRr z`K+zD`eHtgT4SLuzUa}auUGj^$1kbFW<<8k_)d`8_xEm!}HY_kqcBmvZ7FrK~j z{5|ZX=EJ-}wA_DPS&*9Au?hP%hDC?2I1$bnpP$c&kB7f<-gYgV zVqQmcU|P>F`=sQ6BJiT2N#ntZOx*Lm9G0eX3`+M;xK}YM2PHGC6?4kUBm|fHFvm7` z+*8I+l|Rt+^UdzuoRvc2#&$@A%%sF$oaFW7{Jl3EVfcWmy6`>-bSf3d0m5b-bJb{Z<<(K{3FQF6pL0whJ>N;9Y7HTAG(@1j@SWjff0R0f5 z$Wc{J#1&fJzVJy#kx)OBKCEgnPquiGk@c#8mqh{D0sE!<-*Ycr5E*ob8&d$gSt8Au zbQ%#_;6hDiuN|%)FDPrd8VZ0Q>%UYM#{?9h6_$Cvlf^nh{pNU=JgK57b6iOHN)axIW}MDgkc>#XXOuS+;Uy%NO(H z%JG;qfAzkb4;++y{>N$C{7m}8@Y?bR$C)LBG!Z~jxUeZ?I$n$(hkjTOs4ijkSj0T} zKLvIW{uXm_ele4uO?wopk=G*N9APmqHF=&KX5BRvpPf8b%R;f|*&OK3v2UiWR7Y`n zoU?3amBd}d!dAqM)*GwK2P;`;t~o;8(j{rK6h29Yf5TZX8E8yqMem(x840X_dCFLI z;7qZ>abp|zN++vS&rcm1wKlhz@@C{U+^C(`7e%zO2NjuASu~ybELpPpvnrsQVu8i* zo(BwSkU4ETvc2zPtdJ%B?TS+@PG#&e3Hi;?0GzTr)cpjXeiXMiS{z^f)7*{pzm?g~ zp)(6qo5qUz*P?B=d5|brrEfl5?@;3wshf2FGV2%quCbg1RU75JY2cbxYk<1BtAm>7 zx=4{udXbxUty2|cRYZQN5_Ef3>AmIdwK;5UaaWjYI!ZfcRpL}%ubD@}9lkm9qhZjw zqG%%E^Sow8Y1@4{?S0;>X||IZ(c`x(%TbQM93nTL*9CazPTQaLYU{Dpv3+fb&}%X< za3{PW72~aHr`^7+51`(GhXjl}nXB;YT^DC7k&I88J)DC#wd6GBS?p(3sp+V0PAigH zTftmhT^bgu85z`mz#0fWTBj6N>eVuSbTB~7^X*PeNE)|7xb?Prxb6D8x%I|^l{N`H z`2IdqALXN*Yc}K1pnWu+UP3juDVaNkIGB6r$5u273_=JeE;y}N#<#B~dOgQKf7=yy zb~E3#G0!8@Eyz@1Q5_5;*X1MJF6?NH;2V#-WY9EGQjLM)*cV>7wU2C0GiZE=>AL}J z?2HkScX7Cvs0*hK#_Z|zteeTUA(n6f2kK{FR!)` z>{siIxQD-d5>KCZ%yr%m^U5D?V4oUkzh+~=a$BnR^;z#5oIC!Ith0g;?65+MvWH+_; zICXdBSz#4*;gw5Lm6HjaqE{(&m=7o5e(rPb*w$eq>$Q>{Lz$s?Qc0Zmv z>*KWkR1y@=(NJqOGW*epyb1=Wn26o3Vu4Y^Q|^(1BgIVfT1TQ3n!n?|263H)2m7)U zwM9bo69MHM72grGs_PJaqe7*0qpXxYj$GH6eTucOO%;i}icek25T# zh9Pe>_x$r!{w3x0S-Q&!e*17PcSHH*GV93~gh#U4lJ74Pv6QRcT4fZavQ^Fut!n=r zX}ihlB;f7uvg{Dov%1gPkLC{->=r=#)t{-=%b$%$d+n%s0|4m;)Q`(MVD z6(tu(F9~^uJz1L4Hc9c6vP~m|%J_TESRH)6h9OhvLi-QZHM^>6{2RNIrg)w?QjEJ1 z9>zc?RFCW3NcNf-tfRCSm9FhH!uQvi?={6%d3t<|n=7piOskS6gtjWD8Dw%Nm}PRS zFn_YreqDX|PY^##%Lb$c`U^_RCxit0iDw{l{G-gfFalR^C+PalQjq|Q%7iCd86Dvk zGroC&WBYKd{ekeO&2A5o+bLm>xSpXoqVVepU5~^bn;1%dqYenYiaKDD#~-Q&RtQ(VwFop}Ll^Ts2w$&K(xqUpSbWn-PGa*<^x5SSQ#C1n&E zKg+sWSI@q1@SXdSc^#~L7<=}XLZw&2_5H=x!;X7L`a{Qmt-V-XWU<%H91vHSJIwN6 zv`D>y$JXkgW@_4l=B!fOQJ7Wb?5hy#Bz<_vnpCD{@dTu}B-0`uu3qGjA^g?JzyyOr z+|4ebytpj2JUge|O;*gm%|ftJka?7JWLnCHqTSO?_4}02S8}TPnHkd9=Q9_Su1Aj6 z2Wtm%EL--WBn6K#&fx^pL1&IrcZ6ni^A)+_?l-mvFnBNf> zGxu2R^*<9Wx_Jn9pCwVv^XQbns&qo+SnbiLLdOAQ^R%Y#J8zdgA$I1lpUa6F-+APT z^$Vgq)U5o~^7gN{V=B0mmVFx_7lqWn5XcR$d`WZnk)H4kS@j85UQ>2R5{`u{I%79m z&A^qUZ5qK*36XG%u-bZNdc;?9-T5u#CV$uES?Q+9(J{R=pY#@gkK;N_%eG{3?wN~W z9M~|d&LPM(ttQ={LhqWZs!!xRN>M(+go`5HEOYjmNXtaRlKj74Mq1|$rb{-txtnt# zv)j(YJm*mN?fjabkhwkIT++aphIVkzm6Gw&9u;%@VXei?5an~U18Ps=lC95kfu6Z4 z#tD_BonG7bw^!dl1)AOtb5bax1*X*b@kNZlPB$a|C~CypPR!{q4KUWKx^vHCy?8KFl8xZ=JXx zqFs6V1m16zV0~nYMl$m(BA()UK!b!&z%M>0xB(tx?P{*H!8(6>I9<8nV_aagoXhPr zjb*u#7~SA{b(4)kdinC9*q@K<<3saCQF-03iP;egYpUH&Fej%s8O~*OuvtqWbY41k zrlZqu-G5?qi1|$ZN&~jIyxlG)w-FFR5o*JCJBXGA6O4Hcw@zKy@v#FUjgOPyjq+n1sZkr*JQ9hNNS(?hc*;>rm#18v~pNab{#91u?tvORFD*yJ{05__7fx5V-1UE|SZ0MJacCL7^tl%Jpe@16b zFkd|W(%jzMz5OG*((cz3pACs>m37oill6enuPGlgRT$my$W*M*+sc35XOB=!<@^{g zkNq*wFm>%XjAe~P*?~%=Jts%qo>pDZx@4^1DetlzZn+`=%dgF-;gw;!o)h|ZcDyOj-^8Hg6tw_(b^PLAAIk!8lbBdp}JOu;S}7uR!xhxd2ieqqiTRIZx6jr=Vo-V66A z=u8DaAU$W$1NYD>%$ihWz?g4;)y#3O&nt>+K-EOida#0+5cbJH+d&No!`lg)Xu--8 z#+S;NIL{S=(auUJu0M=f6$fC!q#6H-bw7wUp}DNtXs`6I5YF9{tCM=#@F(^->)L9M zR5f&LO=xwiXOmS_t+W_((E6eONrg+?=~sMa?zJ|2L# zv&lHmYxIeWa!uJb?{dp3W?mv&_(kmkl#kyEvO)e19v`CLouXSwY7B(EkG03p?Z-_wgIqzW2Fb%6oV|k(p!R zD;EbSDj!`RGO8};pK&i8!}B}tHs&#m*9&8uDbJUTXAzveRkzkzt~b33it#YI^VgvH ztj3C_p`C)N4%a}uJ`C#CvgK)THtxKxHjuN-unK7yv|gqm6eN+YoY`VjJ8moCh9q@Q zRGUav>suoJMNoIq`zz!YJIbP*hltAhF-gUh3<~Ob9hp!R{gjNbc(g22)x(p&^ujAF zczoqW<9Wpt_f{C6lIDa0uNlkAv}bx<$NGG6?B<;0^%;+4zWQ=!EvT%#)jTEL{kx}= zjeb=7!CF4ykgsGvI0T2r$S~=9yCzPK);DJ8mXAdokJCoesVskaX!+C4Dj1%9 zTwxVcq&1_$+o44?&h4DRC3yV~_wblyrPI^Fo}}XB$JzCnzx*$)zcy6EFwr9PQ>V1` z+Uv1R?i*-?S}TZCEe9^6u4Z5a@GvEA`@9t!YjQ!5Sp=PYv+4Xwl+-2pxjbJ8Wp=Ps zoKor8XRBx>*AXtgM&0tkOxlE*|eEvpYj-CWfw<;VC<<91zM>o zB%e*~syf2LY)s!gj@hM_&`Mu``%$2U?&(Y>Ej}I%J+fHLEj{7vRg;(;S0Pg&jI;BHnq5VLKi4jp zFk7_~4J$_1R1DvngTT<+qX8!zzQk#sA~Z-mA8)!0pX@Q4(AFG39VjpVbNa{fAq(B# z-z~5l-#@>X3=SQsS*$?ss3DR;OdkR78#Xj;Fw^jNB5+c zEIWsovf`D4^q~KOl*(wJQGVLF-IW4Ot=${<6WO@a$fjKM1O*=j$!l z^G~|={SoFmGm&c2pCntt0KE_}YJaFvJ%PFy&h?D%Ykgujt3a@f^GJfIg%-sfp%OW$4BQVJ%syKy4G&ijr|v$`jwOGj@Btc6Bo6Fm9>Fhd(pE$syEG)YQ||ugj`<~ zS99?5#7ce)W!e)H!P~^{p?@RYHSy_3NxRC5X4gc)v$>?c!E#94JoAi5V*#_uGU5Ei zw~BLjKCdE6i=Ei*DK)LbCljCtp^#{pwJxkBK3g$t2j7>F1I`)Yx&dO+5Y^o zY`HQDo&3HDvrwr`!th+)CeAegV6KRSDfZuKW&E|vm0;f_ZV zj%?MW_{EHfh)#cz94mj3fD#;)kR&XrNO#O!uChefaD0PZpPw^M^1F#m?JE6eN5`C+ zEMHF51Q?K1o<&Id0@O{T5kCFv9flwzK4Is*nV~3@jM?{{4fJ>GLSLk2A5&jdb>%2o zB=dHtX~pYS3=Tfw_4X*tYEPVx*M$*|2M%v3U$!}Mdn{F_O31DMQGEld@b{4uP(1Pm zR~wtxozlE7nhI8Tr{;1S3UrRtCBlkp!;~vda0;y|@(K+yPJPL`ThQvA3ZCp-kP5)g#BG)6H671LvEZ( zS{bO@Mps{f)dgPfZLZaYQ!2CfCDtccUn!ZZjh*K~CcA~-q^QrDK8v`;2ViKozawOD zDuEsu_7D(kCqgK1w>AQZ4j)<-HaV)vG5|`IB49{pIe?R*_d##iJ5K2M*ZbXmp80zZ znE+JF?nbk^IcCG?KZXY=0b2kycugdVhN^8)nXUlJ?II|r%M>@0UPMIWwu_nlsPh>4HbLyL)G?WT5MA$q@j9O=rs`83E{E zq3(gK`aZhZc)qIBpb&f#fcA|5P%jrD!&ccrZ0=Cq1F)mSl~q$lB$?eOfWT)4ZMtdn zgxAgQE<{{UHeDYV?QZ6zVPB}%YqPIEGxM^LYDP+Q#jm=6Hi^h0oqUe)piv1k-I|F( zNT2(iww~7^KoyKIfB=gWu>^QcDzBYMU~<0;TKHH6NZGGb%d4o$#gBF1~CIM410>~(QJ{$o-AO~9#ggxC%u(2^om0b3m0gSHR(GyXX}@kcBV7@!Sww^ZST4FKnTZyT#Hq_Dh#x*nvNCn&YKgtomPWc<~sf9VD~m2-d)~u&?6FPJP5O% z1jLw`#d{z75W4XqU=$B8@HErT`i%J%A0X@xoPfCT%9ZCTqc-|BnDq6tKv6otCM(!m zt@-^T?v>pOV3aT^aU_m(vz1o_0fajP4&#Boz(I_dy9zY2@oX8b*DDV3-jBe9P%dneydJegb(P^Sj9BHz*yfpYX+4?u z_I3xz-sm}B-UbAI5TU`=BIw$L=2?kX^0v8k=W8g-7A|T5(|5rq3K{o>?!ys6*Fli# z2-+6n&zf5^U@~*ha17F8oaVzlf8HTP!PZDpdN85sta*J1OakOzh~vai0T^ql5ZJ@W z!b_gt5d&W@BYNc}CRQa5|3*4^uDqL)2u~)FRIz&xUOUe^#=XV!+1{G!fMGC?j=OyH zh`AxCjVyAenm?H&QM%}{i)g#yI`)p)zOwCzUzO&55d0$5@iW)1f*7 zpyh>wmQsNOI z(rN(hL#p-};fo#oHi6jX=-2#DKlUE;u79|V{lrOa_(~NjfGTPTv^X5lCoRhQTddTO zBC~D2;V~T!vj>qgVN_sGnq-ExlmO*yA(ucNFKYS`3P1L!>)d@_V0JHN@4@@_?X!OM z&)+mNjFV2H=fe9pb*16R454(gYTX@H)!7xE24d>sw@T)9HR$!Oa+<43o%pX@*Ty()qFbZ+@q&9o%cMl>(A!w_i6^- zp~LO$6C(CFB3)H@n1A|{A`>W0atsQN20P*>;nv#>OBh^;0I~Vk1}`49h+hG|t$i=$ z_bZfMMh#e5FsH>`AC%|VRUSbG)eQP`_w#dYawJ*~bXUcFdxi5mfXQ<@lWJ+3>hdfG z6sL`~&GY^H=sXQJM%HljUcQ+FxuIz*UT+22<~$_yw-s73f`3@O*T4WwEhgQA!YSw$i2{l);srvAC zu%@a2Nd{`gkGv#BIE_MW21(;~)@sz0o+ec40qPDZ-%m1~o_nB`xXu zL%Ds74{=CMx(+v|!V&wjZyU;$AI zu*P-L6dD|CNAHgy=i7&J%U(Y}7-Q?tkbLKTGIlnOaWCs#&hC;-#cW-gY#WdEjYzQR zPRRMy;oTh=e3cVD^44#!<}Q4+J09YKk^U*fq}ywEHBq)rhTml|%o)FqoXdnq*3KGB zw|+ceQJHv~FDc*&S_rsX}pJ>BePF^A1dfZVA1qVW~45;v0d zdWfZtPo=|x&)r%K)Ff7inKW3XpeI2H}F9}v8`URP2A6ZK*gPT2KDMv$zB%Jn(5_hRt--MBBv3fCsqz_$= zA-IH(DA|Tn1rsj~aj9O}EH`duC#?YbxebAW%3JnkW}@My66|pW?KkI-FPer9Ljvudkl&$(3Uh!G?-iGI-eT7}u{*=8i zgh6kw1iPZSc#{Yy9YG+6*Pm>0mdHQI2xTAK(=}mX{_K=O0dEd{m;H9*3I%We>X+Tr zkJ;gv6t853*LwnwGMYqkr2ZO|0MY!Hx3%Op&-l)_Iro@9bH1|7r8OoPy?@u<%V4=TyW?PY9kSD{QeRL4*w3(Q3Hw_wZU*Emj} z*!Tyimto;1ndD(BAO579FAb|_+9yUKLXJ}{-B`Kit{mtn7N({`OHETMP+Mi1A~>7V zCQPZc@32!ao8;5~RH}DR#@gMzE?thNO<-nY(Kp<1V_sPMaF9@ra6vEpW6uYR&coYn zXJ?Qay#(*Z@3o?~^L4Z{3=RbE(?NPht+TE*F>Jv_<24 z$sVXkhHm+(V4ElC1~Sm$Yalr09S$q zjon1buCFPCzBd_D$nOf>AIn!>sreh@OkIM=cw3N}W;F$y_|Qr&UZ<<%=`h zjtqBCjan1n)XAC~l%)|x{{5AiBx>>6o>ES8%Z=ozak_81vtG(ky7dqbOac8V$^ z=^09HZ!GQ^*{Yw)3qs5IfZ~9hTY|&`AZW0t%VNImU9KOf<0tyk!QsdeoAqwcCSXX6 zuZEsXtPO;5Z|$3i(xO9Jjb)q~(S!q^sk(cS-k!kNgV|RI!O`iHDPik_&co2*z*@%x zf`0sUeIfK20J>(8-?_UO@p0KlJtKDr#=)OJg?MjEYaqE`8DmC zVTk_63w-qFH>md=MdsP<3;a4mgHKT#tim8NFQ*vSWIC_QP@X>-jdHV~sVaYUF)ROS zI=YuAK^P1B&y7&$2rsCAK6^Odug|xAJ1^^^`0SZgTFo=BB@|XyKi&_DxqY5R(Vustrt-3PU-osl+~f4HLZ2fQ<0F znjj-(K)MEoWpx`{xFJEO!jFbgkUn*7;_AOz+rwJCXUFn&P zvFMw(}4Z5Y}A^Qs#P>E)Ho*vt#obMm0*uzQbEdvH{HIb#V^`B zQy^Kym6Akdv<{tt-#3b1weFH5?M1$!L-A_mdj3Wr$m!)q;J!a062l!k+UmVbcR>yP z9n7Lz$vhu-6_qt(%&U@nG5Mg$0_!SBtVk{v3xaH6=6txyZeOtE1ezadJ>*qxLle!q z=Z42-{gJf&(MWq^kmRb&@{Z{{Wao3CM4IKd^4T0Hr%0JoJJ3`-T1-IEx>U*-8 zsi|!;YV@EEy#0*Qgo@!orPbVHW_i}QQg0F3F>?-moHpX^Z+mU623xsBQ7Wn@?w-Y~ zi`T!eTJYWpQi}pq zNV1mD^o_u-=40mE3gMD9TGNJYbSaaMZghWKnbK!il5&@^k@jKSDU!X0HgqHW$57#t z#Jp+AOK?%j5Aj&0kG-~Np3ZBFU)JxIkaU&dU+J`=e?)Gqp`pTwvx+`{il5>VIK8Xm=Nfi8RdNelH~d4e zi_6$A9`0AidmLHD^9c!WH#35(1Rv!rUf)(%j5_ef*9 zl!1&o1pRKx?8rrn+vXI6AuGC*Bi@jhY~YVGzj^=u#_kyT*V;M`?01y~f{`caF(Yh+ z63AgM`oF`7K7m_zkRk=SwAFrBw4JNC;FJ=oe4!_F5I2Z_)naY-Io#0vo0*9L$wOGd#LoZ~u&E;r?H%K<|D~g_8A6WR43fofCN|ibU!M)x|vGt#20N z2IxJH4_Kp~N?BP(7h5!sLXRdg6>F!>+AC3T>;sHeIkRu=23`zdj6#4JcZ}c zTi5f=9K&5m#mLy0;w8r#9=PrZ+PCsSxPfy~-{gb1uFiO5^$MrrK&0SG8^Dd4Z;Jq^a+Z$Eu9U##mlW7CTD8imw-r)1AyeIP~P zF@3*Z)GhRSS;_=iX-V^>dI$J&)ae*AYtc}yij?e$N_8D6EVoL^(vo@E@Z0d>z263? zx$vbG zK48?xJ5edCDY682tHY?V4`nXPA!OZMA|xj0%ll?yn?>Y5X89Xr+=HR|cIj4@1w6}C z+7Ly~Oukia#a<`rjO5cR2TuW7321%r+kSJ8_w_m2%{C*IZ||Fo<9~n zI@?^%lD(+>Wa_>DcLqQ$L&=???jF1~E}Tw1UiF!oAmTML&6|732Lo1{@3!5Gb7^5V z%RjO#s_351Q!PxP3ghF&{jp7pvZJ?kw2LRIjcOW)s(#4e9|tvPCeYA=zA;=se32?k z!={9uK2oS$0M(3WqKuNwPBD6;#PIVAYHlLCYEKX6eNQRi?epi}e+&IVnrJ?q;$seo z%(r^yb*IevTF1P6l@h%!w8qn=P+!4>S33FkU!5FD!$V1E4*Gt4dM`Vs5VFH9&R{-< zA&->yGZ6ESX{1Ag)+Rl6D%?mPY0B)7NfC_MtRL0<3irctgXHYmG*p}6d0=Jc!MpU1 z6P-)mr}O5q3Q9_)n0ygf^ndJRIyC6jT?$;?_Y=l`ST5pItA4rcv@$o322=ydYakxj zQ)<|YVO7>-HOpUf&_S{abP@+m-^!a(7J(s=2ZV3$m6;CdiK(flfI(b_pz#wKHb48q zoXF45zXI5%k<99qFO_XmQ6xUz%7>;57RAib#~PE0C~7|KK70Sos0StY`*)pNb@yKU zG0}@T@Lg7hJDng)0!TQFmoK?mn0LP822BmeQbNF%#)zob*DsSnBSAg3$6WhD&D3&C zEs*qSuBDp@kjU!>F2*Z>${fj4zlinBeoeM!DTHz4q#7t|83I1MT^pWc7pPP*fre!g zVz)G%p6mmg5IsJfY$PBey~P&X7X=NNC~UuWle2U4MW7 zVr4QoFCX99c88xckGNgvn|TiITnJ^2!$+>xr4_Me*Cn@z5B}J&{P9l#ZKOM+$fH7M zM!AKX;(Wukv@GPb*!sI!m~KUS&O1n4K{sx?=krGc$Hi)5A1Lhm*h`K2iRmFlK(nz1 z^fXrh_uObetToZAn>MZs>ID}tRt$8J5Jo5x8eX~W!qi58DCVsI<&V0)1Pb{`m3ZTY32x zq#cPJ>4WCq_i2AzUYB7ct1@k^gYEjqzzm#fpLt(m*R5z-Q*&K zQq4v&uuc#~Ab-911Hx_B_A!LDNd$|>xw@RfZ6m{OT-CVSwv&uLkkl@yx3|*$XjoE{ zytw&d)3L}sMTD_8j@7yIWBWCbkF$jEtL!4wIIHfB_aH<$UQc?1hNoj}FX)&K9A{!p zc+=f~B!$dkGh5p~VmEpfH4{J8`l7T_I?8?Ui(5sbvht{>7fsQ_2)W;O-ysoXx}`;O zYo(L2L5S5$GR7scmm?^GZf>uwX6Y=Qg{U5K&`GuW)J0TZSjfvqcmPqK7$A{3Qgbwr zDx?mCV`s-?Y3w9t{cI4rHYw8rGxG>5x@zuy?emei6(G}*KJC5Q6JwQ-bZ`x%SGvi1 z2^#i&PofLZ^+*e0!zeOMv?>s)IlOZ0-8tPbZy@DD8aq|F9FGzrm+mZ+$CREY z5HHvf?@I>_pP1h`b0R(0vYDwT>q-0K!SA~ysvKofkYqO9hO<#`-Os0DUA1wmIRiBB zqf~01t6d+bf04I{giO7+oW>)%*-~zb0`e#`=%)yYm71DNN)vbNUYAA`yog(N#e!co z&p~PJ3B~zPpaGCUjRZ|QCBPnMtpSjSXQdyqX02~Gfwp>`r5Dgm>aCkQF8s86g^(~R zvhNJE2}}}#Y-t10)N8tAI-jrjqGj=xI6~B9&ZZ0eVWJdr!RU@rf)Uy1fYXx2t;i? z5!=~~rd(if;+kw)G#;TT)b7- z);jyw9=nVgml8@{c&}*No!=L|03H(lfznFJ<2hU55_)qtgCv96uZS$^xcS}Gk6pv% z-4v%iaS9L<5BqxkLTHBwR5>^t1hB01^!CDHSAk%u!)$}+uDNzJtxRmS*R%+52|=-uxQOUZK`P8w z5monxI3^?@2&8^ane}{|KUJNXdE!brYMaOS0hOW>!vSCPx68=cEGrM{=|*GRrQZWS zpOnwHUS8gY&n%+KM#MarW_!Id{Nwdz4CQh%i1;uc|Tejqt>XLr_n-;bgT6akm>cY*j@epzfLnb7^mqd*eVI}YeIhz@lM zJf6<()dX_NCher&RhtiW^><~G;M@Z2Z4Dqe_u%NgGYtC0UIf?D6lwh-5n`= z={KHBE1c*?KQ1+Be)2iMkD#ZmJ6Z!C^u3xPxbgN^0lU;c0@hcQG;9iqJbiEIfebQf z0w`QIb;-rG|*AjZ5l&Xlhhg-ny zBtMc`)q#to+b_Mf(U*kNn&EI@m)z2&-EDgRkuI`NS$$DVzjrNlj*I*+MVUcH&ZSFX z`)`dUK0ezLr-=vqzfWkI1$@yNo=5SSx%aGA8Xi6zQJ-UmJ63kJsA7I|JB~ft{RK&$ z`NdQ~oFKd**V7h_9er3kGrn__!Y2trP$mEzpS$j3n6m5F9D((PCl-9n6^+g^tZ^DmA>rqAIaL);~2rl zn|JE^{V(j_m1f3d5ef6q)cJS3$_N%z}#3+nY zGYpo_f(+dQVm9*Az zk3r<}di!AY@K&<4vA?8IF*z`6l-ak-=15aQ-Qp>SeaC(ojrIji6L(>p*75kpO^4ULUqg>-Z*;)Gg;kR4HnNi`w-O3k5J?q6i>(mbWFvYf{u8>^g* z)QNLis$9p2;hZF#`v<^Fm&qG8Zqe_2WC9p5JYxt{)Zrmc%%FS9J_YXZxHBRV56l-D zaeN~wI5jh)(?*ys;q%{j|5wuGfiEpR^CI;?D}4ciCn;efo%Bd}yvw^KXpAN;$EK`^ ziaGd3pX#o~KzaQWH+)lRW8PPngt8X&(M)pU_RqVC>qO-I6G~5SJ4vVlIK`7zOih(= z!HjzPc(Fv1L#B;oKgw_M^+FQvUx)utkRn}}5NhH?chQW3#D?0`gCz)f?ImFwWBs6r z>f4sFudMJI&T5z1y2b0fQwWTCt-kZZ9XVbk6;EuZY*~W6brOT}ao9(K4elceA+yVT zeoFP8O`_;1ntEH}b)okL;g1W!rnJ1;@Ed)y%--I}9dOxY;!z6ePe#g2~ z{k7cj-lNp<)_GLxM2Gf=@d&mXt!Bsaai!?TzgqYfqzsj6P#qBW7aX9p7u9d)E9sh@ z;FRPmPm6}qKYCKEH9DJc|6O(^+hw!>D_q}aFT9b~QVoQwh1>C%sBU}A344T}(o0!5 zdgrc$>o7ww&}Lq=*|GH5{1sUE-Oz{*p8y0uU(B2NCuN~Z^c}IHT!9V_8P1cpt1#+s zMOR5FI_*p(`SjwB2)$oCJQMB*FL!i0jn1>D84>fIWkZsi&gC}DKvc&H)AgyW<1=H0 zxda%tq(SCgyb?vMe<$VQ>Njo*sn7`->};qfM<2!Wanq{A8xGliGu)^1~cu2^!W(aauQ*rIJXb^ z#RIcpS)xx4xkqrX=^C(H#gtUDBIjIuqP^L9*~LOw+mV?rN^AkyKdsMm-(_9`A{Opz(a z>wBlzmSQ~-Z2$O?Y=S3|wc%&6oq?L8O-5})uUa44^ou4Mo`k>Rj-@@e3>J$e-CP^R zBzTc8s1qdN9myZ+D~d41#>9?^`^JgyBC2Wpbw+J{a<|a-hwZN!5%a0%O>Lr&lSu^} zI2qWLbcI!hWn0y5@&i&!#LL7@H;!m0Y@{e8dD;J3Z`rvrofs0DzZE3=$EhPc-Zooz!I`r``%eI!2zSE1CGzwyaUT24c&9l7grtlrRt` z>vC9MV_$C&ML632nI7jKSpT$Y{~t_;Y)O@YG)S_sR~a9C8nfd!$yFjW_UsoJ?wGWc zM}BFeQ#!uG^v8)}eRn`!ZuI zb({i~tg}5e!5>ofGK(Cy{Y%Yjg#(bjuZtmPkI9ZC-728GbuGDpFB*oyALpLqzMk(m zDGYqoPWRE8gt8u_<7jkX+#x$x@1P9*m_o8trT39%E=xgD^O&{HBN4j$q_$sIWqdn>RP{5%jMrP1h_7|7$dQ`ypyV0oND605Cm!=IB%y^eW<1 zoq=zmOz)?3t?V~w{U7q$Jm!-j)Uk zkm9^2M+@`5BsWM}gt#1MCwAs($cZS+RUDLh!~4w+HUod*AX+%LXM^G%O|P0mQ0aEr zgjFjV+(lOs?0|0w3A={}m-cVy{@M%QYoN(I<_D$9GDa9%J*_Efj zr!I-I_qp=siBr+quXXAZ*@9oL&fdeaS8tM)m^7Mvc+;72$o6eC;&r6m+otTJDA1N0h@l<9KA zIE0;y3x#qD;w1#50)omKlAuhGq}|rEHjWC%qVjZ64M)P+;rbLlr;_?!4|TL&MaH6P z+7_tG#{~A&&OPSN`g%qu@dr^N`|bF1))5a7e_WD_)Vb<+glhupANUx~GtF0Ii?>G8 zGz|WH|94FL%FJJmM*pa$d0IYQrm(A`<0Xruk4Su4j!|-m_D=cBx8EwY@T3z zalSZ*k&fj1iPA>|;m0xahb_gTs-rH+3byJWj#a4BH6z47yD4R1d!&fI69=SJpr`+x zeF9!%Z6Q?Gl3E;Sgqc(8DX{Q<6)h4tIY2tmn^>G6an~@ z(Ro(xUasD+Zf}cpzOA;u>U&+fXkDz!m!6;0&aV2_SS~}EQI&D*LH=u#G{p}VeA%3{ z0iId2Q(uVxz>qw1*_qc(j@zF_U#vQU6HDvcbpl?LK$aF8=vMK#4ReF6=dWZ8Q$7G1~Jn z#9gLkr5{Z3wyVi;RW3c@IWFeOG@M_xJ_^=<7K}GTla+Yf zo@r7TP=qNr)iDJJ^+^|@z&R|+iHABk2^m;^3u_> z;L`J#Hqm}2L?eszp5Jv|qc8gTKSLZA2vRidGFkZ|N6*|p_Y-k?DkiYBP$C!XnAQR< zOV5?oYnfIx{$@6&2hkTrYCNlu?R%gSrr4q-(%8b(;a_kn(Wv^2L0^DSwlW--{A%Fo z%Ocen)iV?h*Sx3>B1nN3jh)p}o*B`XO*Ari`&_YPDMC~A$1_lX>Kgk)>c|mkrA=lF ziwZO5gtUvHBj)7YLfz*975<(wbp^~15d4^(LE@3(6>upP&E&sJ@ek!Jr?7pwxFh57 z-S|X+!DlN!URYR|Fk6C`A4@)IDYDay@r3{8%^rs9v_PJl?Rm}4^Hv8gjQ{TDkM`v|kj^1SxjYVS)uW3%nep6wTMjLK}R5cqU} zkX+!X@ma%|AvO-#!7s3*nU4l%3nurhdmIt`$X$<@kxn6izsOv*ez%9S;!kNw>4Swe zrnwajInm1(Z>*~*{|dAGnAq>1p}MLM%Q-;$Is$@B3qpE12?drW^VWVno7rA$rkf~T zttcpZsVe+2=tm1!b12Pg{ovzB!ginL7~)$VRCb}fUPM-PK(*-Q52JE@jP?YzU8lmp zHwB+W9faKBXqB&aK&4|SHa3pkcAWHKpY$ibH`b{h^x1ANROzig>kB>KElq&SNluUR z$R?VH#J}Yqd%b_OFiHI=Up{Z4OSaBnNBhM|P09i{oNdm1bXCgMzzv;s^yaTv7YYhQ zE70#kMnz%^GP&yeH9fk22tpu$o#H!Y8wf0w&73a!WlvP-LwV_5WzH4J!3GVOiOfwJ zF}nt#eAywdv!nA2@+lefG6;Ab{D8H_PzL!(0!#+~l)efT+8+(7%GD0pR-Bh%(#__r zm1TCBSJ)G`BjF*YlHC5HuE|X_#*dCZ5@gm>flQZ$8&EO7KMfUqK4##Gf$H<#$(@i}@oA+PKXBP6ORv%gBPfAE3J0k<<^! z56MfgPJ^E%X(w&I^E2FJFZ*Di7F20$9blq2=7g~Pax;(yc20>)n=5>y7GNcNSdJoc zUVi$;9APzaEHjj*tEa4vB|76*R}n)KoZ)R}Pz){{)Wv)Ceha~CnWK-rH=zQEs{G;0 zS2^1GIc{qrZ&^sLXY?C~j3%bJn2XRAT4;J}7IO#EB|c=Qn89ihnx)B$rDr1p<~jxy zP)%5klWAn!H@MsaNW8DA?q%I(*nr+&%S;AMUe)p|$>D7Vl3cTi!;bW1!;x?|aP$$` zv14hBmsGw!j2BK{UQn<0YM66NZ^)n|z0z49ip)X#GKa)C`dUxIG+i8aN;Eugkr2C< z!nrm5%}HMU2Sf2ND}&1=EMg0eT^Qq6zXKyG<3M=KI}}H}R{iKmrS*-gmBt&Cpq8ic zthoLa26~GVjCWv-WXnr++Y5U>FP!cOFIu^Trb(utuYgCqe6O84kXKj*olIoPekj86 zF?1G-Y|iwHY2^!%?t}pR2#kW-IM%_;|B1c8@t6Qjydo4~^@sH21t5}X%*_ioe;#Ct zyi4%$Qj?3w^zuGcR%&g5_R3X;cLb`%?7Gs!H`T6I+z-;Yvh)eVbiW$k2`G>iU`}c;e9WSy_*UPc7Gdtj zl%=&*V1CXE21o8t4iocRW*9GIi}haWd6TpkA&%AlGRU`^u=`gSX-6x_>^x z3A~X6I8E+l%jzYAL21VC#dTxj8Z!76=9t!>>6S!wm{yz@aoWakJ2Vr$9>y>pOnVG! zj8lI65|uyzq962v>v+(bY!mzQhQbwrdxy1ut0DBvZQ5t?s)c@y4lv#JF>FR>-|%wy zTmofu?wwr*$mvpbmV2EdtVy~`_;u7uH?SCH82Co0esHi_^oqY-NZ0&N!2%KsdD9HSJX4jZ)$;gYabnPDGixyIA;gzp2}vJo0wWyRLK`wA7Zo`LBY<5 zU%YV{U;pQz#~+9DDWA_vtxD>E^H-$t)H6ca#M`#vb*Z!x#E*6u=4!B#_@PVe6`ky? zg4wY(=Z3&1@6(;U9`k-1V^r6qGKAS<^<+l$&rf*a~F*$n4ne~#a=-DR~9SmO` z^n;>rgRG_Usz3ZydA3k7V?7jLN<`9hM4u{kH@Wx~`K8)LY@d$`_npF*Y7(r9Avu52sN|qTB&J{sWpu{| zeP!+d&G=*cSD%B;{5sx;MU}bGFQ;+!=X?!Z2ZmNF5JMR7#GpDwi+&|!X7eQG_ z^qB=lDE*QDbb^KCeOL$h^!I*hT-Vq19;e*m%3r)H1M0iXyCkCc$| z3`%0i9&I2PI}NrMl1;6GUsY2%zT`Z?GS0Ut^85wSHqg8{(E zk~8ZTu$f@G_Ze9@zArZ)&op*;eaig>c_DF~Gc{%ZK}(~gs$)IXL{Kwu`kUUH`}`_E zU0>!@A$(d zRzl%|0DiXU!$b7vjSiD+My=izup7YF#b+41rkV5g;8LW-j7?^rprKf{?+jRd*?KSX zGNBsh%+wmTXar|L8s?@Qd|0;re0j=q|b@lZsfXh?U5Zkqcr7Wd$IM?}oUvo3y zsk9FG%p86QnU&4D`W0Er^CUmsFS7xB?WdmEg-Ej_-ETgwv7Yk}SpofKvbc=vl3xRSPraIUfG+%x_uz%$&UGGFVqHuDAO-fGHcTt{ zIC-QMohdK$JX;~ka3!8hf>f;rT@SR(IBd9v@QnY+`{?w|XWw8i^Kje;bUtNkqkFpG(Yl#-xrGTs}x zy#b_2Kl|co*0n;el52ZLbET!{(@6h0BK%kwIHRi!q_S+GQ@3AjBo-Z~; zj48n+9yO_m3Jd@KCh#P>YFMNz91QLkXyDkJ0e@bKr!FSW!bV4n!~bDrg0#z*T9VVKKgmruS7I2*-{tKZJb29#&G_cNtKKvd86A!32P`nYJ4DDGJ9apQ zh8UY^74YsLL&o)7=)slJhhc<2d!ld`V(`le5B#C6e;hr+bZ4`doX$VHOi1j~PmG1qN<3pi8Zo`lCl zg8^C5b>H)MN-_%E5%26Fkr!AEOBfWUADn8#)7DrJg~^(4;HNLb!kwZ#6m3#+i8uxl zoVuxQ&O~ki_e-Jq5zcC7v;6)VIMy@3Dd%qQGp~Av;B)>wzB~kkghZDl+#&YQ!c@giNd$tgeK!}8_f2}j^}p)*jwN@?YqL` zz5fuiiBuMjVzbP9H#3Lb>kPh-%fS_I--8RB1A#URz}j++bLw!9NozjyP$;hWK9X9Q zlV`HD;O~=irPc5DiQF3sH_@H|_N);B(9pEIHY+noB;q&l|EN6$R_($`f{);$=uaAi zWBxXPg&`bV4bb>~D>eeiS3I5qKxt67@=?|S5WLwRg9JeHl51t|I~uWcuatB%g1qH;X;^U2Y-kDnYIzc8(AI6%w$>(szD?bM9#_+*rQW`$zA9ONO2jn+O_g&!! z0Bv)qU5%%EbOO;}k_u_r4Y-1%i4%$&gp?C6qr06?0=$^@gb1(ZkNV+F zVB6u(WG>-?#*0l9^*YR1{cp$!Tudq`vD2Jj>OO%mKG+4IZv70?l0NResUzW#fJh$! z(oXStOoU}eByu9*YYI74regritsQJCzKaC&xK9K-t(H_rcB2^A*sv?G0Og z+9Q>E<-rZ>hj8tW%DwDZ1I*lVv30*kZUazY5Ujp3&TPD4saj|Xz6@j==hp3lL*ML_Gewy`{$~F7vR(UBY z4oHP1VkvYkhQ1yAhxT~y-7M~YBN}__^!1}*>z@xZ921$t^vNSP*EfAGR${O&vfmYl zb%bEOSz4RD;F%ccDYwZ245UK?1_mlpmd95&vZMhk^WK|a;00WtPvH8`hLeEOftw%y z1)aGi1~*2nYfVV~(@KF(#qvr9c%el&w4_@p%J?-}{Z%x>qGL5~V{c@T>8iXA(pbx6 z6W15}VXSifClC?lT%V*iuGR(W7yXpFAP*eh`s?T~RQ<_Y;4tHb-m^I_na}}J=dY<{hyG!u z#L=yBatJeiHg~!qPD8?WdB^ekF*%EQn~Q4+V-b$X`-4m!bg!NAEHT)w0%u zTEJ5}ncg9`Fp=EQ{%S@%ZF*h(W%C#BTn_;^OJ2ZeCfbC+g}Ovw$e1VG>JktiC%H9i zNB{%7;y=Hcius0n&k13dk9NC}=}J)fG)E7gaciFm+oyi0xyA?seMFpdIjFiwzk9hh z_r@7zCB~x~NqVUmMW5rYd%>g?>nP~kYx#NE6Yp86PhigPaZ;&ljzXd*zb@g)=o=O1Wk82kC~!Vuc1R5l~CWCZ_Q7% zVdcjP=-(gnT~$Ps?x=SFb4u*mtqIVb_9TB-Y7Np(|6$g8={`6Ge=<*y${B0m{4Vj6 z{js#v(aIN>bcakC{@+U2p!(c5IKHIpVzzaHs7QG+!{30-=sCX{sj~h|Nz_>5`L{Cb zCJP7-THRJ}b4#Fa?RY((Ej z)m6N7`1O(1D(ZtP7#bRHzIjmHW#KU_%*5wwx^jD*e9pxHfo_Pba9VV&%_&NXxG{X0 z7=?&)x?lh4oOI)jVzcOSB7KiSHWEBx%eo@4g1`FpKJ54}A?X=P?7M7{3JQ4bH`vx-e)W#u2Q1x1W-{`89lLTuP6J-Gz#I zk$GFma8{AX?023T%_C{PM#f_$9&U(6VJ9W{dMuSxrNZ6H#6N^4gc`0jeY{Ja53dVG zxhh)_n#(EB;Jd0&g8g*HfeY!S7goq26+jX4I-(18UcYUxy&>Zbsm70)jG372Nxv~T zLwFt^Yg72X=-#U7J(@aBPZgd6l~H;O){!t)fJNr^SW15wPC0dHAVbD@9?oU2wT<(J za3t$<#~)tvf;QPrUEe(o^&SsK220#2`+AS_7zsHPS*GVQ&^}o@2|u6l6*+91ZDs#lDV>N! z9i`McomhlHS|AFPxaz z2M3w4d$tzm@a|{y`;~p*5!$VT?|pW(34Mp2NA}oZm)Ha(!1dwUX~#7(&vD$BwTWyIggxRz`3(eq%+#0g1AUm3?@lS))T9}gUPhe z(X%8ghS`fIeWQ8Gvb~E7Ig&`Pak5ITtD%>YVS&h6%+(_H0*2>F zor2bzG?mOaGy|&Usq(AmKQnsz8u^-rqQ|u2_X?zs7Hyo9;0qZZQNfPn>iIo2i7sFW z@6>Rm8#k*8C45!_>JiiBjP04uW3>DvOO$iFhSjTpfCA=MdNzvj_;^cDSCY;M^@iaf0xf6|-b``|D9`&bcZLp>_YxGCP%~ z$7pT@UISjHXRqC3juRa=-;y=&FF7A9b?zy7CBRkbiAaYyLT9s9fN}ZRpHBJSd5g zqlQ(XssU8y(eWoFvJB?cq2U3EV4P}bH-efNU2$T0_hAV$gIKy%ab(fl@sH4oWA_*4 z=5}Ua40q@xDY#XKidcq`JipIH`i{eL8rPqGIFgBywdj}A1=qgGwCtB!E_T^=@ln`` zwsFh*Wfv15G(dCGkutWlND&%*8F+vKgoCA9`ZDRwO{KFDAL-YmL0bZ7WF*SJIcv&k zcO0FQtqMI1kwaW)cs14E1xd#`4B}g8-nQ7aO01|vJm6FK(|3Z6dk` zM7f!j1seodgkBz^aW{>-xm9^;r*2Agp1Kg%7xZMzSdYt+W9dXP9l zx1Z-(wrkd%jnqPuXAK{BeYjyArvCsE_}f635PeK-Uee5q*)FBBx9EY7?1d2gN`dn| z@y7u8&D#ayCN6^8)}Vq{@nMCamml@W>KC_-h%||HERkn(B&4EFqit6Tmv%)e5p{tt zUyU!Ga5)ak$6xIbRJDQAFw|F7lZRw8MK2upUeq7yPOw^!Mv0~=KgVONN|hbU4|we9 z9{mURPZ;f}9YOK$j}1KD{n7PfmE57}lU?B>8e~1%=zJ{;!(fv&kwT*?ERS;M^?`c4 z^?Fd51LUDse7`p}lIh{V<@q#e$O5H*pPv!aK15KcR$j%o8NO;5`!sa%(2pymcD$a+FXMd+w;QjVhZ&qhKGE$Dl~~HZq+6WX^ z`cHo{e+%}*=_q>C!l4_w)yeR)*G_3zxDwn)Rb4WkMx3dkh##~HdJUk#6vck;eYo2o zkA8SP#*DF{0~50)lS259NeQxR!k?z=qx$A`yP_1+fyn$f7%n$+OCIU*&7Hjy4&?1_ zMVd0oM#~~0e$~Y*BhFAVIXvh=ai+_gn8diMuus3z*XHlJW5IT)qRzF2-I-`$$O}aY zEVrBUNz26b%@kMa)DX4i@4N%{)tt^IG?~jvKR#mg)_RGYBkJMAENJv3%9s1{gDgvw zTi(!vL~qg0RFnBK?Bh^EDT2QK$?x-dw^S?@HL)%e|oxk zOE(ncI?B0UVG121B*{qR3;w!7W&qR`JKCE5L10+g<2=;sD%fJ{V)EsYc_D(DC?j!h zRc1!kTpOKY4@k;g{G09=m$Nab!hgWpFs|2v*JzfA;H0z9$zw1An@G^0-Nyu6&%V$g zap>`bPtZM*A_Izyw1_IiGbuxfx?;pR(TQFxeGD4vm5Y54XCJOd8IGwz!|g`WAnR%W zC@@_6hYil_hk z)n~xQtLz>@LKNWzN<`V%ECICe;oT?l3&fFd9}Lf94~55^oyb&G*$CxpA;j zTwm#Wxk}MV$=et}i0e!Z9aw!BNwWjuEI-^wQFFQteIV_7y6o3opAe@Y!Jz+kHtk(& z2)8MPDZ`_?xvn_Z(Jb&m?vu|Q3JYIW!Xk5Mxqth;TFhZmcVgI}EMoPwGo(S*kws)3 z4Wf)bd?#)B`-=_%G4yE2DJ_Ux{J29X}TI!V~l z>S%7Di$l1E><4`6?erOuI2n5HgiZLa60n4Cj>~GGU1R}pB(9!ws{iSre$S11BTMKk zW8u-8J+(u>MHLevg6~rrCoj{M+-T}go7btk9-Btv zz~lPQDB`WlY@Rg%e)eV|+gHSN_6i#^7@;sNNnEtfN*}KbufdbE+6u@o7n+I^WpUev zjQd7F@Vhr9tO8>fsF1tYWb9#aT{01}&1a77xOMRk=~0TYgvgs8%?>aN=x9TOE1s@0 z<{a-L{3{xd48wYA-C#%6fou6ogAtjE$}nSOgc@)2qj|#h7rP(fHh58y4;X zyPL-I5G`;Yv>7+`+}s2&S5MxiGR}DNS_X?d>p-r6}oF%F7|Qx*YRq zYWW^xR1)CVM1VjsjEHD?jx{HozZSOnGXZF7`M6})%){kb9{vn&Y@dj=|E%=6)R-8c zd}Sf0Cml^Ix;;lvw3Fr1?rD>{yBuvrMEGxljHxIu4%|O=>o6T)&y)6H%wPYj=CCV- zVjsH1gzvWH`q8U}sQA8KGDrjcp@klFM{wHF-#Q$O+a;SB)a29Q9|oen5E|Ry4!g^8 zv}v)JcQclZMLPqzZASVK9p5)v{GS+_Ln_SlrmH})qXdm!4jPFgB{=mZIT=%-An1ct zxoqhhm)(#H{FKoCZpOJAL4~hOi_`T&6{bnEC1+1Ir5J#`4m#Lre}=F=^giF5 z>uw5foZ~YW!(Xx846F9%h}?aU@soY|=;$4xo%G$75U26=gGYDiO*6qZAPB00TZ7P; zHQAdG#l)|?*H1@PTEV!C-}fYfP!j{E{Ao&_3!1vZF0FMzd2*be8Rvz_Wfp^kYeME+ zO5W>eW!D7>{WJ|7;1Fj6J>cxQO4c^AL0Xr;_=Rp7;j99y#0ri|?#$IFIM%5+p=vGY z8ESZ&FfB;OZQdqkDh;GuqPo0lHlO&LW*SsR*9{EZ0)0D}(aYnMI zr5Qpp^FK$wXcbzn8t*|9R3IZ{I645RzCfH&%xA>!3ymJT*B9a(xq5a;L ziy6QzUD;7Qu9t6|GITeha2 z-V};8d$~i=vyVxGysg||GUlrSc(cD!*JtS2ipd;gU!LZ+hHb#0 z!S}D)pAID>H!YUn==zw(TP5NM!&oGU z@-7e77Bm^D{>&8KRnMaIfjt(b{xF`B9NWjsg96khr@QIg!)`40i6Q7!!L>asF)mb2 z`aAI!Zad1dH4Pdj!)XvJEg^X8g-9SsZk~c=rKu@#Jem~VSDVe!Vk-{d(aqc$KU7p) zYNrvJ*)rulTqS1|V+2t}F1QuGqQfN~n`T|8=TXw`x-~a2O(WydZ)&J-lG(7=C?lMR!dF|KdoSX z1PwM!*z{WKi-$;G%8iOz+?zXzV~$G9eUl%{f8`)Xa@_AeHbV#P=X8!)GfZWF#lCFJ z*tBAKfHhcKd1SfRiQaxdi~t=3w)tT8kc~dxpOy=@xOx@>`KOmF#e<}e@Xs}2DSSd#a#TTw0f&oUB;UDX9hJb9*&uaE$-Bo{xMLM{&!UwxWM}; z!;5M$6}*h)M=+la!c<3DSj-Q3-IVm%S9-fJnPU-4-4i-XoEjj6D{EjGY>-3^_$r`u z)&6&XBc&dQV_9i>-O?)h9ZJi@2~uWeYbo4&LH*H(#L)9EjQ5qt9*wa_=)>=?gFAXt z{sy*%S+U8WN)Lr*+-xzTbSIqm+l%8<_=5zw>fmQvV(6)_9$_;s{s8Di2df&4_Vj25 zj;9w%91jvz{^|*lLmkbuOk>PjcnT-k{FO{=jE1cfV@vTjK8Bwf8&6AlZ`{q68~v|Q z_`eZ(&xH2y(0c3DvF<xa20bflC|9_^4GAvuM3rp70X2UlU`;huQkuQBJND&oP z>aJj!`OjmtN-cUZQ{g`#?`S00j$`eEMS3=)ym{KVUy`qx;-=Rxy>DM0T41s6<_|NRKawFMygjccuSy zSU@*uWjGp%`8P>=yYZn*PUdh6;pXmSsB{!m<89B7oV?G@qIiMQT9nY(nh(!toA zj<;bpmMd6*wqqxa%n01~$V~=RGQ69R=7FbUuGd*u)azW)CTe7Br|({cSpWi{tia=6 znoR)W0$$^HVrx`u8!rK9)&za_P5l5M1{VS_9T+*e0q8`0CG}+$6~oQLmik>Sr@*k! z`(fd+Ku^hw<%nZoJ$Ed9 zP2J6&IC$SAuZcFkq%5eVghALbGc)tv3fQ2wY2H8>(E$FB+mc(=o5Y!iL2Q^Nnn=a= zy_!J~a|Z4<0A!1|anZl5w(mYCGK2#LI5B9r!3~kB zq4z}-G+bSweEl;Z*89A(@ZV--ms=iLH6hlXd{6kYu-*wea4!%GPM;5#2q}Mxl7Mfu z%ieUx$e^pcYyaIqiO%#K3W%-y|@@QhBdF>c8X<1JBja`Jz9&jW3(9XfrF*p z`XpnJ#N0?#`x>+y%C$9$NwqZI;aNB!NlODXTnFKHrJav0nsNPtZ`Sf@LjWCvX)?+u zQH!3N2(UZPRq}C)sD)J3@p{1GjZ=c%El?mIz8vXw*LL~Du$jbjIf@1mU#z`nSUt(1 z3bFWf)BtlKHT+z&z~Gh}yb?vxWA>m~DyZE8H>jZ}LXqvz{5w)$XC%&tZy%bFknoe9 zxk+o!hsk<90V66EmqGqsNfuCiENsywjA9}YFdjfU|6-=Z2Ya#J&}De&_y%&(nbe%V z)`)%CeN3d`n`oVV2cukIg9bu;zL`ZhleDUS4+PXKeE{J|(*fpQgq@JZaV}1X_$QoE?v914nigwmS}dFH z1L1v^HV=M6%-{ipc>2Nbvy;QG$g9Pp#&}JNE1r+Lzn=%+y2jo~rtIFVrZFu&`2+{t zE)H@-%1tLP_ucROm)j$X+b`&<43mh-RO?Yd&T*`g6m}tJAIHF7Up+|AesY#gaNiZ5NTxJbg}5(i(V}PZ-2u*XJ|uu5`F-arewL z` zbV-BSOm$q$2dsp*)<8;RVZt@|gr@QOKJ@49%eWXXbS^mF91%`c{7fN5;GRQ8+QWjI z=Jt*Hqf#3y1JE*d)%e9W(ydLY_GkgOp3m-ZcMCvXnbF-3XesNC>D|8*hjo-s4jBEc z?`KYC9W|9XZcV z@?$}c!ucqP&Yx=1yx{;kjkSW-x5I7Jc!EK*k$zM>hbfCx%+ePITO{8~OafDv;2P&i zN3KY)J$URoN3f{QG%60c1c(6<_vEhq&ubZSR7=+zqF>u5wKYFM2NJ-CF4YLi5z=2d zQ8cb?IhWa5G|p_76l^!z>S= za=8_Uq2g?N+2IJLeQ|NoPg*}pVX9o8=Z~g}_*+_A?X<+HsjqSBTa8+R8JxVWYNU2z@6~aO{CdT2#be6zv~(R*b8^w4DO?{ z15JL_(rSfX_bk(e?S5qlSy}B^EY-gSQfLzMSfm_S31>NoHg;2UY_-*Uz>wlt zY6tKlBAzF7j3>~NmsB z!WWW=5W<4_YrCF=MAu=JWVYruca6HIc6n&uC0l2uO=^}10z!UM;(d5AstczLwBRnAR*mI3`j}C&@D)J$B>fJH8dzKE!`=NFfhcv_}%xj*0cYG z{c7IKTx-^~zIh(U`8lrtT8UcGt&0A+>g9RV`+*pHl;{`m1->`nCtI+p*|GwfFNw2e zS?tiWdzh7D@xIx-NW zN$hUN{(yEm==&8h7UK^vV8d70Q#zMZ$u^s~42E>Z_5@)@{_KN3*n{s^;C_=sK~kIV zLx#KXw=Xs6uG2a@R7k(z|J%L&C#R%G{(g~cVac^slEmLi2&G-0VENtn6qln_$!bkRu@%OeR;0PRERl5E_F6@(>^Ale zqJ+G3P_CKKAJ3xUo=1{|{UoNL(I?(D)`d%TkJ=A5W@6I}!!ZWLV!ol!g=XFZe!o!B zs#%^Ed?#Fj-Sb`n{kE;K%Gz3PIAFRZUwd8Du+aU+`0Q&Q3?`w{MH=<)a=kHiV6OAQ z&BK9Z5I6wo4H6!*zpLH`h>N6a?e$?3_Ceu?0D#aPguKYSNOmLTx)F{GU`0ZX!)b;& z{$QiOd~TbKQ@;cHa9G=x^J=xrR~}4^H>k<+cq~he_&)RN$4@d+jTM<=Obn1c+s4>( z6)Tx3Ia}~4k`<<;m8V%@0rFxgKYf>+m~nxl_f+O*UgjihFTCl66A1xz*1U$;zzyFn@NS)v`H<#ifcb>6$$z5Fb?2 z?X{sN{dx*GkkCl6tlu*H!57b=5kp9ziy?9!E=!i(;l%d@?Dg$tXe)1l`7SWXFNcb~ zLT9Uj4V+~A*Pd8-J~>sb2jyIHNyOXz;pl#QPSip)S)?hOV)1|Vx_nTR3*p=!qonhy za*xF$cLx(1cU=+2-up#?vpwhsky}GZ>R%P9mHwdInXXiaiW#_n_K&tE+4cEJ7v6kL zKSn%}oL4q|2Z}=uQaRAB_PYbsGZzghFd! zvx)qgd_@{QD$QeDTKZwQ&4<$&cHp*$@+oHn4}PzSaVWDY=UY3*)bqj1XWlHl#RFpu z|C-;=!pU(*v9!MDqBIa~N}$^@_an)sZv@u6_O{cu;MJ6;_62b}*j@~*>E8Nz`uBG$ zwSykWZR?r%D%ixjsng>b`@rmr6z?vxfD((cf zyeEv`CLiJ6=MwEv2o*l!frO3|~gFmf; zU&)Phgx?{i?APvp{m&`@bULiA`i2Mb!w65?3y5%5Y1hK60infDwt~KD!e``-zL4He zajm_7hp{PG*j!l5K%2L$?q-l2$H)!YJAbhPka>jpQkN$oeE-RskH8R!M?#9V*bJUl zgA>d&(y!BtLFbRZ+y35v12%Xt1z?6h34BZQYVc{>TK*!{i8;G$GKIezP=hyz{X;V+ z!<@BAV%9GVg*dNVO?EZ*n-lsrpV4#Mkq!l-)S?gueI-!(ZRZs|`*SmVh1&IVIL#*v zFaPu-M~ezUZmZ$bU@YpL8&eSFl8fKu4rxqDtK>}-a)g(Yc2i#c_JcCrHpf;@7dmoG z=z_E4dLcE9aOqu52o3KNGu-KH>UFB&#<^5Kp+^23wK<6~Gt*~wwe}YuiVdHyH1^7o z3&AlhrHfu3D@d-(;d#k?*j5ZpX`9$3&7u{%55Xr-Lf)4CLE}CBN>12@nCAL!T@Ib~ zmQL5kYk6!XS0>lvZ;&0#X6RUJj2pHmM~?jaiDtcPQcV$W2&U1&P4PhoGpMlyGq zC2u0zUa&>SM{}@^0l}9N0I(L-D*d=R zAoih(_GHiTQ`IV;mS=Fh-oIKa$!q?Z%9S}=ra4OU4UKab=GSwQz<_3HW^*> z)jh|^?YTL!=hVC5UoZC7e2`w8Q`cU@AFWTs*8Z}UOZrVFE{`4;C&U&pxCT2#-! zeDMOC zXEFAO0FA=snuTthmw8XYhs~f^c=PUV3r0lV;;iRhAb}tmOmxQY(@Ny#ScX^xzcbzO!c7Z8Ob6U48v9tJ;Fuwj zpXf^3U~BtBqT8MCv(jcRB7BIl*&cDnFK0s2MNy&*9?m%xQE<=M>V-SG7En2QzWFD- z-%pO~#>1GcB}_vqrqeAAXK~39SZpY?_NK)iexug8eXv(w&A9Z-a<*)_3x=O zPL+Q~YpjFxAvs~?w4?rLxzlAr8vRDoLu<-lPQX>lWv-&7unOV5hWwqx@TbmXs~}}a zSl=sQDhRJH&f!QGi!tb<)7nC@Ew3TP)GseGl^rpan@`UNRVV8!=ij6&-FAB}Uo?MM z{Z*K;`_*=8IYVz%vM!tKK1Q{V*STM95{jLRZ!65e7B}pj&Hb~QdzbR_M~lpt@0pE* ztS?tI+Wp_Wrp}@)n~7e$39am*&2GlL$29;K71jHegOIaPzG7_|iWcE$1#Ha{Uv%*4 z7OUvMoc_A^I=ph$7@%``qarC|g5N)!B>aAoB>7BWbO~9bGPzW*QxtTS+W5#C#{ckU zk0NSR@@oPiIbYKG`Lea>bgpf-}_{= zFTjnv?O)@TY_d!lHzV~eDp#(!k3cFp+t%qDVwQ1*vQ5*4MuQ;;v$B3hi~K_4;U