Tested the case where no email is provided in the h5 file. Fixed scicat login to try a different method if the first try fails (there is a discrepancy between the deployed scicat instance and the latest scicatlive version 3.2.5). Including two very small h5 files (with and without email) in the examples/ folder for future tests.

Author: davramov

examples/tomo_scan_email.h5

@@ -130,7 +130,7 @@ def upload_raw_dataset(
     file_name = scicat_metadata.get("/measurement/sample/file_name")
     description = build_search_terms(file_name)
     appended_keywords = description.split()
-
+    logger.info(f"email: {scicat_metadata.get('/measurement/sample/experimenter/email')}")
     dataset = RawDataset(
         owner=scicat_metadata.get("/measurement/sample/experiment/pi") or "Unknown",
         contactEmail=clean_email(scicat_metadata.get("/measurement/sample/experimenter/email"))
@@ -266,14 +266,58 @@ def _get_data_sample(file, sample_size=10):
     return data_sample
 
 
-def clean_email(email: str):
-    if email:
-        if not email or email.upper() == "NONE":
-            # this is a brutal case, but the beamline sometimes puts in "None" and
-            # the new scicat backend hates that.
-            return UNKNOWN_EMAIL
-        return email.replace(" ", "").replace(",", "").replace("'", "")
-    return None
+def clean_email(email: any) -> str:
+    """
+    Clean the provided email address.
+
+    This function ensures that the input is a valid email address.
+    It returns a default email if:
+      - The input is not a string,
+      - The input is empty after stripping,
+      - The input equals "NONE" (case-insensitive), or
+      - The input does not contain an "@" symbol.
+
+    Parameters
+    ----------
+    email : any
+        The raw email value extracted from metadata.
+
+    Returns
+    -------
+    str
+        A cleaned email address if valid, otherwise the default unknown email.
+
+    Example
+    -------
+    >>> clean_email("  user@example.com  ")
+    'user@example.com'
+    >>> clean_email("garbage")
+    'unknown@example.com'
+    >>> clean_email(None)
+    'unknown@example.com'
+    """
+    # Check that the email is a string
+    if not isinstance(email, str):
+        logger.info(f"Input email is not a string. Returning {UNKNOWN_EMAIL}")
+        return UNKNOWN_EMAIL
+
+    # Remove surrounding whitespace
+    cleaned = email.strip()
+
+    # Fallback if the email is empty, equals "NONE", or lacks an "@" symbol
+    if not cleaned or cleaned.upper() == "NONE" or "@" not in cleaned:
+        logger.info(f"Invalid email address. Returning {UNKNOWN_EMAIL}")
+        return UNKNOWN_EMAIL
+
+    # Optionally, remove spaces from inside the email (typically invalid in an email address)
+    cleaned = cleaned.replace(" ", "")
+
+    # Final verification: ensure that the cleaned email contains "@".
+    if "@" not in cleaned:
+        logger.info(f"Invalid email address: {cleaned}. Returning {UNKNOWN_EMAIL}")
+        return UNKNOWN_EMAIL
+
+    return cleaned
 
 
 scicat_metadata_keys = [
@@ -351,16 +395,32 @@ def clean_email(email: str):
 ]
 
 
+def test_ingest_raw_tomo() -> bool:
+    from orchestration.flows.scicat.ingest import ingest_dataset
+    TOMO_INGESTOR_MODULE = "orchestration.flows.bl832.ingest_tomo832"
+    file_path = "examples/tomo_scan_no_email.h5"
+    print(f"Ingesting {file_path} with {TOMO_INGESTOR_MODULE}")
+    try:
+        ingest_dataset(file_path, TOMO_INGESTOR_MODULE)
+        return True
+    except Exception as e:
+        print(f"SciCat ingest failed with {e}")
+        return False
+
+
 if __name__ == "__main__":
-    ingest(
-        ScicatClient(
-            # "http://localhost:3000/api/v3",
-            os.environ.get("SCICAT_API_URL"),
-            None,
-            os.environ.get("SCICAT_INGEST_USER"),
-            os.environ.get("SCICAT_INGEST_PASSWORD"),
-        ),
-        "/Users/dylanmcreynolds/data/beamlines/8.3.2/raw/20231013_065251_MSB_Book1_Proj77_Cell3_Gen2_Li_R2G_FastCharge_DuringCharge0.h5",
-        [],
-        log_level="DEBUG",
-    )
+    # ingest(
+    #     ScicatClient(
+    #         # "http://localhost:3000/api/v3",
+    #         os.environ.get("SCICAT_API_URL"),
+    #         None,
+    #         os.environ.get("SCICAT_INGEST_USER"),
+    #         os.environ.get("SCICAT_INGEST_PASSWORD"),
+    #     ),
+    #     "/Users/dylanmcreynolds/data/beamlines/8.3.2/raw/"
+    #     "20231013_065251_MSB_Book1_Proj77_Cell3_Gen2_Li_R2G_FastCharge_DuringCharge0.h5",
+    #     [],
+    #     log_level="DEBUG",
+    # )
+
+    test_ingest_raw_tomo()

examples/tomo_scan_no_email.h5

@@ -2,11 +2,12 @@
 import os
 from typing import List
 
-from pyscicat.client import from_credentials
+from pyscicat.client import ScicatClient, from_credentials
 from prefect import flow, task, get_run_logger
 
 from orchestration.flows.scicat.utils import Issue
 
+
 @flow(name="scicat_dataset_ingest")
 def ingest_dataset(file_path: str, ingestor: str):
     """ Ingest a file into SciCat.
@@ -23,7 +24,7 @@ def ingest_dataset(file_path: str, ingestor: str):
 
 @task(name="ingest_scicat")
 def ingest_dataset_task(file_path: str, ingestor_module: str):
-    """ Ingest a file into SciCat. 
+    """ Ingest a file into SciCat.
 
     Parameters
     ----------
@@ -33,14 +34,15 @@ def ingest_dataset_task(file_path: str, ingestor_module: str):
         Thy python module that contains the ingest function, e.g. "foo.bar.ingestor"
     """
     logger = get_run_logger()
+    import dotenv
+    dotenv.load_dotenv()
     SCICAT_API_URL = os.getenv("SCICAT_API_URL")
     SCICAT_INGEST_USER = os.getenv("SCICAT_INGEST_USER")
     SCICAT_INGEST_PASSWORD = os.getenv("SCICAT_INGEST_PASSWORD")
 
+    # files come in with the full pasth on the server that they
+    # were loaded from.
 
-    # files come in with the full pasth on the server that they 
-    # were loaded from. 
-    
     # relative path: raw/...
     # ingestor api maps /globa/cfs/cdirs/als/data_mover to /data_mover
     # so we want to prepend /data_mover/8.3.2
@@ -50,10 +52,37 @@ def ingest_dataset_task(file_path: str, ingestor_module: str):
     logger.info(
         f"Sending ingest job to {SCICAT_API_URL} for file {file_path}"
     )
-    scicat_client = from_credentials(
-        SCICAT_API_URL,
-        SCICAT_INGEST_USER,
-        SCICAT_INGEST_PASSWORD)
+    try:
+        scicat_client = from_credentials(
+            SCICAT_API_URL,
+            SCICAT_INGEST_USER,
+            SCICAT_INGEST_PASSWORD)
+    except Exception as e:
+        logger.warning(f"Failed to create SciCat client using pyscicat method: {e}")
+
+    try:
+        import requests
+        from urllib.parse import urljoin
+
+        url = urljoin(SCICAT_API_URL, "auth/login")
+        logger.info(url)
+        response = requests.post(
+            url=url,
+            json={"username": SCICAT_INGEST_USER, "password": SCICAT_INGEST_PASSWORD},
+            stream=False,
+            verify=True,
+        )
+        logger.info(f"Login response: {response.json()}")
+        scicat_client = ScicatClient(SCICAT_API_URL, response.json()["access_token"])
+        logger.info("Logged in to SciCat.")
+
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Failed to log in to SciCat: {e}")
+        raise e
+    except Exception as e:
+        logger.error(f"Failed to log in to SciCat: {e}")
+        raise e
+
     ingestor_module = importlib.import_module(ingestor_module)
     issues: List[Issue] = []
     new_dataset_id = ingestor_module.ingest(
@@ -74,4 +103,4 @@ def ingest_dataset_task(file_path: str, ingestor_module: str):
 
     from dotenv import load_dotenv
     load_dotenv()
-    ingest_dataset(sys.argv[1], sys.argv[2])
+    ingest_dataset(sys.argv[1], sys.argv[2])