Ecs processing

dev/DEV_GARPOS_QUIRKS_DEMO.py

@@ -70,7 +70,7 @@ def main():
             "cutoff_elevation": 7,}
     }
 
-    override_survey_parsing = False
+    override_survey_parsing = True
     override_garpos_parsing = False
 
     # station = "NCC1"
@@ -95,12 +95,12 @@ def main():
     # mid_processer.midprocess_sync_s3()
 
 
-    stations = ['NTH1','NCC1','NBR1','GCC1']
+    stations = ['GCC1']
     for station in stations:
         wfh.set_network_station_campaign(network_id=network, station_id=station, campaign_id=campaign)
         wfh.midprocess_parse_surveys(override=override_survey_parsing,write_intermediate=True)
-        mid_processer = wfh.midprocess_get_processor()
-        mid_processer.prepare_shotdata_garpos()
+        #mid_processer = wfh.midprocess_get_processor()
+        #mid_processer.prepare_shotdata_garpos(overwrite=True)
 
     # wfh.midprocess_parse_surveys(override=override_survey_parsing,write_intermediate=True)
     # wfh.midprocess_prep_garpos(override=override_garpos_parsing,custom_filters=filter_config,survey_id="2025_A_1126_1")

dev/dev_ecs.py

@@ -0,0 +1,50 @@
+# download, preprocess, and postprocess SV3 data
+import os
+from pathlib import Path
+
+os.environ["GARPOS_PATH"] = str(Path.home() / "Project/SeaFloorGeodesy" / "garpos")
+os.environ["WORKING_ENVIRONMENT"] = "ECS"
+os.environ["S3_SYNC_BUCKET"] = "seafloor-public-bucket-bucket83908e77-gprctmuztrim"
+os.environ["MAIN_DIRECTORY_ECS"] = "/Volumes/DunbarSSD/Project/SeafloorGeodesy/ECSDEMO"
+os.environ["DYLD_LIBRARY_PATH"] = (
+    os.environ.get("CONDA_PREFIX", "")
+    + "/lib:"
+    + os.environ.get("DYLD_LIBRARY_PATH", "")
+)
+PRIDE_DIR = Path.home() / ".PRIDE_PPPAR_BIN"
+os.environ["PATH"] += os.pathsep + str(PRIDE_DIR)
+from es_sfgtools.config.env_config import Environment, WorkingEnvironment
+Environment.load_working_environment()
+from es_sfgtools.workflows.workflow_handler import WorkflowHandler
+from es_sfgtools.workflows.midprocess import IntermediateDataProcessor
+from es_sfgtools.workflows.pipelines.sv3_pipeline import SV3PipelineECS
+from es_sfgtools.data_mgmt.assetcatalog.schemas import AssetEntry
+
+def main():
+    wfh:WorkflowHandler = WorkflowHandler()
+    network = "cascadia-community"
+    station = "NSS1"
+    campaign = "2024_A"
+    wfh.set_network_station_campaign(
+        network_id=network, station_id=station, campaign_id=campaign
+    )
+    pl = SV3PipelineECS(
+        directory_handler=wfh.directory_handler,
+        asset_catalog=wfh.asset_catalog,
+    )
+    pl.set_network_station_campaign(
+        network_id=network, station_id=station, campaign_id=campaign
+
+    )
+
+    #pl.config.rinex_config.override = True
+    #pl.get_rinex_files()
+    # wfh.ingest_catalog_archive_data()
+    # wfh.ingest_download_archive_data()
+    #pl.process_rinex()
+    #pl.update_shotdata()
+    dataPostProcessor: IntermediateDataProcessor = wfh.midprocess_get_processor(override_metadata_require=True)
+    dataPostProcessor.parse_surveys()
+
+if __name__ == "__main__":
+    main()

linux_environment.yml

@@ -58,5 +58,6 @@ dependencies:
     - gnatss
     - regex
     - cloudpathlib[s3]
+    - hatanaka
 variables:
   CGO_ENABLED: 1

mac_environment.yml

@@ -53,5 +53,6 @@ dependencies:
     - gnatss
     - regex
     - cloudpathlib[s3]
+    - hatanaka
 variables:
   CGO_ENABLED: 1

src/es_sfgtools/._version.py

@@ -0,0 +1 @@
+version = "0.5.3"

src/es_sfgtools/__init__.py

@@ -1,3 +1,3 @@
 # src/es_sfgtools/__init__.py
 
-from ._version import version as __version__
+__version__ = "0.5.3"

src/es_sfgtools/config/env_config.py

@@ -14,12 +14,13 @@ class WorkingEnvironment(Enum):
 
     LOCAL = "LOCAL"
     GEOLAB = "GEOLAB"
+    ECS = "ECS"
 
 
 WORKING_ENVIRONMENT_KEY = "WORKING_ENVIRONMENT"
 S3_SYNC_BUCKET_KEY = "S3_SYNC_BUCKET"
 MAIN_DIRECTORY_GEOLAB_KEY = "MAIN_DIRECTORY_GEOLAB"
-
+MAIN_DIRECORY_ECS_KEY = "MAIN_DIRECTORY_ECS"
 
 class Environment:
     """
@@ -32,6 +33,7 @@ class methods.
     _working_environment: WorkingEnvironment = WorkingEnvironment.LOCAL
     _s3_sync_bucket: str | None = None
     _main_directory_GEOLAB: str | None = None
+    _main_directory_ECS: str | None = None
 
     @classmethod
     def working_environment(cls) -> WorkingEnvironment:
@@ -48,6 +50,11 @@ def main_directory_GEOLAB(cls) -> str | None:
         """Returns the main directory for the GEOLAB environment, if configured."""
         return cls._main_directory_GEOLAB
 
+    @classmethod
+    def main_directory_ECS(cls) -> str | None:
+        """Returns the main directory for the ECS environment, if configured."""
+        return cls._main_directory_ECS
+
     @classmethod
     def load_working_environment(cls) -> None:
         """
@@ -74,6 +81,15 @@ def load_working_environment(cls) -> None:
                         f"{MAIN_DIRECTORY_GEOLAB_KEY} environment variable must be set in GEOLAB environment."
                     )
                 cls._main_directory_GEOLAB = md_geolab
+
+            case "ECS":
+                cls._working_environment = WorkingEnvironment.ECS
+                md_ecs = os.environ.get(MAIN_DIRECORY_ECS_KEY)
+                if md_ecs is None:
+                    raise ValueError(
+                        f"{MAIN_DIRECORY_ECS_KEY} environment variable must be set in ECS environment."
+                    )
+                cls._main_directory_ECS = md_ecs
             case _:
                 raise ValueError(
                     f"Unknown WORKING_ENVIRONMENT: {env_str}. "