Config: Combine Sources & Destinations (#79)

Closes #75

We use `data_sources` as a single list in place of sources and
destinations. its up to the user to provide the correct credentials.

Author: bh2smith

README.md

@@ -6,24 +6,16 @@
 
 Configuration is provided in a single YAML file. Refer to the existing `config.yaml` for an overview.
 The configuration file consists of three main sections:
-- `sources`: Defines available data sources
-- `destinations`: Defines available data destinations
+- `data_sources`: Defines available databases
 - `jobs`: Defines synchronization jobs that connect sources to destinations
 
-#### Source Definitions
+#### Data Source Definitions
 
 Sources are defined as a list of configurations, each containing:
 - `name`: String. A unique identifier for referencing this source in jobs
 - `type`: String. Must be either `dune` or `postgres`
 - `key`: String. Connection details, supports environment variable templating using `${VAR_NAME}` syntax such as `${DB_URL}` or `${DUNE_API_KEY}` ([see environment setup](#define-environment))
 
-#### Destination Definitions
-
-Destinations are defined similarly to sources:
-- `name`: String. A unique identifier for referencing this destination in jobs
-- `type`: String. Must be either `dune` or `postgres`
-- `key`: String. Connection details, supports environment variable templating using `${VAR_NAME}` syntax
-
 #### Job Parameters
 
 Each job in the `jobs` list should contain:

config.yaml

@@ -1,23 +1,15 @@
-sources:
-  - name: Dune1
+data_sources:
+  - name: Dune
     type: dune
     key: ${DUNE_API_KEY}
-  - name: PG1
-    type: postgres
-    key: ${DB_URL}
-
-destinations:
-  - name: Dune1
-    type: dune
-    key: ${DUNE_API_KEY}
-  - name: PG1
+  - name: PG
     type: postgres
     key: ${DB_URL}
 
 jobs:
   - name: Sync Parameterized Dune Query with Multiple Types to Postgres
     source:
-      ref: Dune1
+      ref: Dune
       query_id: 4238114
       query_engine: medium
       poll_frequency: 5
@@ -32,16 +24,16 @@ jobs:
           value: "10"
           type: NUMBER
     destination:
-      ref: PG1
+      ref: PG
       table_name: results_4238114
       if_exists: replace
 
   - name: Table Independent Query to Dune
     source:
-      ref: PG1
+      ref: PG
       query_string: "SELECT 1 as number, '\\x1234'::bytea as my_bytes;"
     destination:
-      ref: Dune1
+      ref: Dune
       table_name: dune_sync_test_table
 
   - name: Sync Parameterized Dune Query with Multiple Types to Postgres

src/config.py

@@ -189,23 +189,16 @@ def load_from_yaml(cls, file_path: Path | str = "config.yaml") -> RuntimeConfig:
         with open(file_path, "rb") as _handle:
             data = yaml.safe_load(_handle)
 
-        # Load sources map
+        # Load data sources map
         sources = {}
-        for source in data.get("sources", []):
-            sources[str(source["name"])] = DbRef.from_dict(source)
-
-        # Load destinations map
-        destinations = {}
-        for destination in data.get("destinations", []):
-            destinations[str(destination["name"])] = DbRef.from_dict(destination)
+        for db_ref in data.get("data_sources", []):
+            sources[str(db_ref["name"])] = DbRef.from_dict(db_ref)
 
         jobs = []
         for job_config in data.get("jobs", []):
             source = cls._build_source(job_config["source"], sources)
-            destination = cls._build_destination(
-                job_config["destination"], destinations
-            )
-            jobs.append(Job(source, destination))
+            dest = cls._build_destination(job_config["destination"], sources)
+            jobs.append(Job(source, dest))
 
         return cls(jobs=jobs)
 

tests/fixtures/config/basic.yaml

@@ -1,20 +1,12 @@
 ---
-sources:
+data_sources:
   - name: dune
     type: dune
     key: ${DUNE_API_KEY}
   - name: postgres
     type: postgres
     key: ${DB_URL}
 
-destinations:
-  - name: postgres
-    type: postgres
-    key: ${DB_URL}
-  - name: dune
-    type: dune
-    key: ${DUNE_API_KEY}
-
 jobs:
   - name: Download simple test query to local postgres
     source:

tests/fixtures/config/buggy.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: dune
     type: dune
     key: fake-key
-
-destinations:
   - name: postgres
     type: postgres
     key: fake-pg-key

tests/fixtures/config/dune_to_postgres.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: dune
     type: dune
     key: fake-key
-
-destinations:
   - name: postgres
     type: postgres
     key: postgresql://postgres:postgres@localhost:5432/postgres

tests/fixtures/config/invalid_sql_file.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: postgres
     type: postgres
     key: ${DB_URL}
-
-destinations:
   - name: dune
     type: dune
     key: fake-pg-key

tests/fixtures/config/postgres_to_dune.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: dune
     type: dune
     key: fake-key
-
-destinations:
   - name: postgres
     type: postgres
     key: fake-pg-key

tests/fixtures/config/sql_file.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: postgres
     type: postgres
     key: ${DB_URL}
-
-destinations:
   - name: dune
     type: dune
     key: fake-key

tests/fixtures/config/unsupported_dest.yaml

@@ -1,10 +1,8 @@
 ---
-sources:
+data_sources:
   - name: dune
     type: dune
     key: fake-key
-
-destinations:
   - name: sqlite
     type: sqlite
     key: fake-pg-key