New model-building pipeline

config/config.py.template

@@ -58,40 +58,48 @@ settings = {
             "SSH_USER": "WCER AD User",
             "SSH_PASS": "WCER AD Password",
             "SSH_PORT": 22
+        },
+        "LOCAL_FILE": {
+
         }
     },
     "GAME_SOURCE_MAP":
     {
-        "ALL_YOU_CAN_ET":       { "source":"OPENGAMEDATA_BQ",    "database":"all_you_can_et",       "table":"all_you_can_et_daily",       "schema":"OPENGAMEDATA_BIGQUERY" },
-        "AQUALAB":              { "source":"OPENGAMEDATA_BQ",    "database":"aqualab",              "table":"aqualab_daily",              "schema":"OPENGAMEDATA_BIGQUERY" },
-        "BACTERIA":             { "source":"OPENGAMEDATA_BQ",    "database":"bacteria",             "table":"bacteria_daily",             "schema":"OPENGAMEDATA_BIGQUERY" },
-        "BALLOON":              { "source":"OPENGAMEDATA_BQ",    "database":"balloon",              "table":"balloon_daily",              "schema":"OPENGAMEDATA_BIGQUERY" },
-        "BLOOM":                { "source":"OPENGAMEDATA_BQ",    "database":"bloom",                "table":"bloom_daily",                "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CENSIO_MATCH":         { "source":"OPENGAMEDATA_BQ",    "database":"censio_match",         "table":"censio_match_daily",         "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CENSIO_SLIDE":         { "source":"OPENGAMEDATA_BQ",    "database":"censio_slide",         "table":"censio_slide_daily",         "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CENSIO_STACK":         { "source":"OPENGAMEDATA_BQ",    "database":"censio_stack",         "table":"censio_stack_daily",         "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CRUSH_STATION":        { "source":"OPENGAMEDATA_BQ",    "database":"crush_station",        "table":"crush_station_daily",        "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CRYSTAL":              { "source":"OPENGAMEDATA_BQ",    "database":"crystal",              "table":"crystal_daily",              "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CYCLE_CARBON":         { "source":"OPENGAMEDATA_BQ",    "database":"cycle_carbon",         "table":"cycle_carbon_daily",         "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CYCLE_NITROGEN":       { "source":"OPENGAMEDATA_BQ",    "database":"cycle_nitrogen",       "table":"cycle_nitrogen_daily",       "schema":"OPENGAMEDATA_BIGQUERY" },
-        "CYCLE_WATER":          { "source":"OPENGAMEDATA_BQ",    "database":"cycle_water",          "table":"cycle_water_daily",          "schema":"OPENGAMEDATA_BIGQUERY" },
-        "EARTHQUAKE":           { "source":"OPENGAMEDATA_BQ",    "database":"earthquake",           "table":"earthquake_daily",           "schema":"OPENGAMEDATA_BIGQUERY" },
-        "GWAKKAMOLE":           { "source":"OPENGAMEDATA_BQ",    "database":"gwakkamole",           "table":"earthquake_daily",           "schema":"OPENGAMEDATA_BIGQUERY" },
-        "ICECUBE":              { "source":"OPENGAMEDATA_BQ",    "database":"icecube",              "table":"icecube_daily",              "schema":"OPENGAMEDATA_BIGQUERY" },
-        "JOURNALISM":           { "source":"OPENGAMEDATA_BQ",    "database":"journalism",           "table":"journalism_daily",           "schema":"OPENGAMEDATA_BIGQUERY" },
-        "JOWILDER":             { "source":"OPENGAMEDATA_BQ",    "database":"jowilder",             "table":"jowilder_daily",             "schema":"OPENGAMEDATA_BIGQUERY" },
-        "LAKELAND":             { "source":"OPENGAMEDATA_BQ",    "database":"lakeland",             "table":"lakeland_daily",             "schema":"OPENGAMEDATA_BIGQUERY" },
-        "MAGNET":               { "source":"OPENGAMEDATA_BQ",    "database":"magnet",               "table":"magnet_daily",               "schema":"OPENGAMEDATA_BIGQUERY" },
-        "MASHOPOLIS":           { "source":"OPENGAMEDATA_BQ",    "database":"mashopolis",           "table":"mashopolis_daily",           "schema":"OPENGAMEDATA_BIGQUERY" },
-        "PENGUINS":             { "source":"OPENGAMEDATA_BQ",    "database":"penguins",             "table":"penguins_daily",             "schema":"OPENGAMEDATA_BIGQUERY" },
+        "ALL_YOU_CAN_ET":       {
+            "events_from" : { "source":"OPENGAMEDATA_BQ",       "table_schema":"OPENGAMEDATA_BIGQUERY" ,   "database":"all_you_can_et",       "table":"all_you_can_et_daily" },
+            "events_to"   : { "source":"OPENGAMEDATA_BQ",       "table_schema":"OPENGAMEDATA_BIGQUERY",    "database":"all_you_can_et",       "table":"all_you_can_et_daily" },
+            "feats_from"  : { "source":"OPENGAMEDATA_BQ",       "table_schema":"OPENGAMEDATA_BIGQUERY",    "database":"all_you_can_et",       "table":"all_you_can_et_daily" },
+            "feats_to"    : { "source":"OPENGAMEDATA_BQ",       "table_schema":"OPENGAMEDATA_BIGQUERY",    "database":"all_you_can_et",       "table":"all_you_can_et_daily" },
+        },
+        "AQUALAB":              { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"aqualab",              "table":"aqualab_daily" },
+        "BACTERIA":             { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"bacteria",             "table":"bacteria_daily" },
+        "BALLOON":              { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"balloon",              "table":"balloon_daily" },
+        "BLOOM":                { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"bloom",                "table":"bloom_daily" },
+        "CENSIO_MATCH":         { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"censio_match",         "table":"censio_match_daily" },
+        "CENSIO_SLIDE":         { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"censio_slide",         "table":"censio_slide_daily" },
+        "CENSIO_STACK":         { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"censio_stack",         "table":"censio_stack_daily" },
+        "CRUSH_STATION":        { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"crush_station",        "table":"crush_station_daily" },
+        "CRYSTAL":              { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"crystal",              "table":"crystal_daily" },
+        "CYCLE_CARBON":         { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"cycle_carbon",         "table":"cycle_carbon_daily" },
+        "CYCLE_NITROGEN":       { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"cycle_nitrogen",       "table":"cycle_nitrogen_daily" },
+        "CYCLE_WATER":          { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"cycle_water",          "table":"cycle_water_daily" },
+        "EARTHQUAKE":           { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"earthquake",           "table":"earthquake_daily" },
+        "GWAKKAMOLE":           { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"gwakkamole",           "table":"earthquake_daily" },
+        "ICECUBE":              { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"icecube",              "table":"icecube_daily" },
+        "JOURNALISM":           { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"journalism",           "table":"journalism_daily" },
+        "JOWILDER":             { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"jowilder",             "table":"jowilder_daily" },
+        "LAKELAND":             { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"lakeland",             "table":"lakeland_daily" },
+        "MAGNET":               { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"magnet",               "table":"magnet_daily" },
+        "MASHOPOLIS":           { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"mashopolis",           "table":"mashopolis_daily" },
+        "PENGUINS":             { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"penguins",             "table":"penguins_daily" },
         "PENNYCOOK":            { "source":"OPENGAMEDATA_BQ",    "database":"pennycook",            "table":"pennycook_daily",            "schema":"OPENGAMEDATA_BIGQUERY" },
-        "SHADOWSPECT":          { "source":"OPENGAMEDATA_BQ",    "database":"shadowspect",          "table":"shadowspect_daily",          "schema":"OPENGAMEDATA_BIGQUERY" },
-        "SHIPWRECKS":           { "source":"SHIPWRECKS_BQ",      "database":"analytics_269167605",  "table":"events",                     "schema":"FIREBASE"              },
-        "THERMOVR":             { "source":"OPENGAMEDATA_BQ",    "database":"thermovr",             "table":"thermovr_daily",             "schema":"OPENGAMEDATA_BIGQUERY" },
-        "THERMOLAB":            { "source":"OPENGAMEDATA_BQ",    "database":"thermolab",            "table":"thermolab_daily",            "schema":"OPENGAMEDATA_BIGQUERY" },
-        "TRANSFORMATION_QUEST": { "source":"OPENGAMEDATA_BQ",    "database":"transformation_quest", "table":"transformation_quest_daily", "schema":"OPENGAMEDATA_BIGQUERY" },
-        "WAVES":                { "source":"OPENGAMEDATA_BQ",    "database":"waves",                "table":"waves_daily",                "schema":"OPENGAMEDATA_BIGQUERY" },
-        "WEATHER_STATION":      { "source":"OPENGAMEDATA_BQ",    "database":"weather_station",      "table":"weather_station_daily",      "schema":"OPENGAMEDATA_BIGQUERY" },
-        "WIND":                 { "source":"OPENGAMEDATA_BQ",    "database":"wind",                 "table":"wind_daily",                 "schema":"OPENGAMEDATA_BIGQUERY" },
+        "SHADOWSPECT":          { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"shadowspect",          "table":"shadowspect_daily" },
+        "SHIPWRECKS":           { "source":"SHIPWRECKS_BQ",      "schema":"FIREBASE",               "database":"analytics_269167605",  "table":"events" },
+        "THERMOVR":             { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"thermovr",             "table":"thermovr_daily" },
+        "THERMOLAB":            { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"thermolab",            "table":"thermolab_daily" },
+        "TRANSFORMATION_QUEST": { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"transformation_quest", "table":"transformation_quest_daily" },
+        "WAVES":                { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"waves",                "table":"waves_daily" },
+        "WEATHER_STATION":      { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"weather_station",      "table":"weather_station_daily" },
+        "WIND":                 { "source":"OPENGAMEDATA_BQ",    "schema":"OPENGAMEDATA_BIGQUERY",  "database":"wind",                 "table":"wind_daily" },
     },
 }

main.py

@@ -1,12 +1,11 @@
 # import standard libraries
 import cProfile
 import pstats
-#import datetime
-import argparse
 import os
 import sys
 from argparse import Namespace
 from pathlib import Path
+from typing import List
 
 # import 3rd-party libraries
 
@@ -18,30 +17,29 @@
 from src.ogd import games
 from src.ogd.core.exec.Commands import OGDCommands
 from src.ogd.core.exec.Parsers import OGDParsers
-from src.ogd.core.schemas.configs.ConfigSchema import ConfigSchema
+from src.ogd.core.configs.CoreConfig import CoreConfig
 
 
 ## This section of code is what runs main itself. Just need something to get it
 #  started.
-config = ConfigSchema(name="config.py", all_elements=settings)
+config = CoreConfig.FromDict(name="config.py", unparsed_elements=settings)
 Logger.InitializeLogger(level=config.DebugLevel, use_logfile=config.UseLogFile)
 # Logger.Log(f"Running {sys.argv[0]}...", logging.INFO)
 # set up parent parsers with arguments for each class of command
-games_folder : Path = Path(games.__file__) if Path(games.__file__).is_dir() else Path(games.__file__).parent
-games_list = [name.upper() for name in os.listdir(games_folder) if (os.path.isdir(games_folder / name) and name != "__pycache__")]
+games_folder : Path      = Path(games.__file__) if Path(games.__file__).is_dir() else Path(games.__file__).parent
+games_list   : List[str] = [name.upper() for name in os.listdir(games_folder) if (os.path.isdir(games_folder / name) and name != "__pycache__")]
 # set up main parser, with one sub-parser per-command.
 parser = OGDParsers.CommandParser(games_list=games_list)
 
 args : Namespace = parser.parse_args()
 
-
 success : bool
 if config.WithProfiling:
     profiler = cProfile.Profile()
     profiler.enable()
 if args is not None:
     cmd = (args.command or "help").lower()
-    dest = Path(args.destination if args.destination != "" else "./") if 'destination' in args else config.DataDirectory
+    dest = Path(args.destination if args.destination != "" else config.DataDirectory) if 'destination' in args else config.DataDirectory
     match cmd:
         case "export":
             success = OGDCommands.RunExport(args=args, config=config, destination=dest, with_events=True, with_features=True)

requirements.txt

@@ -1,3 +1,3 @@
 numpy==2.1.*
 pandas==2.2.*
-opengamedata-common==1.2.*
+opengamedata-common==2.0.*