From c272250e5e15c24228cfcdcb68af03881d6cbba5 Mon Sep 17 00:00:00 2001 From: HL1 Date: Sun, 31 May 2020 17:59:51 -0500 Subject: [PATCH 1/2] added the config system --- app/config.yml | 11 +++++++ app/utils/config.py | 78 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 app/config.yml create mode 100644 app/utils/config.py diff --git a/app/config.yml b/app/config.yml new file mode 100644 index 0000000..8be9f12 --- /dev/null +++ b/app/config.yml @@ -0,0 +1,11 @@ +app_name: OpenFDA +secret_key: a passphrase for peppering +base_dir: +app_dir: /app +data_dir: /data +log_dir: /log +fda_data_url: https://download.open.fda.gov/drug/ndc/drug-ndc-0001-of-0001.json.zip +server: + host: 0.0.0.0 +sqllite_db_name: drugs.db +json_file_name: \ No newline at end of file diff --git a/app/utils/config.py b/app/utils/config.py new file mode 100644 index 0000000..1517b7c --- /dev/null +++ b/app/utils/config.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +"""apps.utils.config: imports and finalize config.yml""" + +from pathlib import Path +import re +import glob +import os + +# third-party +import yaml + +class Config: + """ + a class that: + 1) loads yml file on initialization + 2) allows injecting python code to the config dictionary + 3) replaces yml references with their values + """ + def __init__(self,value=Path(__file__).resolve(strict=True).parent.parent.absolute() / "config.yml"): + # config.yml location + self.yml_config_path = value + self.configs = {} + self.load_yaml() + + def load_yaml(self): + """ + puts config.yml content in a dictionary + """ + self.configs = yaml.safe_load(open(self.yml_config_path)) + + def replace_dynamic(self,key,new_value): + """ + replace any mention of for the key specificed with a new value + """ + self.configs[key] = new_value if self.configs[key] == '' else self.configs[key] + + def replace_references(self): + """ + replace any mention of with its value + """ + # loop through each key value pair and replace with their values. + for key, value in self.configs.items(): + if isinstance(value, str): + vars_to_replace = re.findall(r'(?=\<[A-z]+\>)(?!\)([A-z><]+)',value) + for var in vars_to_replace: + if isinstance(self.configs[var[1:-1]], Path): + self.configs[key] = Path(value.replace(var,str(self.configs[var[1:-1]]))) + # make directories if they do not exist + Path(value.replace(var,str(self.configs[var[1:-1]]))).mkdir(parents=True, exist_ok=True) + else: + self.configs[key] = value.replace(var,self.configs[var[1:-1]]) + + def get(self,key): + """ + return the value for they key requested + """ + return self.configs[key] + + def all(self): + """ + returns all configs + """ + return self.configs + +# initiate config +config = Config() +# get the realtive path and assign it to base_dir +config.replace_dynamic('base_dir',Path(__file__).resolve(strict=True).parent.parent.parent.absolute()) +# subsitute all references with their values +config.replace_references() +# get the latest json file +config.replace_dynamic('json_file_name',os.path.basename(max(glob.glob(str(config.get('data_dir')) + '/*.json'), key=os.path.getctime))) + + +if __name__ == "__main__": + print(config.all()) +else: + config = config.all() \ No newline at end of file From 492fb3eb1e6b42bd38820d74f77cdf4c8af70289 Mon Sep 17 00:00:00 2001 From: HL1 Date: Mon, 1 Jun 2020 06:26:21 -0500 Subject: [PATCH 2/2] rewrote config.py to use yaml standards --- app/config.yml | 10 +++--- app/utils/config.py | 88 ++++++++++++++++++--------------------------- 2 files changed, 39 insertions(+), 59 deletions(-) diff --git a/app/config.yml b/app/config.yml index 8be9f12..5b213b6 100644 --- a/app/config.yml +++ b/app/config.yml @@ -1,11 +1,11 @@ app_name: OpenFDA secret_key: a passphrase for peppering -base_dir: -app_dir: /app -data_dir: /data -log_dir: /log +base_dir: !PY base_dir +app_dir: !PY app_dir +data_dir: !PY data_dir +log_dir: !PY log_dir fda_data_url: https://download.open.fda.gov/drug/ndc/drug-ndc-0001-of-0001.json.zip server: host: 0.0.0.0 sqllite_db_name: drugs.db -json_file_name: \ No newline at end of file +json_file_name: !PY json_file_name \ No newline at end of file diff --git a/app/utils/config.py b/app/utils/config.py index 1517b7c..36c255e 100644 --- a/app/utils/config.py +++ b/app/utils/config.py @@ -2,77 +2,57 @@ """apps.utils.config: imports and finalize config.yml""" from pathlib import Path -import re import glob -import os - -# third-party import yaml +import os -class Config: +class PyVar(yaml.YAMLObject): """ - a class that: - 1) loads yml file on initialization - 2) allows injecting python code to the config dictionary - 3) replaces yml references with their values + a class to create a custom !PY tag in yaml + The !PY tag allows calling python code stored in a dictionary """ - def __init__(self,value=Path(__file__).resolve(strict=True).parent.parent.absolute() / "config.yml"): - # config.yml location - self.yml_config_path = value - self.configs = {} - self.load_yaml() + # this is a PYYAML requirment and is case sensitive + yaml_tag = u'!PY' - def load_yaml(self): + def __init__(self): """ - puts config.yml content in a dictionary + starts with empty dictionary """ - self.configs = yaml.safe_load(open(self.yml_config_path)) + self.py_dict = {} - def replace_dynamic(self,key,new_value): + def get(self,key): """ - replace any mention of for the key specificed with a new value + retrieves a value from the dictionary """ - self.configs[key] = new_value if self.configs[key] == '' else self.configs[key] + return self.py_dict[key] - def replace_references(self): + def add(self,key,value): """ - replace any mention of with its value + adds a new key value pair to a dictionary """ - # loop through each key value pair and replace with their values. - for key, value in self.configs.items(): - if isinstance(value, str): - vars_to_replace = re.findall(r'(?=\<[A-z]+\>)(?!\)([A-z><]+)',value) - for var in vars_to_replace: - if isinstance(self.configs[var[1:-1]], Path): - self.configs[key] = Path(value.replace(var,str(self.configs[var[1:-1]]))) - # make directories if they do not exist - Path(value.replace(var,str(self.configs[var[1:-1]]))).mkdir(parents=True, exist_ok=True) - else: - self.configs[key] = value.replace(var,self.configs[var[1:-1]]) + self.py_dict[key] = value - def get(self,key): - """ - return the value for they key requested - """ - return self.configs[key] - def all(self): - """ - returns all configs - """ - return self.configs +# construct all they key value pairs that can be referenced by the yaml file +# creating a function like this is required for yaml.SafeLoader.add_constructor +def yaml_construct(loader,node): + c = PyVar() + ## start adding python variables here + c.add('base_dir',Path(__file__).resolve(strict=True).parent.parent.parent.absolute()) + c.add('app_dir', c.get('base_dir') / 'app') + c.add('data_dir', c.get('app_dir') / 'data') + c.add('log_dir', c.get('app_dir') / 'log') + c.add('json_file_name',os.path.basename(max(glob.glob(str(c.get('data_dir')) + '/*.json'), key=os.path.getctime))) + return c.get(node.value) + +# Required for safe_load +yaml.SafeLoader.add_constructor('!PY', yaml_construct) -# initiate config -config = Config() -# get the realtive path and assign it to base_dir -config.replace_dynamic('base_dir',Path(__file__).resolve(strict=True).parent.parent.parent.absolute()) -# subsitute all references with their values -config.replace_references() -# get the latest json file -config.replace_dynamic('json_file_name',os.path.basename(max(glob.glob(str(config.get('data_dir')) + '/*.json'), key=os.path.getctime))) +# config.yml location +configyml_path = Path(__file__).resolve(strict=True).parent.parent.absolute() / "config.yml" +# load config.yml +config = yaml.safe_load(open(configyml_path)) if __name__ == "__main__": - print(config.all()) -else: - config = config.all() \ No newline at end of file + print(config) \ No newline at end of file