diff --git a/src/database.ini b/src/database.ini new file mode 100644 index 0000000..57a9049 --- /dev/null +++ b/src/database.ini @@ -0,0 +1,6 @@ +[postgresql] +host=localhost +database=pokemon +user=postgres +password=postgres +port = 5432 diff --git a/src/extract_data.py b/src/extract_data.py index e69de29..a0aef20 100644 --- a/src/extract_data.py +++ b/src/extract_data.py @@ -0,0 +1,3 @@ +from utils import extract_to_csv + +extract_to_csv() diff --git a/src/load.py b/src/load.py index e69de29..3af1a26 100644 --- a/src/load.py +++ b/src/load.py @@ -0,0 +1,13 @@ +from utils import save_in_db +from sqlalchemy import create_engine +import configparser + +config = configparser.ConfigParser() +config.read('database.ini') +db_params = config['postgresql'] + +db_url = f"postgresql://{db_params['user']}:{db_params['password']}@{db_params['host']}:{db_params['port']}/{db_params['database']}" +engine = create_engine(db_url, echo=True) + +save_in_db() + diff --git a/src/utils.py b/src/utils.py index 9be4171..ed8bb02 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,16 +1,17 @@ from constants import COLUMNS_LIST, GENERATION -import requests -import pandas as pd -import numpy as np from concurrent.futures import ThreadPoolExecutor +from sqlalchemy import create_engine +from pathlib import Path +import pandas as pd +import configparser +import requests + def fetch_data(index): res = requests.get(f'https://pokeapi.co/api/v2/pokemon/{index}').json() dic = {x: res[x] for x in COLUMNS_LIST} - values = np.array(list(dic.values())).reshape(1, 6) - df = pd.DataFrame((values), columns=list(dic.keys())) - return df - + return dic + def transform_data(): data_list = [] with ThreadPoolExecutor(max_workers=10) as executor: @@ -20,3 +21,10 @@ def transform_data(): df = pd.DataFrame(data_list) return df +def extract_to_csv(): + filepath = Path('../data/exctract.csv') + filepath.parent.mkdir(parents=True, exist_ok=True) + return transform_data().to_csv(filepath, index=False) + +def save_in_db(): + return transform_data().to_sql('pokemons', engine, if_exists='replace', index=False)