Added the parsers

Author: fabinhojorge

README.md

@@ -96,7 +96,9 @@ A formula usada para o cálculo da Temperatura média compensada (TC) é:
 
 ## TO DO
 * [ ] Lista das estações: http://www.inmet.gov.br/projetos/rede/pesquisa/lista_estacao.php -> Pegar o id delas para usar como parametro de consulta -> Tem Estações que não estão nessa lista. Existem estações de aeroportos como a ID 82022. Checar depois em um range de 82000 até 84000. Precisa modificar um pouco para pegar os dados de aeroporto porque tem mais 1 hifen.
-
+* [ ] Tratamento para login com usuario errado.
+* [ ] Exportar para arquivo
+* [ ] Criar classe observation 
 
 
 ## Saiba mais

config.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Author: Fabio Rodrigues Jorge
+Email: [email protected]
+Description: Config file.
+"""
+
+LOGIN = {'url': 'http://www.inmet.gov.br/projetos/rede/pesquisa/inicio.php',
+         'username': '<USER>',
+         'password': '<PASS>',
+         }

extract_data.py

@@ -10,7 +10,9 @@
 from selenium import webdriver
 from bs4 import BeautifulSoup
 import time
-# import datetime
+from config import LOGIN
+from datetime import datetime
+from station import Station
 # import csv
 
 
@@ -32,59 +34,82 @@ def get_page(_driver, _url):
     return BeautifulSoup(_driver.page_source, 'html.parser')
 
 
+def inmet_login(_driver):
+    """This function navigates to the login page and do the login."""
+
+    get_page(_driver, LOGIN['url'])
+
+    _driver.find_element_by_name("mCod").send_keys(LOGIN['username'])
+    _driver.find_element_by_name("mSenha").send_keys(LOGIN['password'])
+    _driver.find_element_by_name("btnProcesso").click()
+
+
 def load_station_numbers(_path):
+    """It returns the list of stations."""
     _f = open(_path)
     _station_numbers = _f.readlines()
     _station_numbers = list(map(lambda x: x.replace("\n", ""), _station_numbers))
     _f.close()
     return _station_numbers
 
 
+def weather_station_parser(_data):
+
+    ds = _data.split('\n')
+    s = Station(ds[1][20:], ds[2][20:], ds[3][20:], ds[4][20:], ds[5], ds[6][20:])
+    print(s)
+    return s
+
+
+def weather_observation_parser(_data):
+    ds = _data.split('\n')
+    ds = list(filter(None, ds))
+    # header is ds[0]
+    print(ds[0])
+    print(*ds[1:], sep='\n')
+
+
 # if __name__ == '__main__':
 
 print(">> WebCrawler Started <<")
 
 driver = init_webdriver_config(30)
 
-
 # -- Login --
-login_url = "http://www.inmet.gov.br/projetos/rede/pesquisa/inicio.php"
-
-soup = get_page(driver, login_url)
-
-username = "<USER>"
-password = "<PASS>"
+inmet_login(driver)
 
-driver.find_element_by_name("mCod").send_keys(username)
-driver.find_element_by_name("mSenha").send_keys(password)
-driver.find_element_by_name("btnProcesso").click()
 
 # -- Extract --
 url_pattern = "http://www.inmet.gov.br/projetos/rede/pesquisa/gera_serie_txt_mensal.php?&mRelEstacao={omm_code}" \
       "&btnProcesso=serie&mRelDtInicio={start_date}&mRelDtFim={end_date}&mAtributos={attributes}"
 
-
 station_numbers = load_station_numbers("./data/station_numbers.txt")
 
-start_date = "14/07/2014"
-end_date = "14/08/2019"
-attributes = "1,,,,,,,,,,,,1,1,1,1,"
+start_date = "01/07/2000"
+end_date = datetime.now().strftime("%d/%m/%Y")
+# attributes = "1,,,,,,,,,,,,1,1,1,1,"
+attributes = "1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1"
 
 count_success = 0
 count_error = 0
+
 for omm_code in station_numbers[:4]:
     url = url_pattern.format(omm_code=omm_code, start_date=start_date, end_date=end_date, attributes=attributes)
     soup = get_page(driver, url)
 
-    data = soup.select('pre')
+    soup_pre = soup.select('pre')
 
-    if 'Não existem dados disponiveis' in data[0].text:
+    if 'Não existem dados disponiveis' in soup_pre[0].text:
         count_error += 1
     else:
         count_success += 1
-        print(data[0].text)
+        content = soup_pre[0].text.split('--------------------')
+
+        station = weather_station_parser(content[2])
+        weather_observation_parser(content[4])
+
 
-driver.quit()
+# driver.quit()
 
 print(">> WebCrawler Finished <<")
 print("SUCCESS: {0}\nERROR: {1}\nTOTAL: {2}".format(count_success, count_error,

station.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from .state import State
+from state import State
 
 
 class Station:
@@ -20,5 +20,5 @@ def __init__(self, name_complete, lat, lng, alt, situation, op_date):
 		self.op_start_date = op_date
 
 	def __str__(self):
-		return "Estação: {0}\nLatitude: {1}\nLongitude: {2}\nAltitude: {3}\nSituação: {4}\nEm operação desde {0}"\
-			.format(self.name, self.omm_code, self.lat, self.lng, self.alt, self.situation, self.op_start_date)
+		return "Estação: {0}\nLatitude: {1}\nLongitude: {2}\nAltitude: {3}\nSituação: {4}\nEm operação desde {5}"\
+			.format(self.name, self.lat, self.lng, self.alt, self.situation, self.op_start_date)