scraping

Author: furas

__scraping__/blockchain.info/main.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.18
+# https://stackoverflow.com/questions/61858764/is-there-an-easy-way-to-access-all-transactions-recorded-in-a-bitcoin-block-with/
+# 
+# https://www.blockchain.com/api/blockchain_api
+
+import requests
+
+r = requests.get('https://blockchain.info/block-height/100?format=json')
+data = r.json()
+
+#print(r.text)
+#print(data)
+print(data['blocks'][0]['hash'])

__scraping__/epicgames.com - free games - requests/main.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.18
+# https://stackoverflow.com/questions/61876744/scraper-returns-null-result/
+
+import requests
+
+url = 'https://store-site-backend-static.ak.epicgames.com/freeGamesPromotions?locale=en-US&country=PL&allowCountries=PL'
+
+r = requests.get(url)
+
+data = r.json()
+
+#print(r.text)
+
+for item in data['data']['Catalog']['searchStore']['elements']:
+    print(item['title'])
+    offers = item['promotions']['promotionalOffers']
+    for offer in offers:
+        print(offer['promotionalOffers'][0]['startDate'])
+        print(offer['promotionalOffers'][0]['endDate'])
+

__scraping__/fcainfoweb.nic.in/main.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.28
+# 
+
+from selenium import webdriver 
+from selenium.webdriver.support.ui import Select
+import pandas as pd
+import time
+
+# --- functions ---
+
+def get_data(start_date, end_date, product):
+    
+    # select `Variation Report`
+    driver.find_element_by_id('ctl00_MainContent_Rbl_Rpt_type_1').click()
+    
+    # select `Daily Variant`
+    element_variation = driver.find_element_by_id ('ctl00_MainContent_Ddl_Rpt_Option1')
+    drop_variation = Select(element_variation)
+    drop_variation.select_by_visible_text('Daily Variation')
+
+    # select `product` before `date` because `end_date` opens calendar which blocks `product` list
+    element_commodity = driver.find_element_by_id ('ctl00_MainContent_Lst_Commodity')
+    drop_commodity = Select(element_commodity)
+    drop_commodity.select_by_visible_text(product)
+
+    # select `start_date` and `end_date`    
+    driver.find_element_by_id('ctl00_MainContent_Txt_FrmDate').send_keys(start_date)
+    driver.find_element_by_id('ctl00_MainContent_Txt_ToDate').send_keys(end_date)
+    
+    # click button `Get Data`
+    driver.find_element_by_id('ctl00_MainContent_btn_getdata1').click()
+
+    time.sleep(3)  # sometimes it need to wait for loading page
+    
+    #second table is the one that we want    
+    table = pd.read_html(driver.page_source)[2]
+
+    print(len(table))
+    print(table)
+    
+    # go back
+    driver.find_element_by_id('btn_back').click()
+
+    time.sleep(3)  # sometimes it need to wait for loading page
+
+    return table
+
+# --- main ---
+
+driver = webdriver.Firefox()
+
+driver.get('https://fcainfoweb.nic.in/Reports/Report_Menu_Web.aspx')
+
+start_date = '01/05/2020'
+end_date   = '27/05/2020'
+
+for number, product in enumerate( ('Rice', 'Wheat', 'Tomato', 'Sugar') ):
+    table = get_data(start_date, end_date, product)
+    # for first product create file, for other products append to existing file
+    if number == 0:
+        mode = 'w'
+    else:
+        mode = 'a'
+    # standard engine `xlsxwriter` can't append so I had to use `openpyxl`
+    with pd.ExcelWriter('output.xlsx', engine='openpyxl', mode=mode) as writer:
+        table.to_excel(writer, sheet_name=product, index=False)
+

__scraping__/fcainfoweb.nic.in/output.xlsx

@@ -0,0 +1,14 @@
+
+# date: 2020.05.26
+# https://stackoverflow.com/questions/61994836/bs4-web-scraping-searching-div-class/
+
+import requests
+from bs4 import BeautifulSoup
+
+headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0'}
+r = requests.get('https://www.google.com/search?q=titanic+movie', headers=headers)
+
+soup = BeautifulSoup(r.content, 'html.parser')
+item = soup.find('div', class_="srBp4 Vrkhme")
+print(item.get_text(strip=True, separator=' '))
+

__scraping__/google.com-movie/main-beautifulsoup.py

@@ -0,0 +1,14 @@
+
+# date: 2020.05.26
+# https://stackoverflow.com/questions/61994836/bs4-web-scraping-searching-div-class/
+
+import selenium.webdriver
+        
+url = 'https://www.google.com/search?q=titanic+movie'
+
+driver = selenium.webdriver.Firefox()
+driver.get(url)
+
+item = driver.find_element_by_class_name('srBp4.Vrkhme')
+print(item.text.strip())
+        

__scraping__/google.com-movie/main-selenium.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.25
+# https://stackoverflow.com/questions/62003463/web-scraping-hedge-fund-data-with-beautifulsoup
+
+import selenium.webdriver
+import time
+
+url = 'https://hedgefollow.com/funds/Duquesne+Family+Office'
+
+driver = selenium.webdriver.Firefox()
+driver.get(url)
+
+time.sleep(3)
+
+table = driver.find_element_by_id('dgtopHolders')
+
+print('--- headers ---')
+
+row = table.find_elements_by_tag_name('tr')[0]
+for cell in row.find_elements_by_tag_name('th'):
+    print(cell.text)
+    
+print('--- data ---')
+
+for row in table.find_elements_by_tag_name('tr')[1:]:
+    for cell in row.find_elements_by_tag_name('td'):
+        print(cell.text)
+    print('---')
+

__scraping__/hedgefollow.com - selenium/main.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.24
+# https://stackoverflow.com/questions/61981006/extracting-p-from-div-class-python-to-get-addresses/
+
+import requests
+import BeautifulSoup
+import csv
+import urllib2
+import time
+
+initial_url = "https://www.lifetime.life"
+
+response = requests.get("https://www.lifetime.life/view-all-locations.html")
+soup = BeautifulSoup.BeautifulSoup(response.text)
+
+with open('gyms2.csv', 'w') as gf:
+    gymwriter = csv.writer(gf)
+    for a in soup.findAll('a'):
+        if '/life-time-locations/' in a['href']:
+            gymurl = urllib2.urlparse.urljoin(initial_url, a.get('href'))
+            print(gymurl)
+            
+            response = requests.get(gymurl)
+            sub_soup = BeautifulSoup.BeautifulSoup(response.text)
+            
+            try:
+                address_line = sub_soup.find('p', {'class': 'small m-b-sm p-t-1'}).find('span', {'class': 'btn-icon-text'})
+
+                gymrow = [gymurl, address_line.text]
+                print(gymrow)
+                gymwriter.writerow(gymrow)
+                time.sleep(3)
+            except Exception as ex:
+                print(ex)
+                

__scraping__/lifetime.life/main-python2.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.24
+# https://stackoverflow.com/questions/61981006/extracting-p-from-div-class-python-to-get-addresses/
+
+import requests
+from bs4 import BeautifulSoup
+import urllib.parse
+import csv
+import time
+
+initial_url = "https://www.lifetime.life"
+
+response = requests.get("https://www.lifetime.life/view-all-locations.html")
+soup = BeautifulSoup(response.text)
+
+with open('gyms2.csv', 'w') as gf:
+    gymwriter = csv.writer(gf)
+    for a in soup.findAll('a'):
+        if '/life-time-locations/' in a['href']:
+            gymurl = urllib.parse.urljoin(initial_url, a.get('href'))
+            print(gymurl)
+            
+            response = requests.get(gymurl)
+            sub_soup = BeautifulSoup(response.text)
+
+            try:
+                address_line = sub_soup.select('p.small.m-b-sm.p-t-1 span.btn-icon-text')
+                gymrow = [gymurl, address_line[0].text.strip()]
+                print(gymrow)
+                gymwriter.writerow(gymrow)
+                time.sleep(3)
+            except Exception as ex:
+                print(ex)
+
+

__scraping__/lifetime.life/main-python3.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+# date: 2020.05.25
+# https://stackoverflow.com/questions/62000520/extracting-html-data-using-python/
+
+import requests
+from bs4 import BeautifulSoup
+import csv
+
+url = 'https://www.marketscreener.com/MICROSOFT-CORPORATION-4835/company/'
+
+r = requests.get(url) #, headers={'user-agent': 'Mozilla/5.0'})
+soup = BeautifulSoup(r.content, 'html.parser')
+
+all_tables = []
+
+for table in soup.select("table table.nfvtTab"):
+    table_rows = []
+    for tr in table.select('tr'):
+        row = []
+        for td in tr.select('td'):
+            #print(td)
+            item = td.get_text(strip=True, separator=' ')
+            #print(item)
+            row.append(item)
+        table_rows.append(row)
+    all_tables.append(table_rows)
+
+# add headers for nested columns
+
+#Sales per Business     
+all_tables[0][0].insert(2, '2018')
+all_tables[0][0].insert(4, '2019')
+all_tables[0][1].insert(0, '')
+all_tables[0][1].insert(5, '')
+
+# create one row with headers
+headers = [f'{a} {b}'.strip() for a,b in zip(all_tables[0][0], all_tables[0][1])]
+print('new:', headers)
+all_tables[0][0] = headers  # set new headers in first row
+del all_tables[0][1]        # remove second row
+
+#Sales  per region
+all_tables[1][0].insert(2, '2018')
+all_tables[1][0].insert(4, '2019')
+all_tables[1][1].insert(0, '')
+all_tables[1][1].insert(5, '')
+
+# create one row with headers
+headers = [f'{a} {b}'.strip() for a,b in zip(all_tables[1][0], all_tables[1][1])]
+print('new:', headers)
+all_tables[1][0] = headers  # set new headers in first row
+del all_tables[1][1]        # remove second row
+
+#Equities
+all_tables[3][0].insert(4, 'Free-Float %')
+all_tables[3][0].insert(6, 'Company-owned shares %')
+
+for number, table in enumerate(all_tables, 1):
+    print('---', number, '---')
+    for row in table:
+        print(row)
+
+for number, table in enumerate(all_tables, 1):
+    with open(f'table{number}.csv', 'w') as f:
+        csv_writer = csv.writer(f)
+        csv_writer.writerows(table)
+

__scraping__/marketscreener.com/main.py

@@ -0,0 +1,4 @@
+,2018 USD (in Million),2018 %,2019 USD (in Million),2019 %,Delta
+More Personal Computing,"42,276",38.4%,"45,698",36.4%,+8.09%
+Productivity and Business Processes,"35,865",32.6%,"41,160",32.8%,+14.76%
+Intelligent Cloud,"32,219",29.2%,"38,985",31.1%,+21%

__scraping__/marketscreener.com/table1.csv

@@ -0,0 +1,3 @@
+,2018 USD (in Million),2018 %,2019 USD (in Million),2019 %,Delta
+United States,"55,926",50.8%,"64,199",51.2%,+14.79%
+Other Countries,"54,434",49.4%,"61,644",49.1%,+13.25%

__scraping__/marketscreener.com/table2.csv

@@ -0,0 +1,11 @@
+Name,Age,Since,Title
+Satya Nadella,52,2014,Chief Executive Officer & Non-Independent Director
+Bradford Smith,60,2015,President & Chief Legal Officer
+John Thompson,69,2014,Independent Chairman
+Kirk Koenigsbauer,51,2020,COO & VP-Experiences & Devices Group
+Amy E. Hood,47,2013,Chief Financial Officer & Executive Vice President
+James Kevin Scott,54,-,Chief Technology Officer & Executive VP
+John W. Stanton,64,2014,Independent Director
+Teri L. List-Stoll,57,2014,Independent Director
+Charles Scharf,53,2014,Independent Director
+Sandra E. Peterson,60,2015,Independent Director

__scraping__/marketscreener.com/table3.csv

@@ -0,0 +1,2 @@
+,Vote,Quantity,Free-Float,Free-Float %,Company-owned shares,Company-owned shares %,Total Float
+Stock A,1,"7,583,440,247","7,475,252,172",98.6%,0,0.0%,98.6%

__scraping__/marketscreener.com/table4.csv

@@ -0,0 +1,11 @@
+Name,Equities,%
+"The Vanguard Group, Inc.","603,109,511",7.95%
+Capital Research & Management Co.,"556,573,400",7.34%
+"SSgA Funds Management, Inc.","314,771,248",4.15%
+Fidelity Management & Research Co.,"221,883,722",2.93%
+BlackRock Fund Advisors,"183,455,207",2.42%
+"T. Rowe Price Associates, Inc. (Investment Management)","172,056,401",2.27%
+Capital Research & Management Co. (World Investors),"139,116,236",1.83%
+Putnam LLC,"121,797,960",1.61%
+Geode Capital Management LLC,"115,684,966",1.53%
+Capital Research & Management Co. (International Investors),"103,523,946",1.37%

__scraping__/marketscreener.com/table5.csv

@@ -0,0 +1,17 @@
+import pandas as pd
+
+df = pd.read_csv(f'table1.csv', index_col=0) #, header=[0,1])
+print(df)
+    
+df = pd.read_csv(f'table2.csv', index_col=0) #, header=[0,1])
+print(df)
+    
+df = pd.read_csv(f'table3.csv') #, index_col=0)
+print(df)
+    
+df = pd.read_csv(f'table4.csv', index_col=0)
+print(df)
+    
+df = pd.read_csv(f'table5.csv') #, index_col=0)
+print(df)
+