new examples

Author: furas

__scraping__/corporate.dow.com/main.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.24
+# https://stackoverflow.com/questions/59019810/python-web-scraping-ahref-link-and-articles-not-showing-up-in-source-code
+
+import selenium.webdriver
+        
+url = 'https://corporate.dow.com/en-us/news.html'
+driver = selenium.webdriver.Firefox()
+driver.get(url)
+
+all_items = driver.find_elements_by_xpath('//ul[@class="results__list"]/li')
+for item in all_items:
+    print(item.find_element_by_xpath('.//h3').text)
+    print(item.find_element_by_xpath('.//a').get_attribute('href'))
+    print('---')
+

__scraping__/dps.psx.com.pk - selenium/main.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.23
+# https://stackoverflow.com/questions/59008770/want-to-read-a-tag-data-using-selenium
+
+from selenium import webdriver
+
+driver = webdriver.Firefox()
+driver.get('https://dps.psx.com.pk/')
+
+last_table = driver.find_elements_by_xpath("//table")[-1]
+
+for row in last_table.find_elements_by_xpath(".//tr")[1:]:
+    print(row.find_element_by_xpath(".//td/a[@class='tbl__symbol']").text)
+    print([td.text for td in row.find_elements_by_xpath(".//td[@class='right']")])

__scraping__/medindia.net - requests - BS/main.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.23
+# https://stackoverflow.com/questions/59008426/python-web-scrapping-if-using-all-scalar-values-you-must-pass-an-index
+
+import pandas as pd
+import requests
+import urllib.request
+import time
+from bs4 import BeautifulSoup
+
+url = 'https://www.medindia.net/doctors/drug_information/abacavir.htm'
+response = requests.get(url)
+soup = BeautifulSoup(response.text, "html.parser")
+drug = soup.find(class_='mi-container__fluid')
+#print(drug)
+
+# whole page contain drug content
+items = drug.find_all(class_='report-content drug-widget')
+#print(items)
+
+# extract drug information from drug content into individual variable
+trade_name = items[0].find(class_='drug-content').get_text()
+function = items[1].find(class_='drug-content').get_text()
+contraindications = items[2].find(class_='drug-content').get_text()
+dosage = items[3].find(class_='drug-content').get_text()
+how_to_use = items[4].find(class_='drug-content').get_text()
+warnings = items[5].find(class_='drug-content').get_text()
+storage = items[7].find(class_='drug-content').get_text()
+
+
+drug_stuff = pd.DataFrame({
+    'trade_name': [trade_name],
+    'function': [function],
+    'contraindications': [contraindications],
+    'dosage': [dosage],
+    'how_to_use': [how_to_use],
+    'warnings': [warnings],
+    'storage': [storage],
+})
+
+print(drug_stuff)

__scraping__/money.cnn.com/main.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.23
+# https://stackoverflow.com/questions/59004270/i-want-to-display-first-word-from-1st-list-and-display-10-words-from-2nd-list-an
+
+from bs4 import BeautifulSoup
+import urllib.request
+
+url = 'https://money.cnn.com/data/hotstocks/'
+html = urllib.request.urlopen(url).read()
+soup = BeautifulSoup(html,'lxml')
+
+allbody = soup.find('div', class_='cnnBody_Left wsodContent') 
+names = allbody.find_all('h3')   #I am finding the header tags text
+names = [x.text for x in names]
+#print(names)
+
+contents = allbody.find_all('table', class_='wsod_dataTable wsod_dataTableBigAlt')
+
+tables = [] # keep three tables
+
+for item in contents:
+    data = [] # list for single table
+
+    for tr in item.find_all('tr')[1:]: # find rows in table - skip row with headers `[1:]`
+        a = tr.find('a')  # get only from first column
+        a = a.text.strip()
+
+        span = tr.find('span')  # get only from first column
+        span = span.text.strip()
+
+        data.append( (a, span) )
+
+    tables.append(data)
+
+
+for name, table in zip(names, tables):
+    print('-', name)
+    for a, span in table:
+        print(a, span)

__scraping__/rtrs.tv/main.py

@@ -0,0 +1,21 @@
+from requests import session
+
+from bs4 import BeautifulSoup
+
+url = r'https://www.rtrs.tv/vijesti/index.php'
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
+}
+
+with session() as c:
+
+    r = c.get(url, headers=headers)
+
+    print(r)
+
+    soup = BeautifulSoup(r.text, 'html.parser')
+
+    all_h2 = soup.find_all('h2')
+    for item in all_h2:
+        print(item.text)

__scraping__/std.stheadline.com/main.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.23
+# https://stackoverflow.com/questions/59003700/pythonon-ajax-php-prase-result-is-different-from-on-screen-result
+
+import requests
+
+url = 'http://std.stheadline.com/daily/ajax/ajaxFormerly.php'
+
+params = {
+    'startDate': '2019-11-20',
+    'endDate': '2019-11-22',
+    'type[]': '15',
+    'keyword': '',
+}
+
+r = requests.post(url, data=params)
+
+data = r.json()
+
+print(data['totalCount']) # 47
+
+import pandas as pd
+import io
+
+f = io.StringIO(r.text)
+df = pd.read_json(f)
+
+print(df)

__scraping__/wikipedia-SP500/example-2/main.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3 
+
+# date: 2019.11.23
+# https://stackoverflow.com/questions/59003872/running-for-loop-and-skipping-stocks-with-keyerror-date
+
+from datetime import datetime, timedelta
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+import pandas as pd
+from pandas_datareader import data as web
+
+html = urlopen('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
+soup = BeautifulSoup(html,'lxml')
+sp500_raw = soup.find('table', {'class': 'wikitable sortable'})
+
+spsymbol = []
+
+for row in sp500_raw.findAll('tr')[1:]:
+    spsymbols = row.findAll('td')[0].text.strip()
+    spsymbol.append(spsymbols)
+
+start = datetime(2008, 1, 1).date()
+end = datetime.today().date()
+
+for ticker in spsymbol:
+    print(ticker)
+    try:
+        df = web.get_data_yahoo(ticker, start, end)
+        df = df.reset_index()
+        #print(df.head())
+        df.to_csv(ticker + '.csv', header=True, index=True, columns=['Date', 'High', 'Low', 'Open', 'Close', 'Volume', 'Adj Close'], sep=' ')
+    except Exception as ex:
+        print('Ex:', ex)

beautifulsoup/attribute-with-html/main.py

@@ -0,0 +1,23 @@
+from bs4 import BeautifulSoup as BS
+
+text = '''
+<tr data-title='<img src="url1.jpg" alt="1">' >
+<tr data-title='<img src="url2.jpg" alt="2">' >
+'''
+
+soup = BS(text, 'html.parser')
+
+all_items = soup.find_all('tr', {"data-title": True})
+
+for item in all_items:
+    print('item:', item['data-title'])
+    #print('item:', item.attrs.get('data-title'))
+    #print('item:', item.attrs['data-title'])
+    #print('item:', item.get('data-title'))
+    
+    link = item.get('data-title')
+    s = BS(link, 'html.parser')
+    print('src:', s.find('img')['src'])
+    
+ 
+