TPED-Bot/scraper.py at main · nickt573/TPED-Bot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import requests
from bs4 import BeautifulSoup

def parse_park(park):
    data = []
    soup = None
    try:
        # Gathering main park page
        url = f'https://rcdb.com/qs.htm?qs={park}'
        while True:  # Ensures search leads to park page
            response = requests.get(url, verify=True)
            response.raise_for_status()
            html = response.text
            soup = BeautifulSoup(html, 'html.parser')

            h4 = soup.find_all('h4')
            if len(h4) > 0: # Checks if link redirects to a list of parks
                url = response.url
                break
            else:
                park = park.lower()
                found = False
                try:
                    sections = soup.find_all('section')
                    for section in sections:
                        h3 = section.find('h3')
                        if h3 and "Amusement Park" in h3.text:
                            ps = section.find_all('p')
                            for p in ps:
                                if "Too many" in p.text:
                                    return None
                                title = p.find('a')
                                option_words = title.text.lower().split()
                                if all(word in option_words for word in park.split()) and not found:
                                    url = "https://rcdb.com" + title.get('href')
                                    found = True
                                    break;
                        if found:
                            break
                    else:
                        return None
                except:
                    return None

        park_official = soup.find('div', id='feature').find('h1').text
        data.append((park_official, "park", url))

        # Only looking at the first h4 (operating coasters) and following to page
        for i in range(len(h4)):
            if "Operating Roller Coasters: " in h4[i].text or "SBNO Roller Coasters: " in h4[i].text:
                url = "https://rcdb.com" + h4[i].find('a').get('href')
                response = requests.get(url, verify=True)
                response.raise_for_status()
                html = response.text
                soup = BeautifulSoup(html, 'html.parser')

                ride_table = soup.find('div', class_='stdtbl rer').find('tbody').find_all('tr')
                for tr in ride_table:
                    ride = tr.find_all('td')[1].find('a')
                    if ride.text in "unknown":
                        continue
                    data.append((ride.text, "ride", "https://rcdb.com" + ride.get('href')))
        return data if data else None
    except Exception:
        # Purposely may reach here if h4[i].text causes exception, meaning there are not open coasters
        # May be interrupted from above, so return partial reading at best
        return data if data else None

# for debugging purposes
def main():
    park = input("Park: ")
    results = parse_park(park)
    if results:
        for item in results:
            print(item)
    else:
        print("No park found")

if __name__ == '__main__':
    main()