Merge pull request #8 from decodingml/module-1/fix-linkedin-crawler

fixed linkedin crawler in linkedin.py
decodingml · Apr 26, 2024 · cc4a955 · cc4a955
2 parents 4633cb3 + 09bb415
commit cc4a955
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/course/module-1/crawlers/linkedin.py b/course/module-1/crawlers/linkedin.py
@@ -35,6 +35,12 @@ def extract(self, link: str, **kwargs):
             "Education": self._scrape_education(link),
         }
 
+        self.driver.get(link)
+        time.sleep(5)
+        button = self.driver.find_element(By.CSS_SELECTOR,
+                                          ".app-aware-link.profile-creator-shared-content-view__footer-action")
+        button.click()
+
         # Scrolling and scraping posts
         self.scroll_page()
         soup = BeautifulSoup(self.driver.page_source, "html.parser")