Updated final version of crawler (Issue-#10)

privacy-tech-lab · Aug 19, 2022 · 7badcbe · 7badcbe
1 parent 7cf4db0
commit 7badcbe
Showing 1 changed file with 24 additions and 23 deletions.
diff --git a/Firefox-analysis-mode-crawler/crawler.js b/Firefox-analysis-mode-crawler/crawler.js
@@ -7,6 +7,7 @@ const firefoxOptions = {
     product: 'firefox',
 };
 
+// Read csv file
 const sites = [];
 fs.createReadStream("sites.csv")
   .pipe(parse({ delimiter: ",", from_line: 2 }))
@@ -17,12 +18,14 @@ fs.createReadStream("sites.csv")
     console.log(error.message);
   });
 
-
+// Crawling
 (async () => {
     const browser = await puppeteer.launch(firefoxOptions);
 
     // Allow time to load the extension
     await new Promise(resolve => setTimeout(resolve, 60000));
+
+    // Timestamp
     const a = new Date();
     console.log("Time: ", a.getTime());
 
@@ -38,53 +41,51 @@ fs.createReadStream("sites.csv")
         } 
 
         await new Promise(resolve => setTimeout(resolve, 15000));
+
+        /* Promise.race sets time limit for page.keyboard.down to resolve the issue that 
+           page.keyboard.down is never rejected or resolved on some sites */
 
-        // To solve text box issue
-        try {
-          await Promise.race([page.evaluate(async () => {
-            await document.activeElement.blur()  
-         }), new Promise(resolve => setTimeout(resolve, 5000))]);
-        } catch {}
-
-       await new Promise(resolve => setTimeout(resolve, 5000));
-
+        // Press shift+tab to solve text box issue
+        await Promise.race([page.keyboard.down('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await Promise.race([page.keyboard.down('Tab'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        await Promise.race([page.keyboard.up('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await Promise.race([page.keyboard.up('Tab'), new Promise(resolve => setTimeout(resolve, 1000))]);
 
-        // Promise.race sets time limit for page.keyboard.down to resolve the issue that page.keyboard.down is never rejected or resolved on some sites
+        // Start analysis using keyboard shortcuts
         await Promise.race([page.keyboard.down('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.down('Alt'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.down('KeyA'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await new Promise(resolve => setTimeout(resolve, 1000));
-
         await Promise.race([page.keyboard.up('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.up('Alt'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.up('KeyA'), new Promise(resolve => setTimeout(resolve, 1000))]);
 
         // Allow the site to load after analysis is triggered
-        await new Promise(resolve => setTimeout(resolve, 15000));
-
-        try {
-        await Promise.race([page.evaluate(async () => {
-          await document.activeElement.blur()  
-       }), new Promise(resolve => setTimeout(resolve, 5000))]);
-      } catch {}
-
-        await new Promise(resolve => setTimeout(resolve, 5000));
+        await new Promise(resolve => setTimeout(resolve, 10000));
+
+        await Promise.race([page.keyboard.down('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await Promise.race([page.keyboard.down('Tab'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        await Promise.race([page.keyboard.up('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
+        await Promise.race([page.keyboard.up('Tab'), new Promise(resolve => setTimeout(resolve, 1000))]);
 
+        // Stop analysis using keyboard shortcuts
         await Promise.race([page.keyboard.down('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.down('Alt'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.down('KeyS'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await new Promise(resolve => setTimeout(resolve, 1000));
-
         await Promise.race([page.keyboard.up('Shift'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.up('Alt'), new Promise(resolve => setTimeout(resolve, 1000))]);
         await Promise.race([page.keyboard.up('KeyS'), new Promise(resolve => setTimeout(resolve, 1000))]);
 
         await new Promise(resolve => setTimeout(resolve, 5000));
 
+        // Close the page to resolve the issue that some sites do not allow redirecting
         await page.close();
         console.log("testing done");
     }
     const d = new Date();
     console.log("Time: ", d.getTime());
     console.log("---------------- ALL TESTING DONE ----------------");
-})();
+})();