feat(analytics): optional bot filtering via isbot (SKIP_BOTS=false by default)

.example.env

@@ -87,3 +87,6 @@ REPORT_EMAIL=
 
 # Optional - Support email to show on the app
 CONTACT_EMAIL=
+
+# Optional: do not insert analytics rows for obvious bots (default false)
+SKIP_BOTS=false

package.json

@@ -39,7 +39,7 @@
     "hbs": "4.2.0",
     "helmet": "7.1.0",
     "ioredis": "5.4.2",
-    "isbot": "5.1.19",
+    "isbot": "5.1.31",
     "jsonwebtoken": "9.0.2",
     "knex": "3.1.0",
     "ms": "2.1.3",
@@ -53,7 +53,7 @@
     "pg": "8.13.1",
     "pg-query-stream": "4.7.1",
     "rate-limit-redis": "4.2.0",
-    "useragent": "2.3.0"
+    "ua-parser-js": "2.0.5"
   },
   "devDependencies": {
     "@types/bcryptjs": "2.4.2",

server/env.js

@@ -65,6 +65,7 @@ const spec = {
   REPORT_EMAIL: str({ default: "" }),
   CONTACT_EMAIL: str({ default: "" }),
   NODE_APP_INSTANCE: num({ default: 0 }),
+  SKIP_BOTS: bool({ default: false }), 
 };
 
 for (const key in spec) {

server/queues/visit.js

@@ -1,48 +1,40 @@
-const useragent = require("useragent");
+const env = require("../env");
+const { normaliseUA } = require("../utils/ua");
+const isbot = require("isbot");
 const geoip = require("geoip-lite");
 const URL = require("node:url");
-
 const { removeWww } = require("../utils");
 const query = require("../queries");
 
-const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"];
-const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"];
-
-function filterInBrowser(agent) {
-  return function(item) {
-    return agent.family.toLowerCase().includes(item.toLocaleLowerCase());
-  }
-}
-
-function filterInOs(agent) {
-  return function(item) {
-    return agent.os.family.toLowerCase().includes(item.toLocaleLowerCase());
-  }
-}
 
 module.exports = function({ data }) {
+  // Opt-in: skip analytics rows for obvious bots (does not affect redirects or the link counter)
+  if (env.SKIP_BOTS && isbot(userAgent)) {
+     return Promise.all([query.link.incrementVisit({ id: data.link.id })]);
+  }
   const tasks = [];
 
   tasks.push(query.link.incrementVisit({ id:  data.link.id }));
+  const userAgent = (data.userAgent || data.headers?.["user-agent"] || "");
+
+
+  const { browser, os } = normaliseUA(userAgent);
 
   // the following line is for backward compatibility
   // used to send the whole header to get the user agent
-  const userAgent = data.userAgent || data.headers?.["user-agent"];
-  const agent = useragent.parse(userAgent);
-  const [browser = "Other"] = browsersList.filter(filterInBrowser(agent));
-  const [os = "Other"] = osList.filter(filterInOs(agent));
+
   const referrer =
   data.referrer && removeWww(URL.parse(data.referrer).hostname);
 
   const country = data.country || geoip.lookup(data.ip)?.country;
 
   tasks.push(
     query.visit.add({
-      browser: browser.toLowerCase(),
+      browser: browser, // already lowercased in helper
       country: country || "Unknown",
       link_id: data.link.id,
       user_id: data.link.user_id,
-      os: os.toLowerCase().replace(/\s/gi, ""),
+      os: os, // already lowercased and space-stripped in helper
       referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "Direct"
     })
   );

server/utils/ua.js

@@ -0,0 +1,37 @@
+const UAParser = require("ua-parser-js");
+
+function normaliseOSName(name = "") {
+  const s = String(name).toLowerCase();
+  if (s.includes("windows")) return "windows";
+  if (s.includes("mac os") || s.includes("macos") || s.startsWith("mac")) return "macos";
+  if (s.includes("android")) return "android";
+  if (s.includes("ios")) return "ios";
+  if (s.includes("linux")) return "linux";
+  return "other";
+}
+
+function normaliseUA(ua = "") {
+  const parsed = new UAParser(ua).getResult();
+  console.log('Parsed OS:', parsed.os); 
+  console.log('Parsed Browser:', parsed.browser);
+  const browserName = parsed.browser?.name || "";
+  const osName = parsed.os?.name || "";
+
+  // keep browser behaviour identical to before
+  const browser = (() => {
+    const b = browserName.toLowerCase();
+    if (b.includes("edge"))    return "edge";
+    if (b.includes("chrome"))  return "chrome";
+    if (b.includes("firefox")) return "firefox";
+    if (b.includes("safari"))  return "safari";
+    if (b.includes("opera"))   return "opera";
+    if (b.includes("ie") || b.includes("internet explorer")) return "ie";
+    return "other";
+  })();
+
+  const os = normaliseOSName(osName);
+
+  return { browser, os };
+}
+
+module.exports = { normaliseUA };