diff --git a/.example.env b/.example.env index 6e9c4d3b1..26c3cb8ad 100644 --- a/.example.env +++ b/.example.env @@ -87,3 +87,6 @@ REPORT_EMAIL= # Optional - Support email to show on the app CONTACT_EMAIL= + +# Optional: do not insert analytics rows for obvious bots (default false) +SKIP_BOTS=false diff --git a/package-lock.json b/package-lock.json index 00c1f0bed..b9b202d8f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "cookie-parser": "1.4.7", "cors": "2.8.5", "date-fns": "2.30.0", - "dotenv": "^16.4.7", + "dotenv": "16.4.7", "envalid": "8.0.0", "express": "4.21.2", "express-rate-limit": "7.5.0", @@ -24,7 +24,7 @@ "hbs": "4.2.0", "helmet": "7.1.0", "ioredis": "5.4.2", - "isbot": "5.1.19", + "isbot": "5.1.31", "jsonwebtoken": "9.0.2", "knex": "3.1.0", "ms": "2.1.3", @@ -38,7 +38,7 @@ "pg": "8.13.1", "pg-query-stream": "4.7.1", "rate-limit-redis": "4.2.0", - "useragent": "2.3.0" + "ua-parser-js": "2.0.5" }, "devDependencies": { "@types/bcryptjs": "2.4.2", @@ -1936,6 +1936,26 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-europe-js": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/detect-europe-js/-/detect-europe-js-0.1.2.tgz", + "integrity": "sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, "node_modules/detect-libc": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", @@ -3561,6 +3581,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-standalone-pwa": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-standalone-pwa/-/is-standalone-pwa-0.1.1.tgz", + "integrity": "sha512-9Cbovsa52vNQCjdXOzeQq5CnCbAcRk05aU62K20WO372NrTv0NxibLFCK6lQ4/iZEFdEA3p3t2VNOn8AJ53F5g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, "node_modules/is-string": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.1.1.tgz", @@ -3681,9 +3721,9 @@ "peer": true }, "node_modules/isbot": { - "version": "5.1.19", - "resolved": "https://registry.npmjs.org/isbot/-/isbot-5.1.19.tgz", - "integrity": "sha512-8krWJBGKC3lVymkncvmBTpIEWMD5kKmjAvkM3/Xh6veE0bAydwgSNrI5h493DGrG2UNJCy0HuHpNPSKRy0dBJA==", + "version": "5.1.31", + "resolved": "https://registry.npmjs.org/isbot/-/isbot-5.1.31.tgz", + "integrity": "sha512-DPgQshehErHAqSCKDb3rNW03pa2wS/v5evvUqtxt6TTnHRqAG8FdzcSSJs9656pK6Y+NT7K9R4acEYXLHYfpUQ==", "license": "Unlicense", "engines": { "node": ">=18" @@ -4724,15 +4764,6 @@ "json-pointer": "0.6.2" } }, - "node_modules/os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/own-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz", @@ -5229,12 +5260,6 @@ "node": ">= 0.10" } }, - "node_modules/pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", - "license": "ISC" - }, "node_modules/pump": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", @@ -6359,18 +6384,6 @@ "node": ">=8" } }, - "node_modules/tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "license": "MIT", - "dependencies": { - "os-tmpdir": "~1.0.2" - }, - "engines": { - "node": ">=0.6.0" - } - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -6500,6 +6513,67 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ua-is-frozen": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ua-is-frozen/-/ua-is-frozen-0.1.2.tgz", + "integrity": "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, + "node_modules/ua-parser-js": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.5.tgz", + "integrity": "sha512-sZErtx3rhpvZQanWW5umau4o/snfoLqRcQwQIZ54377WtRzIecnIKvjpkd5JwPcSUMglGnbIgcsQBGAbdi3S9Q==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "license": "AGPL-3.0-or-later", + "dependencies": { + "detect-europe-js": "^0.1.2", + "is-standalone-pwa": "^0.1.1", + "ua-is-frozen": "^0.1.2", + "undici": "^7.12.0" + }, + "bin": { + "ua-parser-js": "script/cli.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ua-parser-js/node_modules/undici": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/uglify-js": { "version": "3.19.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", @@ -6609,32 +6683,6 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, - "node_modules/useragent": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/useragent/-/useragent-2.3.0.tgz", - "integrity": "sha512-4AoH4pxuSvHCjqLO04sU6U/uE65BYza8l/KKBS0b0hnUPWi+cQ2BpeTEwejCSx9SPV5/U03nniDTrWx5NrmKdw==", - "license": "MIT", - "dependencies": { - "lru-cache": "4.1.x", - "tmp": "0.0.x" - } - }, - "node_modules/useragent/node_modules/lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "license": "ISC", - "dependencies": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "node_modules/useragent/node_modules/yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", - "license": "ISC" - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 84ffaa872..3f192a239 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "hbs": "4.2.0", "helmet": "7.1.0", "ioredis": "5.4.2", - "isbot": "5.1.19", + "isbot": "5.1.31", "jsonwebtoken": "9.0.2", "knex": "3.1.0", "ms": "2.1.3", @@ -53,7 +53,7 @@ "pg": "8.13.1", "pg-query-stream": "4.7.1", "rate-limit-redis": "4.2.0", - "useragent": "2.3.0" + "ua-parser-js": "2.0.5" }, "devDependencies": { "@types/bcryptjs": "2.4.2", diff --git a/server/env.js b/server/env.js index 87f7bc373..93d67db16 100644 --- a/server/env.js +++ b/server/env.js @@ -65,6 +65,7 @@ const spec = { REPORT_EMAIL: str({ default: "" }), CONTACT_EMAIL: str({ default: "" }), NODE_APP_INSTANCE: num({ default: 0 }), + SKIP_BOTS: bool({ default: false }), }; for (const key in spec) { diff --git a/server/queues/visit.js b/server/queues/visit.js index a05acd19b..b30803453 100644 --- a/server/queues/visit.js +++ b/server/queues/visit.js @@ -1,36 +1,28 @@ -const useragent = require("useragent"); +const env = require("../env"); +const { normaliseUA } = require("../utils/ua"); +const isbot = require("isbot"); const geoip = require("geoip-lite"); const URL = require("node:url"); - const { removeWww } = require("../utils"); const query = require("../queries"); -const browsersList = ["IE", "Firefox", "Chrome", "Opera", "Safari", "Edge"]; -const osList = ["Windows", "Mac OS", "Linux", "Android", "iOS"]; - -function filterInBrowser(agent) { - return function(item) { - return agent.family.toLowerCase().includes(item.toLocaleLowerCase()); - } -} - -function filterInOs(agent) { - return function(item) { - return agent.os.family.toLowerCase().includes(item.toLocaleLowerCase()); - } -} module.exports = function({ data }) { + // Opt-in: skip analytics rows for obvious bots (does not affect redirects or the link counter) + if (env.SKIP_BOTS && isbot(userAgent)) { + return Promise.all([query.link.incrementVisit({ id: data.link.id })]); + } const tasks = []; tasks.push(query.link.incrementVisit({ id: data.link.id })); + const userAgent = (data.userAgent || data.headers?.["user-agent"] || ""); + + + const { browser, os } = normaliseUA(userAgent); // the following line is for backward compatibility // used to send the whole header to get the user agent - const userAgent = data.userAgent || data.headers?.["user-agent"]; - const agent = useragent.parse(userAgent); - const [browser = "Other"] = browsersList.filter(filterInBrowser(agent)); - const [os = "Other"] = osList.filter(filterInOs(agent)); + const referrer = data.referrer && removeWww(URL.parse(data.referrer).hostname); @@ -38,11 +30,11 @@ module.exports = function({ data }) { tasks.push( query.visit.add({ - browser: browser.toLowerCase(), + browser: browser, // already lowercased in helper country: country || "Unknown", link_id: data.link.id, user_id: data.link.user_id, - os: os.toLowerCase().replace(/\s/gi, ""), + os: os, // already lowercased and space-stripped in helper referrer: (referrer && referrer.replace(/\./gi, "[dot]")) || "Direct" }) ); diff --git a/server/utils/ua.js b/server/utils/ua.js new file mode 100644 index 000000000..9c424c1a3 --- /dev/null +++ b/server/utils/ua.js @@ -0,0 +1,37 @@ +const UAParser = require("ua-parser-js"); + +function normaliseOSName(name = "") { + const s = String(name).toLowerCase(); + if (s.includes("windows")) return "windows"; + if (s.includes("mac os") || s.includes("macos") || s.startsWith("mac")) return "macos"; + if (s.includes("android")) return "android"; + if (s.includes("ios")) return "ios"; + if (s.includes("linux")) return "linux"; + return "other"; +} + +function normaliseUA(ua = "") { + const parsed = new UAParser(ua).getResult(); + console.log('Parsed OS:', parsed.os); + console.log('Parsed Browser:', parsed.browser); + const browserName = parsed.browser?.name || ""; + const osName = parsed.os?.name || ""; + + // keep browser behaviour identical to before + const browser = (() => { + const b = browserName.toLowerCase(); + if (b.includes("edge")) return "edge"; + if (b.includes("chrome")) return "chrome"; + if (b.includes("firefox")) return "firefox"; + if (b.includes("safari")) return "safari"; + if (b.includes("opera")) return "opera"; + if (b.includes("ie") || b.includes("internet explorer")) return "ie"; + return "other"; + })(); + + const os = normaliseOSName(osName); + + return { browser, os }; +} + +module.exports = { normaliseUA };