feat(mv3): Tracking URL resolved/observed count. (#1245)

* fix(mv3): Reverting Telemetry Changes To Use PatchedCountlySDK Signed-off-by: Nishant Arora <[email protected]> * fix(mv3): metrics build Signed-off-by: Nishant Arora <[email protected]> * fix: more reverts + fixing patch Signed-off-by: Nishant Arora <[email protected]> * feat: adding request view. Signed-off-by: Nishant Arora <[email protected]> * Reverting to mainline rc patch * Reverting to mainline rc add-on/src/lib/ipfs-companion.js * feat(telemetry): Signed-off-by: Nishant Arora <[email protected]> * feat(telemetry): Implementing RequestTracker Event Handler Signed-off-by: Nishant Arora <[email protected]> * feat(telemetry): hooking up events. Signed-off-by: Nishant Arora <[email protected]> * fix(types): annotations * fix(telemetry): 🗑️ returning to previous state Signed-off-by: Nishant Arora <[email protected]> * fix(telemetry): ♻️ Refactor Request Tracker Signed-off-by: Nishant Arora <[email protected]> * fix(telemetry): hooking up requests Signed-off-by: Nishant Arora <[email protected]> * fix(telemetry): better types Signed-off-by: Nishant Arora <[email protected]> * fix(countly): 🗑️ more stuff goes, because test need to pass. Signed-off-by: Nishant Arora <[email protected]> * fix(lint): fixed * feat(test): test tracker. * fix: remove only Signed-off-by: Nishant Arora <[email protected]> * fix: 💄 line break Signed-off-by: Nishant Arora <[email protected]> --------- Signed-off-by: Nishant Arora <[email protected]>
ipfs · Jul 28, 2023 · ac5cae6 · ac5cae6
1 parent fe0e159
commit ac5cae6
Show file tree

Hide file tree

Showing 8 changed files with 274 additions and 56 deletions.
diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js
@@ -8,6 +8,7 @@ import LRU from 'lru-cache'
 import pMemoize from 'p-memoize'
 import toMultiaddr from 'uri-to-multiaddr'
 import browser from 'webextension-polyfill'
+import { handleConsentFromState, trackView } from '../lib/telemetry.js'
 import { contextMenuCopyAddressAtPublicGw, contextMenuCopyCanonicalAddress, contextMenuCopyCidAddress, contextMenuCopyPermalink, contextMenuCopyRawCid, contextMenuViewOnGateway, createContextMenus, findValueForContext } from './context-menus.js'
 import createCopier from './copier.js'
 import createDnslinkResolver from './dnslink.js'
@@ -24,7 +25,6 @@ import { guiURLString, migrateOptions, optionDefaults, safeURL, storeMissingOpti
 import { getExtraInfoSpec } from './redirect-handler/blockOrObserve.js'
 import createRuntimeChecks from './runtime-checks.js'
 import { initState, offlinePeerCount } from './state.js'
-import { handleConsentFromState, trackView } from '../lib/telemetry.js'
 
 // this won't work in webworker context. Needs to be enabled manually
 // https://github.com/debug-js/debug/issues/916

diff --git a/add-on/src/lib/ipfs-request.js b/add-on/src/lib/ipfs-request.js
@@ -3,14 +3,15 @@
 
 import debug from 'debug'
 
-import LRU from 'lru-cache'
-import isIPFS from 'is-ipfs'
 import isFQDN from 'is-fqdn'
+import isIPFS from 'is-ipfs'
+import LRU from 'lru-cache'
+import { recoveryPagePath } from './constants.js'
+import { braveNodeType } from './ipfs-client/brave.js'
 import { dropSlash, ipfsUri, pathAtHttpGateway, sameGateway } from './ipfs-path.js'
 import { safeURL } from './options.js'
-import { braveNodeType } from './ipfs-client/brave.js'
-import { recoveryPagePath } from './constants.js'
 import { addRuleToDynamicRuleSetGenerator, isLocalHost, supportsBlock } from './redirect-handler/blockOrObserve.js'
+import { RequestTracker } from './trackers/requestTracker.js'
 
 const log = debug('ipfs-companion:request')
 log.error = debug('ipfs-companion:request:error')
@@ -32,6 +33,8 @@ const recoverableHttpError = (code) => code && code >= 400
 // Tracking late redirects for edge cases such as https://github.com/ipfs-shipyard/ipfs-companion/issues/436
 const onHeadersReceivedRedirect = new Set()
 let addRuleToDynamicRuleSet = null
+const observedRequestTracker = new RequestTracker('url-observed')
+const resolvedRequestTracker = new RequestTracker('url-resolved')
 
 // Request modifier provides event listeners for the various stages of making an HTTP request
 // API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
@@ -144,14 +147,16 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
     async onBeforeRequest (request) {
       const state = getState()
       if (!state.active) return
+      observedRequestTracker.track(request)
 
       // When local IPFS node is unreachable , show recovery page where user can redirect
       // to public gateway.
       if (!state.nodeActive && request.type === 'main_frame' && sameGateway(request.url, state.gwURL)) {
         const publicUri = await ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl: `${dropSlash(runtimeRoot)}${recoveryPagePath}#${encodeURIComponent(publicUri)}`
+          redirectUrl: `${dropSlash(runtimeRoot)}${recoveryPagePath}#${encodeURIComponent(publicUri)}`,
+          request
         })
       }
 
@@ -162,7 +167,8 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
         const redirectUrl = safeURL(request.url, { useLocalhostName: state.useSubdomains }).toString()
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl
+          redirectUrl,
+          request
         })
       }
 
@@ -171,7 +177,8 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
         const redirectUrl = safeURL(request.url, { useLocalhostName: false }).toString()
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl
+          redirectUrl,
+          request
         })
       }
 
@@ -478,8 +485,9 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
  * @param {object} input contains originUrl and redirectUrl.
  * @returns
  */
-function handleRedirection ({ originUrl, redirectUrl }) {
+function handleRedirection ({ originUrl, redirectUrl, request }) {
   if (redirectUrl !== '' && originUrl !== '' && redirectUrl !== originUrl) {
+    resolvedRequestTracker.track(request)
     if (supportsBlock) {
       return { redirectUrl }
     }
@@ -538,7 +546,8 @@ async function redirectToGateway (request, url, state, ipfsPathValidator, runtim
 
   return handleRedirection({
     originUrl: request.url,
-    redirectUrl
+    redirectUrl,
+    request
   })
 }
 
@@ -608,7 +617,8 @@ function normalizedRedirectingProtocolRequest (request, pubGwUrl) {
   if (oldPath !== path && isIPFS.path(path)) {
     return handleRedirection({
       originUrl: request.url,
-      redirectUrl: pathAtHttpGateway(path, pubGwUrl)
+      redirectUrl: pathAtHttpGateway(path, pubGwUrl),
+      request
     })
   }
   return null
@@ -653,7 +663,9 @@ function normalizedUnhandledIpfsProtocol (request, pubGwUrl) {
     // (will be redirected later, if needed)
     return handleRedirection({
       originUrl: request.url,
-      redirectUrl: pathAtHttpGateway(path, pubGwUrl)
+      redirectUrl: pathAtHttpGateway(path, pubGwUrl),
+      request
+
     })
   }
 }

diff --git a/add-on/src/lib/telemetry.ts b/add-on/src/lib/telemetry.ts
@@ -4,6 +4,7 @@ import debug from 'debug'
 import { WebExtensionStorageProvider } from './storage-provider/WebExtensionStorageProvider.js'
 import { CompanionState } from '../types/companion.js'
 import { consentTypes } from '@ipfs-shipyard/ignite-metrics'
+import type { CountlyEvent } from 'countly-web-sdk'
 
 const log = debug('ipfs-companion:telemetry')
 
@@ -51,3 +52,14 @@ export function trackView (view: string, segments: Record<string, string>): void
   log('trackView called for view: ', view)
   metricsProvider.trackView(view, ignoredViewsRegex, segments)
 }
+
+/**
+ * TrackView is a wrapper around ignite-metrics trackView
+ *
+ * @param event
+ * @param segments
+ */
+export function trackEvent (event: CountlyEvent): void {
+  log('trackEvent called for event: ', event)
+  metricsProvider.trackEvent(event)
+}
diff --git a/add-on/src/lib/trackers/requestTracker.ts b/add-on/src/lib/trackers/requestTracker.ts
@@ -0,0 +1,49 @@
+import debug from 'debug'
+import type browser from 'webextension-polyfill'
+import { trackEvent } from '../telemetry.js'
+
+export class RequestTracker {
+  private readonly eventKey: 'url-observed' | 'url-resolved'
+  private readonly flushInterval: number
+  private readonly log: debug.Debugger & { error?: debug.Debugger }
+  private lastSync: number = Date.now()
+  private requestTypeStore: { [key in browser.WebRequest.ResourceType]?: number } = {}
+
+  constructor (eventKey: 'url-observed' | 'url-resolved', flushInterval = 1000 * 60 * 5) {
+    this.eventKey = eventKey
+    this.log = debug(`ipfs-companion:request-tracker:${eventKey}`)
+    this.log.error = debug(`ipfs-companion:request-tracker:${eventKey}:error`)
+    this.flushInterval = flushInterval
+    this.setupFlushScheduler()
+  }
+
+  track ({ type }: browser.WebRequest.OnBeforeRequestDetailsType): void {
+    this.log(`track ${type}`, JSON.stringify(this.requestTypeStore))
+    this.requestTypeStore[type] = (this.requestTypeStore[type] ?? 0) + 1
+  }
+
+  private flushStore (): void {
+    this.log('flushing')
+    const count = Object.values(this.requestTypeStore).reduce((a, b): number => a + b, 0)
+    if (count === 0) {
+      this.log('nothing to flush')
+      return
+    }
+    trackEvent({
+      key: this.eventKey,
+      count,
+      dur: Date.now() - this.lastSync,
+      segmentation: Object.assign({}, this.requestTypeStore) as unknown as Record<string, string>
+    })
+    // reset
+    this.lastSync = Date.now()
+    this.requestTypeStore = {}
+  }
+
+  private setupFlushScheduler (): void {
+    setTimeout(() => {
+      this.flushStore()
+      this.setupFlushScheduler()
+    }, this.flushInterval)
+  }
+}
diff --git a/add-on/src/types/countly.d.ts b/add-on/src/types/countly.d.ts
@@ -0,0 +1,9 @@
+declare module 'countly-web-sdk' {
+  export interface CountlyEvent {
+    key: string
+    count?: number
+    sum?: number
+    dur?: number
+    segmentation?: Record<string, string>
+  }
+}
diff --git a/docs/telemetry/COLLECTED_DATA.md b/docs/telemetry/COLLECTED_DATA.md
@@ -17,7 +17,7 @@ Telemetry is sent to Countly instance at `countly.ipfs.tech`. You can read how t
 As a general rule, we collect only application data; no user data. Some examples of application data we collect are:
 
 | Metric data name    | Metric feature name | Metric trigger                                          | Analytics use                                                     | Notes         |
-|:-------------------:|---------------------|---------------------------------------------------------|-------------------------------------------------------------------|---------------|                                                  |               |
+|:-------------------:|---------------------|---------------------------------------------------------|-------------------------------------------------------------------|---------------|
 | view:welcome        | views               | When the welcome view is shown                          | View count                                                        |               |
 | view:options        | views               | When the options view is shown                          | View count                                                        |               |
 | view:quick-import   | views               | When the quick-import view is shown                     | View count                                                        |               |