feat(mv3): Tracking URL resolved/observed count.

add-on/src/lib/ipfs-companion.js

@@ -8,6 +8,7 @@ import LRU from 'lru-cache'
 import pMemoize from 'p-memoize'
 import toMultiaddr from 'uri-to-multiaddr'
 import browser from 'webextension-polyfill'
+import { handleConsentFromState, trackView } from '../lib/telemetry.js'
 import { contextMenuCopyAddressAtPublicGw, contextMenuCopyCanonicalAddress, contextMenuCopyCidAddress, contextMenuCopyPermalink, contextMenuCopyRawCid, contextMenuViewOnGateway, createContextMenus, findValueForContext } from './context-menus.js'
 import createCopier from './copier.js'
 import createDnslinkResolver from './dnslink.js'
@@ -24,7 +25,6 @@ import { guiURLString, migrateOptions, optionDefaults, safeURL, storeMissingOpti
 import { getExtraInfoSpec } from './redirect-handler/blockOrObserve.js'
 import createRuntimeChecks from './runtime-checks.js'
 import { initState, offlinePeerCount } from './state.js'
-import { handleConsentFromState, trackView } from '../lib/telemetry.js'
 
 // this won't work in webworker context. Needs to be enabled manually
 // https://github.com/debug-js/debug/issues/916

add-on/src/lib/ipfs-request.js

@@ -3,14 +3,15 @@
 
 import debug from 'debug'
 
-import LRU from 'lru-cache'
-import isIPFS from 'is-ipfs'
 import isFQDN from 'is-fqdn'
+import isIPFS from 'is-ipfs'
+import LRU from 'lru-cache'
+import { recoveryPagePath } from './constants.js'
+import { braveNodeType } from './ipfs-client/brave.js'
 import { dropSlash, ipfsUri, pathAtHttpGateway, sameGateway } from './ipfs-path.js'
 import { safeURL } from './options.js'
-import { braveNodeType } from './ipfs-client/brave.js'
-import { recoveryPagePath } from './constants.js'
 import { addRuleToDynamicRuleSetGenerator, isLocalHost, supportsBlock } from './redirect-handler/blockOrObserve.js'
+import { RequestTracker } from './trackers/requestTracker.js'
 
 const log = debug('ipfs-companion:request')
 log.error = debug('ipfs-companion:request:error')
@@ -32,6 +33,8 @@ const recoverableHttpError = (code) => code && code >= 400
 // Tracking late redirects for edge cases such as https://github.com/ipfs-shipyard/ipfs-companion/issues/436
 const onHeadersReceivedRedirect = new Set()
 let addRuleToDynamicRuleSet = null
+const observedRequestTracker = new RequestTracker('url-observed')
+const resolvedRequestTracker = new RequestTracker('url-resolved')
 
 // Request modifier provides event listeners for the various stages of making an HTTP request
 // API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
@@ -144,14 +147,16 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
     async onBeforeRequest (request) {
       const state = getState()
       if (!state.active) return
+      observedRequestTracker.track(request)
 
       // When local IPFS node is unreachable , show recovery page where user can redirect
       // to public gateway.
       if (!state.nodeActive && request.type === 'main_frame' && sameGateway(request.url, state.gwURL)) {
         const publicUri = await ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl: `${dropSlash(runtimeRoot)}${recoveryPagePath}#${encodeURIComponent(publicUri)}`
+          redirectUrl: `${dropSlash(runtimeRoot)}${recoveryPagePath}#${encodeURIComponent(publicUri)}`,
+          request
         })
       }
 
@@ -162,7 +167,8 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
         const redirectUrl = safeURL(request.url, { useLocalhostName: state.useSubdomains }).toString()
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl
+          redirectUrl,
+          request
         })
       }
 
@@ -171,7 +177,8 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
         const redirectUrl = safeURL(request.url, { useLocalhostName: false }).toString()
         return handleRedirection({
           originUrl: request.url,
-          redirectUrl
+          redirectUrl,
+          request
         })
       }
 
@@ -478,8 +485,9 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
  * @param {object} input contains originUrl and redirectUrl.
  * @returns
  */
-function handleRedirection ({ originUrl, redirectUrl }) {
+function handleRedirection ({ originUrl, redirectUrl, request }) {
   if (redirectUrl !== '' && originUrl !== '' && redirectUrl !== originUrl) {
+    resolvedRequestTracker.track(request)
     if (supportsBlock) {
       return { redirectUrl }
     }
@@ -538,7 +546,8 @@ async function redirectToGateway (request, url, state, ipfsPathValidator, runtim
 
   return handleRedirection({
     originUrl: request.url,
-    redirectUrl
+    redirectUrl,
+    request
   })
 }
 
@@ -608,7 +617,8 @@ function normalizedRedirectingProtocolRequest (request, pubGwUrl) {
   if (oldPath !== path && isIPFS.path(path)) {
     return handleRedirection({
       originUrl: request.url,
-      redirectUrl: pathAtHttpGateway(path, pubGwUrl)
+      redirectUrl: pathAtHttpGateway(path, pubGwUrl),
+      request
     })
   }
   return null
@@ -653,7 +663,9 @@ function normalizedUnhandledIpfsProtocol (request, pubGwUrl) {
     // (will be redirected later, if needed)
     return handleRedirection({
       originUrl: request.url,
-      redirectUrl: pathAtHttpGateway(path, pubGwUrl)
+      redirectUrl: pathAtHttpGateway(path, pubGwUrl),
+      request
+
     })
   }
 }

add-on/src/lib/telemetry.ts

@@ -4,6 +4,7 @@ import debug from 'debug'
 import { WebExtensionStorageProvider } from './storage-provider/WebExtensionStorageProvider.js'
 import { CompanionState } from '../types/companion.js'
 import { consentTypes } from '@ipfs-shipyard/ignite-metrics'
+import type { CountlyEvent } from 'countly-web-sdk'
 
 const log = debug('ipfs-companion:telemetry')
 
@@ -51,3 +52,14 @@ export function trackView (view: string, segments: Record<string, string>): void
   log('trackView called for view: ', view)
   metricsProvider.trackView(view, ignoredViewsRegex, segments)
 }
+
+/**
+ * TrackView is a wrapper around ignite-metrics trackView
+ *
+ * @param event
+ * @param segments
+ */
+export function trackEvent (event: CountlyEvent): void {
+  log('trackEvent called for event: ', event)
+  metricsProvider.trackEvent(event)
+}

add-on/src/lib/trackers/requestTracker.ts

@@ -0,0 +1,49 @@
+import debug from 'debug'
+import type browser from 'webextension-polyfill'
+import { trackEvent } from '../telemetry.js'
+
+export class RequestTracker {
+  private readonly eventKey: 'url-observed' | 'url-resolved'
+  private readonly flushInterval: number
+  private readonly log: debug.Debugger & { error?: debug.Debugger }
+  private lastSync: number = Date.now()
+  private requestTypeStore: { [key in browser.WebRequest.ResourceType]?: number } = {}
+
+  constructor (eventKey: 'url-observed' | 'url-resolved', flushInterval = 1000 * 60 * 5) {
+    this.eventKey = eventKey
+    this.log = debug(`ipfs-companion:request-tracker:${eventKey}`)
+    this.log.error = debug(`ipfs-companion:request-tracker:${eventKey}:error`)
+    this.flushInterval = flushInterval
+    this.setupFlushScheduler()
+  }
+
+  track ({ type }: browser.WebRequest.OnBeforeRequestDetailsType): void {
+    this.log(`track ${type}`, JSON.stringify(this.requestTypeStore))
+    this.requestTypeStore[type] = (this.requestTypeStore[type] ?? 0) + 1
+  }
+
+  private flushStore (): void {
+    this.log('flushing')
+    const count = Object.values(this.requestTypeStore).reduce((a, b): number => a + b, 0)
+    if (count === 0) {
+      this.log('nothing to flush')
+      return
+    }
+    trackEvent({
+      key: this.eventKey,
+      count,
+      dur: Date.now() - this.lastSync,
+      segmentation: Object.assign({}, this.requestTypeStore) as unknown as Record<string, string>
+    })
+    // reset
+    this.lastSync = Date.now()
+    this.requestTypeStore = {}
+  }
+
+  private setupFlushScheduler (): void {
+    setTimeout(() => {
+      this.flushStore()
+      this.setupFlushScheduler()
+    }, this.flushInterval)
+  }
+}

add-on/src/types/countly.d.ts

@@ -0,0 +1,9 @@
+declare module 'countly-web-sdk' {
+  export interface CountlyEvent {
+    key: string
+    count?: number
+    sum?: number
+    dur?: number
+    segmentation?: Record<string, string>
+  }
+}