Add 4 new params to broken site pixel metadata (#4873)

Task/Issue URL:
https://app.asana.com/0/1201870266890790/1207382663027468/f

### Description
We want to add `locale` (lang-country, e.g., `en-US`),
`userRefreshCount`, `openerContext` (serp, navigation, external) and
`jsPerformance` (only first contentful paint so far) to breakage reports
to match other platforms.

### Steps to test this PR
- [x] Override privacy configuration url to use
https://jsonblob.com/api/1263877779169861632
- [x] Check that you have content-scope-scripts v. 6.6.0+
- [x] Override
[features.js](https://github.com/duckduckgo/content-scope-scripts/blob/607d1752d7a329199469ed5b9c5ebf945e6d49ff/src/features.js#L45)
file in content-scope-scripts to add `'breakageReporting'`, rebuild and
update [the C-S-S
build.gradle](https://github.com/duckduckgo/Android/blob/a91d3ca35e2675b72c20732a18d30d2b6569bd89/content-scope-scripts/content-scope-scripts-impl/build.gradle#L75)
to use your local path
- [x] Launch and set this build as the default browser app
- [x] Perform a search, then click on a result
- [x] Refresh the page a few times (remembering the count 😁)
- [x] Send a broken site report through the 3-dot menu
- [x] Verify from logcat and/or checking that pixel `ebpf.android*` is
sent correctly with all four new params (and that refresh count
corresponds and openerContext = "serp"
- [x] Click on a link in your current tab to go to a new page
- [x] Refresh a couple more times
- [x] Send a broken site report through the duckfoot dashboard
- [x] Verify the pixel metadata params again in logcat and/or directly
(openerContext should be "navigation")
- [x] Close the app and click on a url link from email (custom tab),
notes or MM (directly to browser), or any other 3rd-party app
- [x] Repeat sending and verifying pixel params
   + [x] openerContext should be "external" in all these cases:

     - opening in a custom tab
     - opening the full browser from the custom tab
     - opening in the full browser directly from another app

---------

Co-authored-by: Ana Capatina <[email protected]>

Author: laghee

app/build.gradle

@@ -348,6 +348,8 @@ dependencies {
     implementation project(':survey-api')
     implementation project(':survey-impl')
 
+    implementation project(':breakage-reporting-impl')
+
     // Deprecated. TODO: Stop using this artifact.
     implementation "androidx.legacy:legacy-support-v4:_"
     debugImplementation Square.leakCanary.android

app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt

@@ -164,6 +164,7 @@ import com.duckduckgo.autofill.api.email.EmailManager
 import com.duckduckgo.autofill.api.passwordgeneration.AutomaticSavedLoginsMonitor
 import com.duckduckgo.autofill.impl.AutofillFireproofDialogSuppressor
 import com.duckduckgo.browser.api.UserBrowserProperties
+import com.duckduckgo.browser.api.brokensite.BrokenSiteContext
 import com.duckduckgo.common.test.CoroutineTestRule
 import com.duckduckgo.common.test.InstantSchedulersRule
 import com.duckduckgo.common.utils.DispatcherProvider
@@ -388,6 +389,9 @@ class BrowserTabViewModelTest {
     @Mock
     private lateinit var mockUserAllowListRepository: UserAllowListRepository
 
+    @Mock
+    private lateinit var mockBrokenSiteContext: BrokenSiteContext
+
     @Mock
     private lateinit var mockFileChooserCallback: ValueCallback<Array<Uri>>
 
@@ -533,6 +537,7 @@ class BrowserTabViewModelTest {
             mockBypassedSSLCertificatesRepository,
             coroutineRule.testScope,
             coroutineRule.testDispatcherProvider,
+            DuckDuckGoUrlDetectorImpl(),
         )
 
         accessibilitySettingsDataStore = AccessibilitySettingsSharedPreferences(
@@ -617,7 +622,7 @@ class BrowserTabViewModelTest {
             newTabPixels = { mockNewTabPixels },
         )
 
-        testee.loadData("abc", null, false)
+        testee.loadData("abc", null, false, false)
         testee.command.observeForever(mockCommandObserver)
     }
 
@@ -2036,21 +2041,21 @@ class BrowserTabViewModelTest {
 
     @Test
     fun whenUrlNullThenSetBrowserNotShowing() = runTest {
-        testee.loadData("id", null, false)
+        testee.loadData("id", null, false, false)
         testee.determineShowBrowser()
         assertEquals(false, testee.browserViewState.value?.browserShowing)
     }
 
     @Test
     fun whenUrlBlankThenSetBrowserNotShowing() = runTest {
-        testee.loadData("id", "  ", false)
+        testee.loadData("id", "  ", false, false)
         testee.determineShowBrowser()
         assertEquals(false, testee.browserViewState.value?.browserShowing)
     }
 
     @Test
     fun whenUrlPresentThenSetBrowserShowing() = runTest {
-        testee.loadData("id", "https://example.com", false)
+        testee.loadData("id", "https://example.com", false, false)
         testee.determineShowBrowser()
         assertEquals(true, testee.browserViewState.value?.browserShowing)
     }
@@ -4944,6 +4949,19 @@ class BrowserTabViewModelTest {
         verify(mockPrivacyProtectionsPopupManager).onPageRefreshTriggeredByUser()
     }
 
+    @Test
+    fun whenOnlyChangeInUrlIsHttpsUpgradeNakedDomainRedirectOrTrailingSlashThenConsiderSameForExternalLaunch() = runTest {
+        val urlA = "https://example.com"
+        val urlB = "http://www.example.com"
+        val urlC = "https://www.example.com/"
+        val urlD = "http://www.example.com/path/"
+
+        assertTrue(testee.urlUnchangedForExternalLaunchPurposes(urlA, urlB))
+        assertTrue(testee.urlUnchangedForExternalLaunchPurposes(urlB, urlC))
+        assertTrue(testee.urlUnchangedForExternalLaunchPurposes(urlA, urlC))
+        assertFalse(testee.urlUnchangedForExternalLaunchPurposes(urlC, urlD))
+    }
+
     @Test
     fun whenPrivacyProtectionsAreToggledThenCorrectPixelsAreSent() = runTest {
         val params = mapOf("test_key" to "test_value")
@@ -5561,7 +5579,7 @@ class BrowserTabViewModelTest {
 
     private fun givenOneActiveTabSelected() {
         selectedTabLiveData.value = TabEntity("TAB_ID", "https://example.com", "", skipHome = false, viewed = true, position = 0)
-        testee.loadData("TAB_ID", "https://example.com", false)
+        testee.loadData("TAB_ID", "https://example.com", false, false)
     }
 
     private fun givenFireproofWebsiteDomain(vararg fireproofWebsitesDomain: String) {
@@ -5577,10 +5595,11 @@ class BrowserTabViewModelTest {
         whenever(site.url).thenReturn(domain)
         whenever(site.nextUrl).thenReturn(domain)
         whenever(site.uri).thenReturn(Uri.parse(domain))
+        whenever(site.realBrokenSiteContext).thenReturn(mockBrokenSiteContext)
         val siteLiveData = MutableLiveData<Site>()
         siteLiveData.value = site
         whenever(mockTabRepository.retrieveSiteData("TAB_ID")).thenReturn(siteLiveData)
-        testee.loadData("TAB_ID", domain, false)
+        testee.loadData("TAB_ID", domain, false, false)
 
         return site
     }
@@ -5608,7 +5627,7 @@ class BrowserTabViewModelTest {
     }
 
     private fun loadTabWithId(tabId: String) {
-        testee.loadData(tabId, initialUrl = null, skipHome = false)
+        testee.loadData(tabId, initialUrl = null, skipHome = false, isExternal = false)
     }
 
     private fun loadUrl(

app/src/androidTest/java/com/duckduckgo/app/tabs/model/TabDataRepositoryTest.kt

@@ -23,6 +23,7 @@ import androidx.lifecycle.MutableLiveData
 import androidx.room.Room
 import androidx.test.platform.app.InstrumentationRegistry
 import com.duckduckgo.app.blockingObserve
+import com.duckduckgo.app.browser.DuckDuckGoUrlDetector
 import com.duckduckgo.app.browser.certificates.BypassedSSLCertificatesRepository
 import com.duckduckgo.app.browser.favicon.FaviconManager
 import com.duckduckgo.app.browser.tabpreview.WebViewPreviewPersister
@@ -411,6 +412,7 @@ class TabDataRepositoryTest {
         webViewPreviewPersister: WebViewPreviewPersister = mock(),
         faviconManager: FaviconManager = mock(),
         tabSwitcherDataStore: TabSwitcherDataStore = mock(),
+        duckDuckGoUrlDetector: DuckDuckGoUrlDetector = mock(),
     ): TabDataRepository {
         return TabDataRepository(
             dao,
@@ -421,6 +423,7 @@ class TabDataRepositoryTest {
                 bypassedSSLCertificatesRepository,
                 coroutinesTestRule.testScope,
                 coroutinesTestRule.testDispatcherProvider,
+                duckDuckGoUrlDetector,
             ),
             webViewPreviewPersister,
             faviconManager,

app/src/main/java/com/duckduckgo/app/brokensite/BrokenSiteActivity.kt

@@ -35,6 +35,7 @@ import com.duckduckgo.appbuildconfig.api.AppBuildConfig
 import com.duckduckgo.browser.api.WebViewVersionProvider
 import com.duckduckgo.browser.api.brokensite.BrokenSiteData
 import com.duckduckgo.browser.api.brokensite.BrokenSiteData.ReportFlow
+import com.duckduckgo.browser.api.brokensite.BrokenSiteOpenerContext
 import com.duckduckgo.common.ui.DuckDuckGoActivity
 import com.duckduckgo.common.ui.view.dialog.DaxAlertDialog
 import com.duckduckgo.common.ui.view.dialog.RadioListAlertDialogBuilder
@@ -89,6 +90,9 @@ class BrokenSiteActivity : DuckDuckGoActivity() {
         val httpErrorCodes = intent.getStringExtra(HTTP_ERROR_CODES).orEmpty()
         val isDesktopMode = intent.getBooleanExtra(IS_DESKTOP_MODE, false)
         val reportFlow = intent.getSerializableExtra<ReportFlow>(REPORT_FLOW)
+        val userRefreshCount = intent.getIntExtra(USER_REFRESH_COUNT, 0)
+        val openerContext = intent.getSerializableExtra<BrokenSiteOpenerContext>(OPENER_CONTEXT)
+        val jsPerformance = intent.getDoubleArrayExtra(JS_PERFORMANCE)
         viewModel.setInitialBrokenSite(
             url = url,
             blockedTrackers = blockedTrackers,
@@ -102,6 +106,9 @@ class BrokenSiteActivity : DuckDuckGoActivity() {
             httpErrorCodes = httpErrorCodes,
             isDesktopMode = isDesktopMode,
             reportFlow = reportFlow,
+            userRefreshCount = userRefreshCount,
+            openerContext = openerContext,
+            jsPerformance = jsPerformance,
         )
     }
 
@@ -248,6 +255,9 @@ class BrokenSiteActivity : DuckDuckGoActivity() {
         private const val HTTP_ERROR_CODES = "HTTP_ERROR_CODES"
         private const val IS_DESKTOP_MODE = "IS_DESKTOP_MODE"
         private const val REPORT_FLOW = "REPORT_FLOW"
+        private const val USER_REFRESH_COUNT = "USER_REFRESH_COUNT"
+        private const val OPENER_CONTEXT = "OPENER_CONTEXT"
+        private const val JS_PERFORMANCE = "JS_PERFORMANCE"
 
         fun intent(
             context: Context,
@@ -266,6 +276,9 @@ class BrokenSiteActivity : DuckDuckGoActivity() {
             intent.putExtra(HTTP_ERROR_CODES, data.httpErrorCodes)
             intent.putExtra(IS_DESKTOP_MODE, data.isDesktopMode)
             intent.putExtra(REPORT_FLOW, data.reportFlow)
+            intent.putExtra(USER_REFRESH_COUNT, data.userRefreshCount)
+            intent.putExtra(OPENER_CONTEXT, data.openerContext)
+            intent.putExtra(JS_PERFORMANCE, data.jsPerformance)
             return intent
         }
     }

app/src/main/java/com/duckduckgo/app/brokensite/BrokenSiteViewModel.kt

@@ -38,6 +38,7 @@ import com.duckduckgo.app.statistics.pixels.Pixel.PixelType.COUNT
 import com.duckduckgo.browser.api.brokensite.BrokenSiteData.ReportFlow
 import com.duckduckgo.browser.api.brokensite.BrokenSiteData.ReportFlow.DASHBOARD
 import com.duckduckgo.browser.api.brokensite.BrokenSiteData.ReportFlow.MENU
+import com.duckduckgo.browser.api.brokensite.BrokenSiteOpenerContext
 import com.duckduckgo.common.utils.SingleLiveEvent
 import com.duckduckgo.common.utils.extractDomain
 import com.duckduckgo.di.scopes.ActivityScope
@@ -112,6 +113,9 @@ class BrokenSiteViewModel @Inject constructor(
     private var httpErrorCodes: String = ""
     private var isDesktopMode: Boolean = false
     private var reportFlow: ReportFlow? = null
+    private var userRefreshCount: Int = 0
+    private var openerContext: BrokenSiteOpenerContext? = null
+    private var jsPerformance: DoubleArray? = null
 
     var shuffledCategories = mutableListOf<BrokenSiteCategory>()
 
@@ -133,6 +137,9 @@ class BrokenSiteViewModel @Inject constructor(
         httpErrorCodes: String,
         isDesktopMode: Boolean,
         reportFlow: ReportFlow?,
+        userRefreshCount: Int,
+        openerContext: BrokenSiteOpenerContext?,
+        jsPerformance: DoubleArray?,
     ) {
         this.url = url
         this.blockedTrackers = blockedTrackers
@@ -146,6 +153,9 @@ class BrokenSiteViewModel @Inject constructor(
         this.httpErrorCodes = httpErrorCodes
         this.isDesktopMode = isDesktopMode
         this.reportFlow = reportFlow
+        this.userRefreshCount = userRefreshCount
+        this.openerContext = openerContext
+        this.jsPerformance = jsPerformance
 
         loadProtectionsState()
     }
@@ -268,6 +278,9 @@ class BrokenSiteViewModel @Inject constructor(
             httpErrorCodes = httpErrorCodes,
             loginSite = loginSite,
             reportFlow = reportFlow?.mapToBrokenSiteModelReportFlow(),
+            userRefreshCount = userRefreshCount,
+            openerContext = openerContext,
+            jsPerformance = jsPerformance,
         )
     }
 

app/src/main/java/com/duckduckgo/app/brokensite/RealBrokenSiteContext.kt

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2024 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.brokensite
+
+import androidx.core.net.toUri
+import com.duckduckgo.app.browser.DuckDuckGoUrlDetector
+import com.duckduckgo.browser.api.brokensite.BrokenSiteContext
+import com.duckduckgo.browser.api.brokensite.BrokenSiteOpenerContext
+import com.duckduckgo.common.utils.isHttp
+import com.duckduckgo.common.utils.isHttps
+import com.duckduckgo.di.scopes.AppScope
+import com.squareup.anvil.annotations.ContributesBinding
+import javax.inject.Inject
+import org.json.JSONArray
+import timber.log.Timber
+
+@ContributesBinding(AppScope::class)
+class RealBrokenSiteContext @Inject constructor(
+    private val duckDuckGoUrlDetector: DuckDuckGoUrlDetector,
+) : BrokenSiteContext {
+
+    override var userRefreshCount: Int = 0
+
+    override var openerContext: BrokenSiteOpenerContext? = null
+
+    override var jsPerformance: DoubleArray? = null
+
+    override fun onUserTriggeredRefresh() {
+        userRefreshCount++
+    }
+
+    override fun inferOpenerContext(
+        referrer: String?,
+        isExternalLaunch: Boolean,
+    ) {
+        if (isExternalLaunch) {
+            openerContext = BrokenSiteOpenerContext.EXTERNAL
+        } else if (referrer != null) {
+            openerContext = when {
+                duckDuckGoUrlDetector.isDuckDuckGoUrl(referrer) -> BrokenSiteOpenerContext.SERP
+                referrer.toUri().isHttp || referrer.toUri().isHttps -> BrokenSiteOpenerContext.NAVIGATION
+                else -> null
+            }
+            Timber.d(
+                "openerContext inferred -> ${openerContext?.context}",
+            )
+        } else {
+            Timber.d("openerContext not inferred because referrer is null")
+        }
+    }
+
+    override fun recordJsPerformance(performanceMetrics: JSONArray) {
+        val recordedJsValues = DoubleArray(performanceMetrics.length())
+        for (i in 0 until performanceMetrics.length()) {
+            recordedJsValues[i] = performanceMetrics.getDouble(i)
+        }
+        jsPerformance = recordedJsValues
+        Timber.d("jsPerformance recorded as $performanceMetrics")
+    }
+}

app/src/main/java/com/duckduckgo/app/brokensite/api/BrokenSiteSender.kt

@@ -33,6 +33,7 @@ import com.duckduckgo.brokensite.api.BrokenSiteLastSentReport
 import com.duckduckgo.common.utils.DispatcherProvider
 import com.duckduckgo.common.utils.absoluteString
 import com.duckduckgo.common.utils.domain
+import com.duckduckgo.common.utils.extensions.toSanitizedLanguageTag
 import com.duckduckgo.di.scopes.AppScope
 import com.duckduckgo.experiments.api.VariantManager
 import com.duckduckgo.feature.toggles.api.FeatureToggle
@@ -86,6 +87,7 @@ class BrokenSiteSubmitter @Inject constructor(
                 !contentBlocking.isAnException(brokenSite.siteUrl)
 
             val vpnOn = runCatching { networkProtectionState.isRunning() }.getOrNull()
+            val locale = appBuildConfig.deviceLocale.toSanitizedLanguageTag()
 
             val params = mutableMapOf(
                 CATEGORY_KEY to brokenSite.category.orEmpty(),
@@ -111,6 +113,10 @@ class BrokenSiteSubmitter @Inject constructor(
                 HTTP_ERROR_CODES_KEY to brokenSite.httpErrorCodes,
                 PROTECTIONS_STATE to protectionsState.toString(),
                 VPN_ON to vpnOn.toString(),
+                LOCALE to locale,
+                USER_REFRESH_COUNT to brokenSite.userRefreshCount.toString(),
+                OPENER_CONTEXT to brokenSite.openerContext?.context.orEmpty(),
+                JS_PERFORMANCE to brokenSite.jsPerformance?.joinToString(",").orEmpty(),
             )
 
             brokenSite.reportFlow?.let { reportFlow ->
@@ -178,6 +184,10 @@ class BrokenSiteSubmitter @Inject constructor(
         private const val LOGIN_SITE = "loginSite"
         private const val REPORT_FLOW = "reportFlow"
         private const val VPN_ON = "vpnOn"
+        private const val LOCALE = "locale"
+        private const val USER_REFRESH_COUNT = "userRefreshCount"
+        private const val OPENER_CONTEXT = "openerContext"
+        private const val JS_PERFORMANCE = "jsPerformance"
     }
 }
 

app/src/main/java/com/duckduckgo/app/brokensite/model/BrokenSite.kt

@@ -18,6 +18,7 @@ package com.duckduckgo.app.brokensite.model
 
 import androidx.annotation.StringRes
 import com.duckduckgo.app.browser.R
+import com.duckduckgo.browser.api.brokensite.BrokenSiteOpenerContext
 
 data class BrokenSite(
     val category: String?,
@@ -36,6 +37,9 @@ data class BrokenSite(
     val httpErrorCodes: String,
     val loginSite: String?,
     val reportFlow: ReportFlow?,
+    val userRefreshCount: Int,
+    val openerContext: BrokenSiteOpenerContext?,
+    val jsPerformance: DoubleArray?,
 )
 
 sealed class BrokenSiteCategory(