feat(MBL-2543): reject secret keys and pick region from key

AGENTS.md

@@ -114,8 +114,7 @@ CustomerIOBuilder(application, <CDP_API_KEY>).apply {
     // Optional: Enable in-app messaging by adding siteId and Region
     addCustomerIOModule(
         ModuleMessagingInApp(
-            // If you're in the EU, set Region.EU
-            MessagingInAppModuleConfig.Builder(<SITE_ID>, Region.US).build()
+            MessagingInAppModuleConfig.Builder(<SITE_ID>).build()
     )
     )
 

core/src/main/kotlin/io/customer/sdk/core/pipeline/DataPipeline.kt

@@ -1,6 +1,7 @@
 package io.customer.sdk.core.pipeline
 
 import io.customer.base.internal.InternalCustomerIOApi
+import io.customer.sdk.data.model.Region
 
 /**
  * Abstraction for sending track events to the data pipeline.
@@ -18,5 +19,10 @@ interface DataPipeline {
      * a consumer may gate on it immediately after identify() without racing async propagation.
      */
     val isUserIdentified: Boolean
+
+    // The API key and region the SDK was set up with, so modules use the same ones
+    val cdpApiKey: String
+    val region: Region
+
     fun track(name: String, properties: Map<String, Any?>)
 }

core/src/main/kotlin/io/customer/sdk/core/util/CioApiKey.kt

@@ -0,0 +1,32 @@
+package io.customer.sdk.core.util
+
+import io.customer.base.internal.InternalCustomerIOApi
+import io.customer.sdk.data.model.Region
+
+/**
+ * Reads the prefix of a Customer.io API key: `{sk|pk}_{us|eu}_...`.
+ * `sk_` keys are secret and must stay on servers, `pk_` keys are public and safe to ship in apps.
+ * The region is read from any key shaped `xx_{us|eu}_`, so only the kind checks depend on the prefix names.
+ * Legacy keys have no prefix. The server validates the rest of the key.
+ */
+@InternalCustomerIOApi
+object CioApiKey {
+    private val prefixRegex = Regex("^[a-z]{2}_(us|eu)_")
+
+    const val SECRET_KEY_ERROR = "Secret (sk_) keys must not be used in apps. Use your public key in apps."
+
+    /**
+     * Any `sk_` key is secret, even with an unknown region, so it is never sent from an app.
+     */
+    fun isSecretKey(key: String): Boolean = key.startsWith("sk_")
+
+    fun isPublicKey(key: String): Boolean = key.startsWith("pk_")
+
+    /**
+     * Region encoded in the key prefix, or null for legacy keys.
+     */
+    fun region(key: String): Region? {
+        val code = prefixRegex.find(key)?.groupValues?.get(1) ?: return null
+        return Region.getRegion(code)
+    }
+}

core/src/test/java/io/customer/sdk/core/util/CioApiKeyTest.kt

@@ -0,0 +1,63 @@
+package io.customer.sdk.core.util
+
+import io.customer.commontest.core.JUnit5Test
+import io.customer.sdk.data.model.Region
+import org.amshove.kluent.shouldBe
+import org.amshove.kluent.shouldBeFalse
+import org.amshove.kluent.shouldBeNull
+import org.amshove.kluent.shouldBeTrue
+import org.junit.jupiter.api.Test
+
+class CioApiKeyTest : JUnit5Test() {
+
+    @Test
+    fun isSecretKey_givenSecretKey_expectTrue() {
+        CioApiKey.isSecretKey(SECRET_US_KEY).shouldBeTrue()
+        CioApiKey.isPublicKey(SECRET_US_KEY).shouldBeFalse()
+    }
+
+    @Test
+    fun isPublicKey_givenPublicKey_expectTrue() {
+        CioApiKey.isPublicKey(PUBLIC_EU_KEY).shouldBeTrue()
+        CioApiKey.isSecretKey(PUBLIC_EU_KEY).shouldBeFalse()
+    }
+
+    @Test
+    fun isPublicKey_givenLegacyKey_expectFalse() {
+        CioApiKey.isPublicKey(LEGACY_KEY).shouldBeFalse()
+        CioApiKey.isSecretKey(LEGACY_KEY).shouldBeFalse()
+    }
+
+    @Test
+    fun isPublicKey_givenAnyPkPrefix_expectTrue() {
+        CioApiKey.isPublicKey("pk_ap_$KEY_BODY").shouldBeTrue()
+        CioApiKey.isPublicKey("pk_$KEY_BODY").shouldBeTrue()
+    }
+
+    @Test
+    fun isSecretKey_givenAnySkPrefix_expectTrue() {
+        CioApiKey.isSecretKey("sk_ap_$KEY_BODY").shouldBeTrue()
+        CioApiKey.isSecretKey("sk_$KEY_BODY").shouldBeTrue()
+    }
+
+    @Test
+    fun region_givenPrefixedKeys_expectRegionFromPrefix() {
+        CioApiKey.region(PUBLIC_EU_KEY) shouldBe Region.EU
+        CioApiKey.region(PUBLIC_US_KEY) shouldBe Region.US
+        CioApiKey.region(SECRET_US_KEY) shouldBe Region.US
+        CioApiKey.region("xk_eu_$KEY_BODY") shouldBe Region.EU
+    }
+
+    @Test
+    fun region_givenLegacyKey_expectNull() {
+        CioApiKey.region(LEGACY_KEY).shouldBeNull()
+    }
+
+    companion object {
+        private const val KEY_BODY = "0123456789abcdefABCDEF0123456789_a1B2c3"
+        const val PUBLIC_US_KEY = "pk_us_$KEY_BODY"
+        const val PUBLIC_EU_KEY = "pk_eu_$KEY_BODY"
+        const val SECRET_US_KEY = "sk_us_$KEY_BODY"
+        const val LEGACY_KEY = "4f3b2a1c0d9e8f7a6b5c"
+    }
+}

datapipelines/src/main/kotlin/io/customer/datapipelines/config/DataPipelinesModuleConfig.kt

@@ -31,6 +31,8 @@ class DataPipelinesModuleConfig(
     // Determines how SDK should handle screen view events
     val screenViewUse: ScreenView
 ) : CustomerIOModuleConfig {
+    // Shared with other modules through DataPipeline so they use the same region
+    internal val region: Region = region
     val apiHost: String = apiHostOverride ?: region.apiHost()
     val cdnHost: String = cdnHostOverride ?: region.cdnHost()
 

datapipelines/src/main/kotlin/io/customer/datapipelines/migration/AnalyticsStorageMigration.kt

@@ -0,0 +1,215 @@
+package io.customer.datapipelines.migration
+
+import android.content.Context
+import android.content.SharedPreferences
+import com.segment.analytics.kotlin.core.Storage
+import io.customer.datapipelines.plugins.CUSTOMER_IO_DATA_PIPELINES
+import io.customer.sdk.core.util.CioApiKey
+import io.customer.sdk.core.util.Logger
+import java.io.File
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonObject
+
+/**
+ * Segment keeps analytics storage per write key: identity in the `analytics-android-<key>` prefs and
+ * queued events in `<key>-<index>` files in the shared `segment-disk-queue` folder. When an app
+ * switches to a public (`pk_`) key, this keeps the user's identity and their unsent events.
+ *
+ * Finished batches need nothing: analytics uploads every finished file in the folder, and each one
+ * carries the key it was written with. Only unfinished batches of this SDK's earlier keys are moved,
+ * since analytics only ever finishes the current key's batch.
+ */
+internal class AnalyticsStorageMigration(
+    private val context: Context,
+    private val logger: Logger
+) {
+    /**
+     * Returns the key the SDK ran with before [newKey]. Uses [savedKey] when there is one. SDK versions
+     * before 4.5.3 didn't save their key, so on a switch to a public key this looks through the keys
+     * with analytics storage instead. Keys whose saved analytics settings name the Customer.io
+     * destination are this SDK's, and the most recently used one is taken. Segment's own SDK, which
+     * an app can run too, never names ours there, so its storage is never taken. If
+     * none name ours, a single other key is taken, and several are left alone.
+     */
+    fun previousKey(savedKey: String?, newKey: String): String? {
+        if (!savedKey.isNullOrEmpty() || !CioApiKey.isPublicKey(newKey)) return savedKey
+
+        return runCatching {
+            val prefsFiles = File(context.applicationInfo.dataDir, PREFS_DIRECTORY).listFiles()
+                ?.filter { file -> file.name.startsWith(PREFS_PREFIX) && file.name.endsWith(PREFS_SUFFIX) }
+                .orEmpty()
+            val keysByLastUse = prefsFiles
+                .sortedByDescending { file -> file.lastModified() }
+                .map { file -> file.name.removePrefix(PREFS_PREFIX).removeSuffix(PREFS_SUFFIX) }
+                .filter { key -> key.isNotEmpty() && key != newKey }
+            val keysBySettings = keysByLastUse.groupBy { key -> savedSettingsOwner(key) }
+            val ownKeys = keysBySettings[SettingsOwner.CUSTOMER_IO].orEmpty()
+            val unknownKeys = keysBySettings[SettingsOwner.UNKNOWN].orEmpty()
+            when {
+                ownKeys.isNotEmpty() -> ownKeys.first()
+                unknownKeys.size == 1 -> unknownKeys.first()
+                else -> {
+                    if (keysByLastUse.isNotEmpty()) {
+                        logger.info("Found ${keysByLastUse.size} earlier analytics storages and none known to be this SDK's, so none were moved to the new public key")
+                    }
+                    null
+                }
+            }
+        }.onFailure { ex ->
+            logger.error("Failed to find previous analytics key: ${ex.message}")
+        }.getOrNull()
+    }
+
+    // Analytics saves the settings it has for each key, and this SDK always lists its destination there
+    private fun savedSettingsOwner(key: String): SettingsOwner {
+        val settings = prefs(key).getString(Storage.Constants.Settings.rawVal, null) ?: return SettingsOwner.UNKNOWN
+        val integrations = runCatching { Json.parseToJsonElement(settings).jsonObject[INTEGRATIONS]?.jsonObject }
+            .getOrNull() ?: return SettingsOwner.UNKNOWN
+        return if (integrations.containsKey(CUSTOMER_IO_DATA_PIPELINES)) SettingsOwner.CUSTOMER_IO else SettingsOwner.OTHER
+    }
+
+    private enum class SettingsOwner { CUSTOMER_IO, OTHER, UNKNOWN }
+
+    /**
+     * Copies the previous key's identity, app version and build, and device id to the new key, keeping
+     * them for the previous key too. Analytics compares the saved build to tell "Application Installed"
+     * from "Application Updated", so without it every switching user would look newly installed.
+     * Must run before the Analytics instance for the new key is created, as it reads these then.
+     */
+    fun copyIdentity(previousKey: String?, newKey: String) {
+        if (!shouldMigrate(previousKey, newKey)) return
+
+        runCatching {
+            val newPrefs = prefs(newKey)
+            val previousValues = prefs(previousKey).all
+            val editor = newPrefs.edit()
+            // Identity is all or nothing on purpose: if the new key has any of it, all of it is kept,
+            // since one key's anonymousId with another key's userId would be a different user
+            if (IDENTITY_KEYS.none { newPrefs.contains(it) }) {
+                IDENTITY_KEYS.forEach { key -> editor.putValue(key, previousValues[key]) }
+            }
+            // App and device values stand alone, so each missing one is filled in
+            APP_KEYS.filterNot { newPrefs.contains(it) }.forEach { key -> editor.putValue(key, previousValues[key]) }
+            editor.commit()
+        }.onFailure { ex ->
+            logger.error("Failed to copy analytics identity to new public key: ${ex.message}")
+        }
+    }
+
+    /**
+     * Adds the events from the unfinished batches of [previousKey] and other keys known to be this
+     * SDK's to the new key's queue as they are, keeping their messageId, timestamp and userId, then
+     * deletes those batches. Includes batches left by an earlier key change that didn't finish. Other
+     * keys' batches, like those of Segment's own SDK running in the same app, are left alone. Runs on
+     * [dispatcher].
+     */
+    fun moveUnfinishedEvents(
+        previousKey: String?,
+        newKey: String,
+        scope: CoroutineScope,
+        dispatcher: CoroutineDispatcher,
+        storage: Storage
+    ) {
+        if (!shouldMigrate(previousKey, newKey)) return
+
+        scope.launch(dispatcher) {
+            runCatching {
+                val eventsDirectory = context.getDir(EVENTS_DIRECTORY, Context.MODE_PRIVATE)
+                // Analytics only finishes the current key's batch, so any other key's is left over
+                val unfinishedBatches = eventsDirectory.listFiles()
+                    ?.filter { file -> file.name.endsWith(TMP_SUFFIX) }
+                    ?.filter { file ->
+                        val batchKey = file.name.removeSuffix(TMP_SUFFIX).substringBeforeLast('-')
+                        batchKey != newKey &&
+                            (batchKey == previousKey || savedSettingsOwner(batchKey) == SettingsOwner.CUSTOMER_IO)
+                    }
+                    .orEmpty()
+                unfinishedBatches.forEach { file ->
+                    // Unknown formats are kept rather than deleted without moving their events
+                    val events = completeEvents(file) ?: return@forEach
+                    events.forEach { event -> storage.write(Storage.Constants.Events, event) }
+                    file.delete()
+                }
+            }.onFailure { ex ->
+                logger.error("Failed to move queued analytics events to new public key: ${ex.message}")
+            }
+        }
+    }
+
+    // Only when switching to a public key, legacy key changes behave as before
+    private fun shouldMigrate(previousKey: String?, newKey: String): Boolean =
+        CioApiKey.isPublicKey(newKey) && !previousKey.isNullOrEmpty() && previousKey != newKey
+
+    /**
+     * An unfinished batch is `{"batch":[` then events split by commas, with no closing yet. Returns
+     * each complete event as written, dropping a last event cut off part way, or null for any other format.
+     */
+    private fun completeEvents(file: File): List<String>? {
+        val contents = file.readText()
+        if (!contents.startsWith(BATCH_START)) return null
+
+        val events = mutableListOf<String>()
+        var depth = 0
+        var inString = false
+        var escaped = false
+        var eventStart = -1
+        for (index in BATCH_START.length until contents.length) {
+            val char = contents[index]
+            when {
+                escaped -> escaped = false
+                inString && char == '\\' -> escaped = true
+                char == '"' -> inString = !inString
+                inString -> Unit
+                char == '{' -> {
+                    if (depth == 0) eventStart = index
+                    depth++
+                }
+                char == '}' -> {
+                    depth--
+                    if (depth == 0) events.add(contents.substring(eventStart, index + 1))
+                }
+            }
+        }
+        return events
+    }
+
+    // Kept as the same type, since the legacy build is saved as a number
+    private fun SharedPreferences.Editor.putValue(key: String, value: Any?) {
+        when (value) {
+            is String -> putString(key, value)
+            is Int -> putInt(key, value)
+            is Long -> putLong(key, value)
+        }
+    }
+
+    private fun prefs(key: String?): SharedPreferences =
+        context.getSharedPreferences("$PREFS_PREFIX$key", Context.MODE_PRIVATE)
+
+    internal companion object {
+        // Names used by Segment's AndroidStorageProvider
+        const val PREFS_PREFIX = "analytics-android-"
+        const val EVENTS_DIRECTORY = "segment-disk-queue"
+        private const val PREFS_DIRECTORY = "shared_prefs"
+        private const val PREFS_SUFFIX = ".xml"
+        private const val INTEGRATIONS = "integrations"
+        private const val TMP_SUFFIX = ".tmp"
+        private const val BATCH_START = "{\"batch\":["
+
+        private val IDENTITY_KEYS = listOf(
+            Storage.Constants.AnonymousId.rawVal,
+            Storage.Constants.UserId.rawVal,
+            Storage.Constants.Traits.rawVal
+        )
+
+        // Read by analytics to tell an install from an update, and to keep the same device id
+        private val APP_KEYS = listOf(
+            Storage.Constants.AppVersion.rawVal,
+            Storage.Constants.AppBuild.rawVal,
+            Storage.Constants.LegacyAppBuild.rawVal,
+            Storage.Constants.DeviceId.rawVal
+        )
+    }
+}