Customizable storage (#252)

* event stream draft * extend KVS to support string and remove * replace storage implementation with event stream * draft AndroidStorage * draft InMemoryStorage * typo fix * update EventPipeline with new storage * update unit tests * fix broken unit tests * fix broken android unit tests * add unit test for event stream * add unit test for KVS * add unit test for storage and event stream * fix android unit tests * address comments * add more comments * add unit tests for android kvs * add unit tests for in memory storage * add EncryptedEventStream * finalize storage creation * finalize encrypted storage --------- Co-authored-by: Wenxi Zeng <[email protected]>
segmentio · Feb 3, 2025 · cff1a1c · cff1a1c
1 parent b958b36
commit cff1a1c
Show file tree

Hide file tree

Showing 26 changed files with 1,828 additions and 566 deletions.
diff --git a/android/src/main/java/com/segment/analytics/kotlin/android/Storage.kt b/android/src/main/java/com/segment/analytics/kotlin/android/Storage.kt
@@ -5,125 +5,52 @@ import android.content.SharedPreferences
 import com.segment.analytics.kotlin.android.utilities.AndroidKVS
 import com.segment.analytics.kotlin.core.Analytics
 import com.segment.analytics.kotlin.core.Storage
-import com.segment.analytics.kotlin.core.Storage.Companion.MAX_PAYLOAD_SIZE
 import com.segment.analytics.kotlin.core.StorageProvider
-import com.segment.analytics.kotlin.core.System
-import com.segment.analytics.kotlin.core.UserInfo
-import com.segment.analytics.kotlin.core.utilities.EventsFileManager
+import com.segment.analytics.kotlin.core.utilities.FileEventStream
+import com.segment.analytics.kotlin.core.utilities.StorageImpl
 import kotlinx.coroutines.CoroutineDispatcher
 import sovran.kotlin.Store
-import sovran.kotlin.Subscriber
-import java.io.File
 
-// Android specific
+@Deprecated("Use StorageProvider to create storage for Android instead")
 class AndroidStorage(
     context: Context,
     private val store: Store,
     writeKey: String,
     private val ioDispatcher: CoroutineDispatcher,
     directory: String? = null,
     subject: String? = null
-) : Subscriber, Storage {
+) : StorageImpl(
+    propertiesFile = AndroidKVS(context.getSharedPreferences("analytics-android-$writeKey", Context.MODE_PRIVATE)),
+    eventStream = FileEventStream(context.getDir(directory ?: "segment-disk-queue", Context.MODE_PRIVATE)),
+    store = store,
+    writeKey = writeKey,
+    fileIndexKey = if(subject == null) "segment.events.file.index.$writeKey" else "segment.events.file.index.$writeKey.$subject",
+    ioDispatcher = ioDispatcher
+)
 
-    private val sharedPreferences: SharedPreferences =
-        context.getSharedPreferences("analytics-android-$writeKey", Context.MODE_PRIVATE)
-    override val storageDirectory: File = context.getDir(directory ?: "segment-disk-queue", Context.MODE_PRIVATE)
-    internal val eventsFile =
-        EventsFileManager(storageDirectory, writeKey, AndroidKVS(sharedPreferences), subject)
-
-    override suspend fun subscribeToStore() {
-        store.subscribe(
-            this,
-            UserInfo::class,
-            initialState = true,
-            handler = ::userInfoUpdate,
-            queue = ioDispatcher
-        )
-        store.subscribe(
-            this,
-            System::class,
-            initialState = true,
-            handler = ::systemUpdate,
-            queue = ioDispatcher
-        )
-    }
-
-    override suspend fun write(key: Storage.Constants, value: String) {
-        when (key) {
-            Storage.Constants.Events -> {
-                if (value.length < MAX_PAYLOAD_SIZE) {
-                    // write to disk
-                    eventsFile.storeEvent(value)
-                } else {
-                    throw Exception("enqueued payload is too large")
-                }
-            }
-            else -> {
-                sharedPreferences.edit().putString(key.rawVal, value).apply()
-            }
-        }
-    }
-
-    /**
-     * @returns the String value for the associated key
-     * for Constants.Events it will return a file url that can be used to read the contents of the events
-     */
-    override fun read(key: Storage.Constants): String? {
-        return when (key) {
-            Storage.Constants.Events -> {
-                eventsFile.read().joinToString()
-            }
-            Storage.Constants.LegacyAppBuild -> {
-                // The legacy app build number was stored as an integer so we have to get it
-                // as an integer and convert it to a String.
-                val noBuild = -1
-                val build = sharedPreferences.getInt(key.rawVal, noBuild)
-                if (build != noBuild) {
-                    return build.toString()
-                } else {
-                    return null
-                }
-            }
-            else -> {
-                sharedPreferences.getString(key.rawVal, null)
-            }
-        }
-    }
-
-    override fun remove(key: Storage.Constants): Boolean {
-        return when (key) {
-            Storage.Constants.Events -> {
-                true
-            }
-            else -> {
-                sharedPreferences.edit().putString(key.rawVal, null).apply()
-                true
-            }
+object AndroidStorageProvider : StorageProvider {
+    override fun createStorage(vararg params: Any): Storage {
+
+        if (params.size < 2 || params[0] !is Analytics || params[1] !is Context) {
+            throw IllegalArgumentException("""
+                Invalid parameters for AndroidStorageProvider. 
+                AndroidStorageProvider requires at least 2 parameters.
+                 The first argument has to be an instance of Analytics,
+                 an the second argument has to be an instance of Context
+            """.trimIndent())
         }
-    }
 
-    override fun removeFile(filePath: String): Boolean {
-        return eventsFile.remove(filePath)
-    }
+        val analytics = params[0] as Analytics
+        val context = params[1] as Context
+        val config = analytics.configuration
 
-    override suspend fun rollover() {
-        eventsFile.rollover()
-    }
-}
+        val eventDirectory = context.getDir("segment-disk-queue", Context.MODE_PRIVATE)
+        val fileIndexKey = "segment.events.file.index.${config.writeKey}"
+        val sharedPreferences: SharedPreferences =
+            context.getSharedPreferences("analytics-android-${config.writeKey}", Context.MODE_PRIVATE)
 
-object AndroidStorageProvider : StorageProvider {
-    override fun getStorage(
-        analytics: Analytics,
-        store: Store,
-        writeKey: String,
-        ioDispatcher: CoroutineDispatcher,
-        application: Any
-    ): Storage {
-        return AndroidStorage(
-            store = store,
-            writeKey = writeKey,
-            ioDispatcher = ioDispatcher,
-            context = application as Context,
-        )
+        val propertiesFile = AndroidKVS(sharedPreferences)
+        val eventStream = FileEventStream(eventDirectory)
+        return StorageImpl(propertiesFile, eventStream, analytics.store, config.writeKey, fileIndexKey, analytics.fileIODispatcher)
     }
 }
diff --git a/android/src/main/java/com/segment/analytics/kotlin/android/utilities/AndroidKVS.kt b/android/src/main/java/com/segment/analytics/kotlin/android/utilities/AndroidKVS.kt
@@ -6,10 +6,23 @@ import com.segment.analytics.kotlin.core.utilities.KVS
 /**
  * A key-value store wrapper for sharedPreferences on Android
  */
-class AndroidKVS(val sharedPreferences: SharedPreferences) : KVS {
-    override fun getInt(key: String, defaultVal: Int): Int =
+class AndroidKVS(val sharedPreferences: SharedPreferences): KVS  {
+
+
+    override fun get(key: String, defaultVal: Int) =
         sharedPreferences.getInt(key, defaultVal)
 
-    override fun putInt(key: String, value: Int): Boolean =
+    override fun get(key: String, defaultVal: String?) =
+        sharedPreferences.getString(key, defaultVal) ?: defaultVal
+
+    override fun put(key: String, value: Int) =
         sharedPreferences.edit().putInt(key, value).commit()
+
+    override fun put(key: String, value: String) =
+        sharedPreferences.edit().putString(key, value).commit()
+
+    override fun remove(key: String): Boolean =
+        sharedPreferences.edit().remove(key).commit()
+
+    override fun contains(key: String) = sharedPreferences.contains(key)
 }
diff --git a/android/src/test/java/com/segment/analytics/kotlin/android/AndroidContextCollectorTests.kt b/android/src/test/java/com/segment/analytics/kotlin/android/AndroidContextCollectorTests.kt
@@ -149,24 +149,5 @@ class AndroidContextCollectorTests {
         }
     }
 
-
-
-    @Test
-    fun `storage directory can be customized`() {
-        val dir = "test"
-        val androidStorage = AndroidStorage(
-            appContext,
-            Store(),
-            "123",
-            UnconfinedTestDispatcher(),
-            dir
-        )
-
-        Assertions.assertTrue(androidStorage.storageDirectory.name.contains(dir))
-        Assertions.assertTrue(androidStorage.eventsFile.directory.name.contains(dir))
-        Assertions.assertTrue(androidStorage.storageDirectory.exists())
-        Assertions.assertTrue(androidStorage.eventsFile.directory.exists())
-    }
-
     private fun JsonElement?.asString(): String? = this?.jsonPrimitive?.content
 }
diff --git a/android/src/test/java/com/segment/analytics/kotlin/android/StorageTests.kt b/android/src/test/java/com/segment/analytics/kotlin/android/StorageTests.kt
@@ -41,7 +41,7 @@ class StorageTests {
     @Nested
     inner class Android {
         private var store = Store()
-        private lateinit var androidStorage: AndroidStorage
+        private lateinit var androidStorage: Storage
         private var mockContext: Context = mockContext()
 
         init {
@@ -74,7 +74,7 @@ class StorageTests {
                 "123",
                 UnconfinedTestDispatcher()
             )
-            androidStorage.subscribeToStore()
+            androidStorage.initialize()
         }
 
 
@@ -208,9 +208,12 @@ class StorageTests {
                     }
                 val stringified: String = Json.encodeToString(event)
                 androidStorage.write(Storage.Constants.Events, stringified)
-                androidStorage.eventsFile.rollover()
-                val storagePath = androidStorage.eventsFile.read()[0]
-                val storageContents = File(storagePath).readText()
+                androidStorage.rollover()
+                val storagePath = androidStorage.read(Storage.Constants.Events)?.let{
+                    it.split(',')[0]
+                }
+                assertNotNull(storagePath)
+                val storageContents = File(storagePath!!).readText()
                 val jsonFormat = Json.decodeFromString(JsonObject.serializer(), storageContents)
                 assertEquals(1, jsonFormat["batch"]!!.jsonArray.size)
             }
@@ -229,8 +232,8 @@ class StorageTests {
                     e
                 }
                 assertNotNull(exception)
-                androidStorage.eventsFile.rollover()
-                assertTrue(androidStorage.eventsFile.read().isEmpty())
+                androidStorage.rollover()
+                assertTrue(androidStorage.read(Storage.Constants.Events).isNullOrEmpty())
             }
 
             @Test
@@ -248,7 +251,7 @@ class StorageTests {
                 val stringified: String = Json.encodeToString(event)
                 androidStorage.write(Storage.Constants.Events, stringified)
 
-                androidStorage.eventsFile.rollover()
+                androidStorage.rollover()
                 val fileUrl = androidStorage.read(Storage.Constants.Events)
                 assertNotNull(fileUrl)
                 fileUrl!!.let {
@@ -270,7 +273,7 @@ class StorageTests {
 
             @Test
             fun `reading events with empty storage return empty list`() = runTest {
-                androidStorage.eventsFile.rollover()
+                androidStorage.rollover()
                 val fileUrls = androidStorage.read(Storage.Constants.Events)
                 assertTrue(fileUrls!!.isEmpty())
             }

diff --git a/android/src/test/java/com/segment/analytics/kotlin/android/utilities/AndroidKVSTest.kt b/android/src/test/java/com/segment/analytics/kotlin/android/utilities/AndroidKVSTest.kt
@@ -0,0 +1,44 @@
+package com.segment.analytics.kotlin.android.utilities
+
+import com.segment.analytics.kotlin.android.utils.MemorySharedPreferences
+import com.segment.analytics.kotlin.core.utilities.KVS
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+
+class AndroidKVSTest {
+
+    private lateinit var prefs: KVS
+
+    @BeforeEach
+    fun setup(){
+        val sharedPreferences = MemorySharedPreferences()
+        prefs = AndroidKVS(sharedPreferences)
+        prefs.put("int", 1)
+        prefs.put("string", "string")
+    }
+
+    @Test
+    fun getTest() {
+        Assertions.assertEquals(1, prefs.get("int", 0))
+        Assertions.assertEquals("string", prefs.get("string", null))
+        Assertions.assertEquals(0, prefs.get("keyNotExists", 0))
+        Assertions.assertEquals(null, prefs.get("keyNotExists", null))
+    }
+
+    @Test
+    fun putTest() {
+        prefs.put("int", 2)
+        prefs.put("string", "stringstring")
+
+        Assertions.assertEquals(2, prefs.get("int", 0))
+        Assertions.assertEquals("stringstring", prefs.get("string", null))
+    }
+
+    @Test
+    fun containsAndRemoveTest() {
+        Assertions.assertTrue(prefs.contains("int"))
+        prefs.remove("int")
+        Assertions.assertFalse(prefs.contains("int"))
+    }
+}
diff --git a/core/src/main/java/com/segment/analytics/kotlin/core/Analytics.kt b/core/src/main/java/com/segment/analytics/kotlin/core/Analytics.kt
@@ -43,14 +43,8 @@ open class Analytics protected constructor(
     }
 
     // use lazy to avoid the instance being leak before fully initialized
-    val storage: Storage by lazy {
-        configuration.storageProvider.getStorage(
-            analytics = this,
-            writeKey = configuration.writeKey,
-            ioDispatcher = fileIODispatcher,
-            store = store,
-            application = configuration.application!!
-        )
+    open val storage: Storage by lazy {
+        configuration.storageProvider.createStorage(this, configuration.application!!)
     }
 
     internal var userInfo: UserInfo = UserInfo.defaultState(storage)
@@ -134,7 +128,7 @@ open class Analytics protected constructor(
                 it.provide(System.defaultState(configuration, storage))
 
                 // subscribe to store after state is provided
-                storage.subscribeToStore()
+                storage.initialize()
                 Telemetry.subscribe(store)
             }
 

diff --git a/core/src/main/java/com/segment/analytics/kotlin/core/Configuration.kt b/core/src/main/java/com/segment/analytics/kotlin/core/Configuration.kt
@@ -25,7 +25,7 @@ import sovran.kotlin.Store
 data class Configuration(
     val writeKey: String,
     var application: Any? = null,
-    val storageProvider: StorageProvider = ConcreteStorageProvider,
+    var storageProvider: StorageProvider = ConcreteStorageProvider,
     var collectDeviceId: Boolean = false,
     var trackApplicationLifecycleEvents: Boolean = false,
     var useLifecycleObserver: Boolean = false,