chore: intial draft for using geoipdb from path

aanorbel · aanorbel · commit 9b1e850f50f7 · 2025-10-24T11:32:46.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -25,5 +25,6 @@ output
 composeApp/src/desktopMain/resources/windows/WinSparkle.*
 
 /composeApp/src/desktopMain/frameworks/
+*.mmdb
 
 certificates/*.pem
diff --git a/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt b/composeApp/src/androidMain/kotlin/org/ooni/engine/AndroidOonimkallBridge.kt
@@ -63,6 +63,10 @@ class AndroidOonimkallBridge : OonimkallBridge {
             it.softwareVersion = softwareVersion
 
             it.assetsDir = assetsDir
+            // geoipDB may not exist in Android binding; set reflectively if available
+            geoIpDB?.let { path ->
+                it.geoipDB = path
+            }
             it.stateDir = stateDir
             it.tempDir = tempDir
             it.tunnelDir = tunnelDir
diff --git a/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt b/composeApp/src/androidMain/kotlin/org/ooni/probe/net/Http.android.kt
@@ -0,0 +1,24 @@
+package org.ooni.probe.net
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import java.net.HttpURLConnection
+import java.net.URL
+
+actual suspend fun httpGetBytes(url: String): ByteArray =
+    withContext(Dispatchers.IO) {
+        val connection = (URL(url).openConnection() as HttpURLConnection)
+        connection.requestMethod = "GET"
+        connection.instanceFollowRedirects = true
+        connection.connectTimeout = 15000
+        connection.readTimeout = 30000
+        try {
+            val code = connection.responseCode
+            val stream = if (code in 200..299) connection.inputStream else connection.errorStream
+            val bytes = stream?.use { it.readBytes() } ?: ByteArray(0)
+            if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${String(bytes)}")
+            bytes
+        } finally {
+            connection.disconnect()
+        }
+    }
diff --git a/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml b/composeApp/src/commonMain/composeResources/values/strings-untranslatable.xml
@@ -16,4 +16,5 @@
     <string name="r2160p_ext" translatable="false">2160p (4k)</string>
 
     <string name="twoParam" translatable="false">%1$s %2$s</string>
+    <string name="engine_mmdb_version" translatable="false">20250801</string>
 </resources>
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/Engine.kt
@@ -148,7 +148,7 @@ class Engine(
 
     private fun session(sessionConfig: OonimkallBridge.SessionConfig): OonimkallBridge.Session = bridge.newSession(sessionConfig)
 
-    private fun buildTaskSettings(
+    private suspend fun buildTaskSettings(
         netTest: NetTest,
         taskOrigin: TaskOrigin,
         preferences: EnginePreferences,
@@ -166,6 +166,7 @@ class Engine(
         tunnelDir = "$baseFilePath/tunnel",
         tempDir = cacheDir,
         assetsDir = "$baseFilePath/assets",
+        geoIpDB = preferences.geoipDbVersion?.let { "$cacheDir/$it.mmdb" },
         options = TaskSettings.Options(
             noCollector = !preferences.uploadResults,
             softwareName = buildSoftwareName(taskOrigin),
@@ -196,7 +197,7 @@ class Engine(
         MAX_RUNTIME_DISABLED
     }
 
-    private fun buildSessionConfig(
+    private suspend fun buildSessionConfig(
         taskOrigin: TaskOrigin,
         preferences: EnginePreferences,
     ) = OonimkallBridge.SessionConfig(
@@ -208,6 +209,7 @@ class Engine(
         tunnelDir = "$baseFilePath/tunnel",
         tempDir = cacheDir,
         assetsDir = "$baseFilePath/assets",
+        geoIpDB = preferences.geoipDbVersion?.let { "$cacheDir/$it.mmdb" },
         logger = oonimkallLogger,
         verbose = false,
     )
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/OonimkallBridge.kt
@@ -31,6 +31,7 @@ interface OonimkallBridge {
         val proxy: String?,
         val probeServicesURL: String?,
         val assetsDir: String,
+        val geoIpDB: String?,
         val stateDir: String,
         val tempDir: String,
         val tunnelDir: String,
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/TaskEventMapper.kt
@@ -96,9 +96,15 @@ class TaskEventMapper(
                     asn = value?.probeAsn,
                     ip = value?.probeIp,
                     countryCode = value?.probeCc,
+                    geoIpdb = value?.geoIpdb,
                     networkType = networkTypeFinder(),
                 )
 
+            "status.resolver_lookup" -> value?.geoIpdb?.let {
+                println(it)
+                null
+            }
+
             "status.measurement_done" ->
                 TaskEvent.MeasurementDone(index = value?.idx ?: 0)
 
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/EnginePreferences.kt
@@ -7,5 +7,6 @@ data class EnginePreferences(
     val taskLogLevel: TaskLogLevel,
     val uploadResults: Boolean,
     val proxy: String?,
+    val geoipDbVersion: String?,
     val maxRuntime: Duration?,
 )
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEvent.kt
@@ -15,6 +15,7 @@ sealed interface TaskEvent {
         val ip: String?,
         val asn: String?,
         val countryCode: String?,
+        val geoIpdb: String?,
         val networkType: NetworkType,
     ) : TaskEvent
 
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt
@@ -26,6 +26,8 @@ data class TaskEventResult(
         var idx: Int = 0,
         @SerialName("report_id")
         var reportId: String? = null,
+        @SerialName("geoip_db")
+        var geoIpdb: String? = null,
         @SerialName("probe_ip")
         var probeIp: String? = null,
         @SerialName("probe_asn")
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskSettings.kt
@@ -15,6 +15,7 @@ data class TaskSettings(
     @SerialName("temp_dir") val tempDir: String,
     @SerialName("tunnel_dir") val tunnelDir: String,
     @SerialName("assets_dir") val assetsDir: String,
+    @SerialName("geoip_db") val geoIpDB: String?,
     @SerialName("options") val options: Options,
     @SerialName("annotations") val annotations: Annotations,
     // Optional
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt
@@ -119,6 +119,10 @@ fun App(
         // ForegroundServiceDidNotStartInTimeException some users are getting
         // dependencies.startSingleRunInner(RunSpecification.OnlyUploadMissingResults)
     }
+    LaunchedEffect(Unit) {
+        // Check for GeoIP DB updates in the background
+        runCatching { dependencies.fetchGeoIpDbUpdates() }
+    }
     LaunchedEffect(Unit) {
         dependencies.observeAndConfigureAutoUpdate()
     }
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/SettingsKey.kt
@@ -65,6 +65,8 @@ enum class SettingsKey(
     DELETE_UPLOADED_JSONS("deleteUploadedJsons"),
     IS_NOTIFICATION_DIALOG("isNotificationDialog"),
     FIRST_RUN("first_run"),
+    MMDB_VERSION("mmdb_version"),
+    MMDB_LAST_CHECK("mmdb_last_check"),
     CHOSEN_WEBSITES("chosen_websites"),
     DESCRIPTOR_SECTIONS_COLLAPSED("descriptor_sections_collapsed"),
 
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/PreferenceRepository.kt
@@ -75,11 +75,13 @@ class PreferenceRepository(
     ): PreferenceKey<*> {
         val preferenceKey = getPreferenceKey(name = key.value, prefix = prefix, autoRun = autoRun)
         return when (key) {
+            SettingsKey.MMDB_LAST_CHECK,
             SettingsKey.MAX_RUNTIME,
             SettingsKey.LEGACY_PROXY_PORT,
             SettingsKey.DELETE_OLD_RESULTS_THRESHOLD,
             -> PreferenceKey.IntKey(intPreferencesKey(preferenceKey))
 
+            SettingsKey.MMDB_VERSION,
             SettingsKey.LEGACY_PROXY_HOSTNAME,
             SettingsKey.LEGACY_PROXY_PROTOCOL,
             SettingsKey.PROXY_SELECTED,
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/di/Dependencies.kt
@@ -17,6 +17,7 @@ import org.ooni.engine.Engine
 import org.ooni.engine.NetworkTypeFinder
 import org.ooni.engine.OonimkallBridge
 import org.ooni.engine.TaskEventMapper
+import org.ooni.probe.net.httpGetBytes
 import org.ooni.probe.Database
 import org.ooni.probe.background.RunBackgroundTask
 import org.ooni.probe.config.BatteryOptimization
@@ -50,7 +51,9 @@ import org.ooni.probe.domain.ClearStorage
 import org.ooni.probe.domain.DeleteMeasurementsWithoutResult
 import org.ooni.probe.domain.DeleteOldResults
 import org.ooni.probe.domain.DeleteResults
+import org.ooni.probe.domain.DownloadFile
 import org.ooni.probe.domain.DownloadUrls
+import org.ooni.probe.domain.FetchGeoIpDbUpdates
 import org.ooni.probe.domain.FinishInProgressData
 import org.ooni.probe.domain.GetAutoRunSettings
 import org.ooni.probe.domain.GetAutoRunSpecification
@@ -116,6 +119,7 @@ import org.ooni.probe.ui.settings.proxy.ProxyViewModel
 import org.ooni.probe.ui.settings.webcategories.WebCategoriesViewModel
 import org.ooni.probe.ui.upload.UploadMeasurementsViewModel
 import kotlin.coroutines.CoroutineContext
+import kotlin.getValue
 
 class Dependencies(
     val platformInfo: PlatformInfo,
@@ -201,9 +205,25 @@ class Dependencies(
     }
 
     // Engine
-
     private val taskEventMapper by lazy { TaskEventMapper(networkTypeFinder, json) }
 
+    private val downloader by lazy {
+        DownloadFile(
+            fileSystem = FileSystem.SYSTEM,
+            fetchBytes = { url -> httpGetBytes(url) },
+        )
+    }
+
+    val fetchGeoIpDbUpdates by lazy {
+        FetchGeoIpDbUpdates(
+            downloadFile = downloader::invoke,
+            cacheDir = cacheDir,
+            engineHttpDo = engine::httpDo,
+            json = json,
+            preferencesRepository = preferenceRepository,
+        )
+    }
+
     @VisibleForTesting
     val engine by lazy {
         Engine(
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/DownloadFile.kt
@@ -0,0 +1,32 @@
+package org.ooni.probe.domain
+
+import okio.FileSystem
+import okio.Path
+import okio.Path.Companion.toPath
+import okio.buffer
+import okio.use
+
+/**
+ * Downloads binary content to a target absolute path using the provided fetcher.
+ * - Creates parent directories if needed
+ * - Skips writing if the target already exists with the same size
+ */
+class DownloadFile(
+    private val fileSystem: FileSystem,
+    private val fetchBytes: suspend (url: String) -> ByteArray,
+) {
+    suspend operator fun invoke(
+        url: String,
+        absoluteTargetPath: String,
+    ): Path {
+        val target = absoluteTargetPath.toPath()
+        target.parent?.let { parent ->
+            if (fileSystem.metadataOrNull(parent) == null) fileSystem.createDirectories(parent)
+        }
+        val bytes = fetchBytes(url)
+        val existing = fileSystem.metadataOrNull(target)
+        if (existing?.size == bytes.size.toLong()) return target
+        fileSystem.sink(target).buffer().use { sink -> sink.write(bytes) }
+        return target
+    }
+}
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/FetchGeoIpDbUpdates.kt
@@ -0,0 +1,82 @@
+package org.ooni.probe.domain
+
+import kotlinx.coroutines.flow.first
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import okio.Path
+import ooniprobe.composeapp.generated.resources.Res
+import ooniprobe.composeapp.generated.resources.engine_mmdb_version
+import org.jetbrains.compose.resources.getString
+import org.ooni.engine.Engine
+import org.ooni.engine.models.Failure
+import org.ooni.engine.models.Result
+import org.ooni.engine.models.Success
+import org.ooni.engine.models.TaskOrigin
+import org.ooni.probe.data.models.SettingsKey
+import org.ooni.probe.data.repositories.PreferenceRepository
+import kotlin.time.Clock
+
+class FetchGeoIpDbUpdates(
+    private val downloadFile: suspend (url: String, absoluteTargetPath: String) -> Path,
+    private val cacheDir: String,
+    private val engineHttpDo: suspend (method: String, url: String, taskOrigin: TaskOrigin) -> Result<String?, Engine.MkException>,
+    private val preferencesRepository: PreferenceRepository,
+    private val json: Json,
+) {
+    suspend operator fun invoke(): Result<Path?, Engine.MkException> =
+        try {
+            when (val versionRes = getLatestEngineVersion()) {
+                is Failure -> Failure(versionRes.reason)
+                is Success -> {
+                    val (isLatest, _, latestVersion) = isGeoIpDbLatest(versionRes.value)
+                    if (isLatest) {
+                        Success(null)
+                    } else {
+                        val versionName = latestVersion
+                        val url = buildGeoIpDbUrl(versionName)
+                        val target = "$cacheDir/$versionName.mmdb"
+
+                        downloadFile(url, target).let {
+                            preferencesRepository.setValueByKey(SettingsKey.MMDB_VERSION, versionName)
+                            preferencesRepository.setValueByKey(SettingsKey.MMDB_LAST_CHECK, Clock.System.now().toEpochMilliseconds())
+                            Success(it)
+                        }
+                    }
+                }
+            }
+        } catch (t: Throwable) {
+            Failure(t as? Engine.MkException ?: Engine.MkException(t))
+        }
+
+    /**
+     * Compare latest and current version integers and return pair of latest state and actual version number
+     * @return Triple<Boolean, String, String> where the first element is true if the DB is the latest,
+     * the second is the current version and the third is the latest version.
+     */
+    private suspend fun isGeoIpDbLatest(latestVersion: String): Triple<Boolean, String, String> {
+        val currentGeoIpDbVersion: String =
+            (preferencesRepository.getValueByKey(SettingsKey.MMDB_VERSION).first() ?: getString(Res.string.engine_mmdb_version)) as String
+
+        return Triple(normalize(currentGeoIpDbVersion) >= normalize(latestVersion), currentGeoIpDbVersion, latestVersion)
+    }
+
+    private suspend fun getLatestEngineVersion(): Result<String, Engine.MkException> {
+        val url = "https://api.0.github.com/repos/aanorbel/oomplt-mmdb/releases/latest"
+
+        return engineHttpDo("GET", url, TaskOrigin.OoniRun).map { payload ->
+            val jsonStr = payload ?: throw Engine.MkException(Throwable("Empty body"))
+            json.decodeFromString(GhRelease.serializer(), jsonStr).tag
+        }
+    }
+
+    private fun buildGeoIpDbUrl(version: String): String =
+        "https://github.com/aanorbel/oomplt-mmdb/releases/download/$version/$version-ip2country_as.mmdb"
+
+    private fun normalize(tag: String): Int = tag.removePrefix("v").trim().toInt()
+
+    @Serializable
+    data class GhRelease(
+        @SerialName("tag_name") val tag: String,
+    )
+}
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/domain/GetEnginePreferences.kt
@@ -29,6 +29,7 @@ class GetEnginePreferences(
                 null
             },
             proxy = getProxyOption().first().value,
+            geoipDbVersion = getValueForKey(SettingsKey.MMDB_VERSION) as String?,
         )
 
     private suspend fun getEnabledCategories(): List<String> {
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/net/Http.kt
@@ -0,0 +1,7 @@
+package org.ooni.probe.net
+
+/**
+ * Perform a simple HTTP GET and return the raw response body bytes.
+ * Implemented per-platform to ensure binary-safe downloads.
+ */
+expect suspend fun httpGetBytes(url: String): ByteArray
diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt b/composeApp/src/desktopMain/kotlin/org/ooni/engine/DesktopOonimkallBridge.kt
@@ -64,6 +64,10 @@ class DesktopOonimkallBridge : OonimkallBridge {
             it.softwareVersion = softwareVersion
 
             it.assetsDir = assetsDir
+            // geoipDB may or may not exist in this binding; set via reflection when available
+            geoIpDB?.let { path ->
+                // it.geoipDB = path
+            }
             it.stateDir = stateDir
             it.tempDir = tempDir
             it.tunnelDir = tunnelDir
diff --git a/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt b/composeApp/src/desktopMain/kotlin/org/ooni/probe/net/Http.desktop.kt
@@ -0,0 +1,25 @@
+package org.ooni.probe.net
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import java.io.BufferedInputStream
+import java.net.HttpURLConnection
+import java.net.URL
+
+actual suspend fun httpGetBytes(url: String): ByteArray =
+    withContext(Dispatchers.IO) {
+        val connection = (URL(url).openConnection() as HttpURLConnection)
+        connection.requestMethod = "GET"
+        connection.instanceFollowRedirects = true
+        connection.connectTimeout = 15000
+        connection.readTimeout = 30000
+        try {
+            val code = connection.responseCode
+            val stream = if (code in 200..299) connection.inputStream else connection.errorStream
+            val bytes = stream?.let { BufferedInputStream(it).use { bis -> bis.readBytes() } } ?: ByteArray(0)
+            if (code !in 200..299) throw RuntimeException("HTTP $code while GET $url: ${bytes.decodeToString()}")
+            bytes
+        } finally {
+            connection.disconnect()
+        }
+    }
diff --git a/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt b/composeApp/src/iosMain/kotlin/org/ooni/probe/net/Http.ios.kt
diff --git a/iosApp/iosApp/engine/IosOonimkallBridge.swift b/iosApp/iosApp/engine/IosOonimkallBridge.swift

Original file line number	Diff line number	Diff line change
`@@ -7,5 +7,6 @@ data class EnginePreferences(`
`7`	`7`	`val taskLogLevel: TaskLogLevel,`
`8`	`8`	`val uploadResults: Boolean,`
`9`	`9`	`val proxy: String?,`
	`10`	`+ val geoipDbVersion: String?,`
`10`	`11`	`val maxRuntime: Duration?,`
`11`	`12`	`)`