diff --git a/.github/workflows/release-sqlite-db.yml b/.github/workflows/release-sqlite-db.yml index b507790..15d155d 100644 --- a/.github/workflows/release-sqlite-db.yml +++ b/.github/workflows/release-sqlite-db.yml @@ -19,6 +19,9 @@ jobs: with: fetch-depth: 0 + - name: Define release name (once for all steps) + run: echo "RELEASE_NAME=$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV + - name: Set up Java 17 uses: actions/setup-java@v3 with: @@ -44,10 +47,6 @@ jobs: ls -l generators/store/build/store-database-fr.db ls -l generators/store/build/store-database-he.db - - name: Define release name - id: relname - run: echo "RELEASE_NAME=$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV - - name: Create and push tag run: | git config --global user.name "github-actions" @@ -58,7 +57,6 @@ jobs: - name: Create GitHub release and upload artifacts uses: softprops/action-gh-release@v2.1.0 - with: tag_name: ${{ env.RELEASE_NAME }} name: ${{ env.RELEASE_NAME }} diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/database/dao/ApplicationsDao.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/database/dao/ApplicationsDao.kt index 4b8af7d..dd1bb06 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/database/dao/ApplicationsDao.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/database/dao/ApplicationsDao.kt @@ -13,7 +13,7 @@ import io.github.kdroidfilter.database.store.Developers * Contains functions for database operations related to applications */ object ApplicationsDao { - + /** * Creates a GooglePlayApplicationInfo from database data */ @@ -129,4 +129,5 @@ object ApplicationsDao { createAppInfoWithExtras(app, developer, category, deviceLanguage, creator) } } -} \ No newline at end of file + +} diff --git a/generators/store/src/jvmMain/kotlin/SqliteStoreBuilder.kt b/generators/store/src/jvmMain/kotlin/SqliteStoreBuilder.kt index f75f6cc..79f1494 100644 --- a/generators/store/src/jvmMain/kotlin/SqliteStoreBuilder.kt +++ b/generators/store/src/jvmMain/kotlin/SqliteStoreBuilder.kt @@ -3,7 +3,11 @@ import co.touchlab.kermit.Logger import com.kdroid.gplayscrapper.core.model.GooglePlayApplicationInfo import com.kdroid.gplayscrapper.services.getGooglePlayApplicationInfo import io.github.kdroidfilter.database.core.AppCategory -import io.github.kdroidfilter.database.store.* +import io.github.kdroidfilter.database.store.App_categoriesQueries +import io.github.kdroidfilter.database.store.ApplicationsQueries +import io.github.kdroidfilter.database.store.Database +import io.github.kdroidfilter.database.store.DevelopersQueries +import io.github.kdroidfilter.database.store.VersionQueries import io.github.kdroidfilter.platformtools.releasefetcher.github.GitHubReleaseFetcher import kotlinx.coroutines.runBlocking import java.net.URL @@ -16,80 +20,7 @@ import java.time.format.DateTimeFormatter object SqliteStoreBuilder { private val logger = Logger.withTag("SqliteStoreBuilder") - /** - * Builds three databases in different languages: English, French, and Hebrew. - * @param appPoliciesDir The directory containing app policies - * @param baseDbPath The base path used to determine the output directory. - * Only the parent directory of this path is used to create the actual database files. - */ - fun buildMultiLanguageDatabases(appPoliciesDir: Path, baseDbPath: Path) { - // Get the parent directory where databases will be stored - val outputDir = baseDbPath.parent - - // Create English database - val englishDbPath = outputDir.resolve("store-database-en.db") - buildDatabase(appPoliciesDir, englishDbPath, "en", "us") - - // Create French database - val frenchDbPath = outputDir.resolve("store-database-fr.db") - buildDatabase(appPoliciesDir, frenchDbPath, "fr", "fr") - - // Create Hebrew database - val hebrewDbPath = outputDir.resolve("store-database-he.db") - buildDatabase(appPoliciesDir, hebrewDbPath, "he", "il") - - logger.i { "✅ Created databases in English, French, and Hebrew" } - } - - /** - * Attempts to download the latest database from GitHub releases. - * @return true if download was successful, false otherwise - */ - private fun downloadLatestDatabase(outputDbPath: Path): Boolean = runBlocking { - try { - logger.i { "🔄 Attempting to download the latest database..." } - val fetcher = GitHubReleaseFetcher(owner = "kdroidFilter", repo = "KDroidDatabase") - val latestRelease = fetcher.getLatestRelease() - - if (latestRelease != null && latestRelease.assets.size > 1) { - // Find the store-database.db asset - val downloadUrl = latestRelease.assets[1].browser_download_url - - logger.i { "📥 Downloading database from: $downloadUrl" } - - Files.createDirectories(outputDbPath.parent) - - // Download the file - URL(downloadUrl).openStream().use { input -> - Files.copy(input, outputDbPath, StandardCopyOption.REPLACE_EXISTING) - } - - // Verify the file was downloaded successfully - if (Files.exists(outputDbPath) && Files.size(outputDbPath) > 0) { - logger.i { "✅ Database downloaded successfully to $outputDbPath" } - return@runBlocking true - } else { - logger.w { "⚠️ Downloaded file is empty or does not exist" } - return@runBlocking false - } - } else { - logger.w { "⚠️ No database assets found in the latest release" } - return@runBlocking false - } - } catch (e: Exception) { - logger.e(e) { "❌ Failed to download the latest database: ${e.message}" } - return@runBlocking false - } - } - fun buildDatabase(appPoliciesDir: Path, outputDbPath: Path, language: String = "en", country: String = "us") { - // First try to download the latest database -// if (downloadLatestDatabase(outputDbPath)) { -// logger.i { "✅ Using downloaded database at $outputDbPath" } -// return // Use the downloaded database -// } - //Disable for testing - // Get release name from environment variable or generate timestamp val releaseName = System.getenv("RELEASE_NAME") ?: LocalDateTime.now() .format(DateTimeFormatter.ofPattern("yyyyMMddHHmm")) @@ -163,16 +94,7 @@ object SqliteStoreBuilder { // Get or create the category val appCategoriesQueries = App_categoriesQueries(createSqlDriver(outputDbPath)) - val category = appCategoriesQueries - .getCategoryByName(categoryName) - .executeAsOneOrNull() ?: run { - // Insert the category if it doesn't exist - appCategoriesQueries.insertCategory( - category_name = categoryName, - description = null - ) - appCategoriesQueries.getCategoryByName(categoryName).executeAsOne() - } + val categoryId = getOrCreateCategory(categoryName, appCategoriesQueries) // Fetch application info from Google Play with specified language and country val appInfo = runBlocking { @@ -184,83 +106,292 @@ object SqliteStoreBuilder { } if (appInfo != null) { - // Create DevelopersQueries instance + // Create queries instances val developersQueries = DevelopersQueries(createSqlDriver(outputDbPath)) + val applicationsQueries = ApplicationsQueries(createSqlDriver(outputDbPath)) + + // Insert application + insertApplicationFromAppInfo( + appInfo = appInfo, + packageName = packageName, + applicationsQueries = applicationsQueries, + developersQueries = developersQueries, + categoryId = categoryId + ) + + count++ + } else { + logger.w { "⚠️ Failed to fetch info for $packageName" } + } + } - // Get or create the developer - val developer = developersQueries - .getDeveloperByDeveloperId(appInfo.developerId) - .executeAsOneOrNull() ?: run { - // Insert the developer if it doesn't exist - developersQueries.insertDeveloper( - developer_id = appInfo.developerId, - name = appInfo.developer, - email = null, - website = appInfo.developerWebsite, - address = null - ) - developersQueries.getDeveloperByDeveloperId(appInfo.developerId).executeAsOne() + logger.i { "✅ Processed $count applications using SQLDelight tables" } + } + + /** + * Downloads the latest databases for all three languages (en, fr, he) from GitHub releases. + * @param baseDbPath The base path used to determine the output directory + * @return Map of language codes to download success status + */ + private fun downloadLatestDatabases(baseDbPath: Path): Map = runBlocking { + val outputDir = baseDbPath.parent + val languages = listOf("en", "fr", "he") + val results = mutableMapOf() + + try { + logger.i { "🔄 Attempting to download the latest databases for all languages..." } + val fetcher = GitHubReleaseFetcher(owner = "kdroidFilter", repo = "KDroidDatabase") + val latestRelease = fetcher.getLatestRelease() + + if (latestRelease != null && latestRelease.assets.isNotEmpty()) { + languages.forEach { lang -> + try { + // Look for language-specific database file + val assetName = "store-database-$lang.db" + val asset = latestRelease.assets.find { it.name == assetName } + + if (asset != null) { + val outputDbPath = outputDir.resolve(assetName) + val downloadUrl = asset.browser_download_url + + logger.i { "📥 Downloading $lang database from: $downloadUrl" } + + Files.createDirectories(outputDbPath.parent) + + // Download the file + URL(downloadUrl).openStream().use { input -> + Files.copy(input, outputDbPath, StandardCopyOption.REPLACE_EXISTING) + } + + // Verify the file was downloaded successfully + if (Files.exists(outputDbPath) && Files.size(outputDbPath) > 0) { + logger.i { "✅ Database $lang downloaded successfully to $outputDbPath" } + results[lang] = true + } else { + logger.w { "⚠️ Downloaded file for $lang is empty or does not exist" } + results[lang] = false + } + } else { + logger.w { "⚠️ No database asset found for language: $lang" } + results[lang] = false } + } catch (e: Exception) { + logger.e(e) { "❌ Failed to download database for $lang: ${e.message}" } + results[lang] = false + } + } + } else { + logger.w { "⚠️ No database assets found in the latest release" } + languages.forEach { results[it] = false } + } + } catch (e: Exception) { + logger.e(e) { "❌ Failed to download databases: ${e.message}" } + languages.forEach { results[it] = false } + } - // Create ApplicationsQueries instance - val applicationsQueries = ApplicationsQueries(createSqlDriver(outputDbPath)) + return@runBlocking results + } + + /** + * Updates packages for existing databases, only adding packages that don't exist in the table + */ + private fun updatePackagesIfNotExists(dir: Path, outputDbPath: Path, language: String = "en", country: String = "us") { + val existingApps = mutableMapOf() + var processedCount = 0 + var addedCount = 0 + + Files.walk(dir) + .filter { Files.isRegularFile(it) && it.toString().endsWith(".json") } + .forEach { file -> + val packageName = file.fileName.toString().substringBeforeLast(".") + processedCount++ + + // Check if the package already exists in the database + val applicationsQueries = ApplicationsQueries(createSqlDriver(outputDbPath)) + val existingApp = applicationsQueries + .getApplicationByAppId(packageName) + .executeAsOneOrNull() + + // Only process if the package doesn't exist + if (existingApp == null) { + val categoryName = file.parent.fileName.toString() + .uppercase().replace('-', '_') + + // Get or create the category + val appCategoriesQueries = App_categoriesQueries(createSqlDriver(outputDbPath)) + val categoryId = getOrCreateCategory(categoryName, appCategoriesQueries) + + // Fetch application info from Google Play with specified language and country + val appInfo = runBlocking { + existingApps.getOrPut(packageName) { + runCatching { + getGooglePlayApplicationInfo(packageName, language, country) + }.getOrNull() + } + } - // Check if the application already exists - val existingApp = applicationsQueries - .getApplicationByAppId(packageName) - .executeAsOneOrNull() - - if (existingApp == null) { - // Insert the application - applicationsQueries.insertApplication( - app_id = packageName, - title = appInfo.title, - description = appInfo.description, - description_html = appInfo.descriptionHTML, - summary = appInfo.summary, - installs = appInfo.installs, - min_installs = appInfo.minInstalls, - real_installs = appInfo.realInstalls, - score = appInfo.score, - ratings = appInfo.ratings, - reviews = appInfo.reviews, - histogram = appInfo.histogram.toString(), - price = appInfo.price, - free = if (appInfo.free) 1 else 0, - currency = appInfo.currency, - sale = if (appInfo.sale) 1 else 0, - sale_time = null, - original_price = appInfo.originalPrice, - sale_text = appInfo.saleText, - offers_iap = if (appInfo.offersIAP) 1 else 0, - in_app_product_price = appInfo.inAppProductPrice, - developer_id = developer.id, - privacy_policy = appInfo.privacyPolicy, - genre = appInfo.genre, - genre_id = appInfo.genreId, - icon = appInfo.icon, - header_image = appInfo.headerImage, - screenshots = appInfo.screenshots.joinToString(","), - video = appInfo.video, - video_image = appInfo.videoImage, - content_rating = appInfo.contentRating, - content_rating_description = appInfo.contentRatingDescription, - ad_supported = if (appInfo.adSupported) 1 else 0, - contains_ads = if (appInfo.containsAds) 1 else 0, - released = appInfo.released, - updated = appInfo.updated, - version = appInfo.version, - comments = null, - url = appInfo.url, - app_category_id = category.id + if (appInfo != null) { + // Create queries instances + val developersQueries = DevelopersQueries(createSqlDriver(outputDbPath)) + + // Insert application + insertApplicationFromAppInfo( + appInfo = appInfo, + packageName = packageName, + applicationsQueries = applicationsQueries, + developersQueries = developersQueries, + categoryId = categoryId ) + + addedCount++ + logger.d { "➕ Added new package: $packageName" } + } else { + logger.w { "⚠️ Failed to fetch info for new package $packageName" } } - count++ } else { - logger.w { "⚠️ Failed to fetch info for $packageName" } + logger.d { "⏭️ Package $packageName already exists, skipping" } } } - logger.i { "✅ Processed $count applications using SQLDelight tables" } + logger.i { "✅ Processed $processedCount packages, added $addedCount new packages for language $language" } + } + + /** + * Gets or creates a category in the database + * Returns the category ID + */ + private fun getOrCreateCategory( + categoryName: String, + appCategoriesQueries: App_categoriesQueries + ): Long { + val category = appCategoriesQueries + .getCategoryByName(categoryName) + .executeAsOneOrNull() ?: run { + // Insert the category if it doesn't exist + appCategoriesQueries.insertCategory( + category_name = categoryName, + description = null + ) + appCategoriesQueries.getCategoryByName(categoryName).executeAsOne() + } + + return category.id + } + + /** + * Inserts or updates an application in the database from GooglePlayApplicationInfo + * Returns the inserted/updated application ID + */ + private fun insertApplicationFromAppInfo( + appInfo: GooglePlayApplicationInfo, + packageName: String, + applicationsQueries: ApplicationsQueries, + developersQueries: DevelopersQueries, + categoryId: Long + ): Long { + // Get or create the developer + val developer = developersQueries + .getDeveloperByDeveloperId(appInfo.developerId) + .executeAsOneOrNull() ?: run { + // Insert the developer if it doesn't exist + developersQueries.insertDeveloper( + developer_id = appInfo.developerId, + name = appInfo.developer, + email = null, + website = appInfo.developerWebsite, + address = null + ) + developersQueries.getDeveloperByDeveloperId(appInfo.developerId).executeAsOne() + } + + // Check if the application already exists + val existingApp = applicationsQueries + .getApplicationByAppId(packageName) + .executeAsOneOrNull() + + if (existingApp == null) { + // Insert the application + applicationsQueries.insertApplication( + app_id = packageName, + title = appInfo.title, + description = appInfo.description, + description_html = appInfo.descriptionHTML, + summary = appInfo.summary, + installs = appInfo.installs, + min_installs = appInfo.minInstalls, + real_installs = appInfo.realInstalls, + score = appInfo.score, + ratings = appInfo.ratings, + reviews = appInfo.reviews, + histogram = appInfo.histogram.toString(), + price = appInfo.price, + free = if (appInfo.free) 1 else 0, + currency = appInfo.currency, + sale = if (appInfo.sale) 1 else 0, + sale_time = null, + original_price = appInfo.originalPrice, + sale_text = appInfo.saleText, + offers_iap = if (appInfo.offersIAP) 1 else 0, + in_app_product_price = appInfo.inAppProductPrice, + developer_id = developer.id, + privacy_policy = appInfo.privacyPolicy, + genre = appInfo.genre, + genre_id = appInfo.genreId, + icon = appInfo.icon, + header_image = appInfo.headerImage, + screenshots = appInfo.screenshots.joinToString(","), + video = appInfo.video, + video_image = appInfo.videoImage, + content_rating = appInfo.contentRating, + content_rating_description = appInfo.contentRatingDescription, + ad_supported = if (appInfo.adSupported) 1 else 0, + contains_ads = if (appInfo.containsAds) 1 else 0, + released = appInfo.released, + updated = appInfo.updated, + version = appInfo.version, + comments = null, + url = appInfo.url, + app_category_id = categoryId + ) + + // Return the ID of the newly inserted application + return applicationsQueries.getApplicationByAppId(packageName).executeAsOne().id + } else { + // Return the ID of the existing application + return existingApp.id + } + } + + /** + * Builds or updates databases for all three languages, downloading existing ones first + */ + fun buildOrUpdateMultiLanguageDatabases(appPoliciesDir: Path, baseDbPath: Path) { + val outputDir = baseDbPath.parent + val languages = mapOf( + "en" to "us", + "fr" to "fr", + "he" to "il" + ) + + logger.i { "🔄 Starting multi-language database build/update process..." } + + // First, try to download existing databases + val downloadResults = downloadLatestDatabases(baseDbPath) + + languages.forEach { (language, country) -> + val dbPath = outputDir.resolve("store-database-$language.db") + + if (downloadResults[language] == true) { + logger.i { "📄 Using downloaded database for $language, updating with new packages only..." } + // Update the downloaded database with only new packages + updatePackagesIfNotExists(appPoliciesDir, dbPath, language, country) + } else { + logger.i { "🏗️ Building new database for $language from scratch..." } + // Build database from scratch + buildDatabase(appPoliciesDir, dbPath, language, country) + } + } + + logger.i { "✅ Completed multi-language database build/update process" } } } diff --git a/generators/store/src/jvmMain/kotlin/SqliteStoreExtractor.kt b/generators/store/src/jvmMain/kotlin/SqliteStoreExtractor.kt index f4795ec..0c37d46 100644 --- a/generators/store/src/jvmMain/kotlin/SqliteStoreExtractor.kt +++ b/generators/store/src/jvmMain/kotlin/SqliteStoreExtractor.kt @@ -12,5 +12,5 @@ fun main() { val baseDbPath = projectDir.resolve("build/store-database.db") // Build databases in multiple languages (English, French, Hebrew) - SqliteStoreBuilder.buildMultiLanguageDatabases(policiesDir, baseDbPath) + SqliteStoreBuilder.buildOrUpdateMultiLanguageDatabases(policiesDir, baseDbPath) }