Test with minimum supported sqlite

Author: simolus3

.github/workflows/tests.yml

@@ -2,6 +2,7 @@ on:
   push:
   pull_request:
 name: "tests"
+
 jobs:
   build:
     name: Testing on ${{ matrix.os }}
@@ -10,11 +11,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-          - os: macos-latest
+        os: [ubuntu-24.04, macos-latest]
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - uses: dart-lang/setup-dart@v1
@@ -55,11 +55,27 @@ jobs:
         run: |
           ./target/release/sqlite3 ":memory:" ".load ./target/release/libpowersync" "select powersync_rs_version()"
 
-      - name: Run dart-based tests
-        # Extension loading fails on macos currently
-        if: matrix.os == 'ubuntu-24.04'
+      - uses: actions/cache@v4
+        id: sqlite_build
+        with:
+          path: dart/.dart_tool/sqlite3/
+          key: ${{ runner.os }}-${{ hashFiles('dart/tool/') }}
+
+      - name: Setup Dart tests
+        working-directory: dart
         run: |
-          cd dart
           dart pub get
-          dart test
+          dart run tool/download_sqlite3.dart
           dart analyze
+      
+      - name: Dart tests on Linux
+        if: runner.os == 'ubuntu'
+        run: |
+          CORE_TEST_SQLITE=.dart_tool/sqlite3/latest/libsqlite3.so dart test
+          CORE_TEST_SQLITE=.dart_tool/sqlite3/minimum/libsqlite3.so dart test
+
+      - name: Dart tests on macOS
+        if: runner.os == 'macOS'
+        run: |
+          CORE_TEST_SQLITE=.dart_tool/sqlite3/latest/libsqlite3.dylib dart test
+          CORE_TEST_SQLITE=.dart_tool/sqlite3/minimum/libsqlite3.dylib dart test

crates/core/src/constants.rs

@@ -5,8 +5,8 @@ pub const FULL_GIT_HASH: &'static str = env!("GIT_HASH");
 
 // We need 3.44 or later to use an `ORDER BY` in an aggregate function invocation.
 //
-// When raising the minimum version requirement, also change the CI to ensure we're testing on the
-// oldest SQLite version we claim to support.
+// When raising the minimum version requirement, also change it in download_sqlite3.dart to ensure
+// we're testing with the minimum version we claim to support.
 pub const MIN_SQLITE_VERSION_NUMBER: c_int = 3044000;
 
 pub fn short_git_hash() -> &'static str {

dart/pubspec.lock

@@ -13,10 +13,10 @@ packages:
     dependency: transitive
     description:
       name: analyzer
-      sha256: f6154230675c44a191f2e20d16eeceb4aa18b30ca732db4efaf94c6a7d43cfa6
+      sha256: de617bfdc64f3d8b00835ec2957441ceca0a29cdf7881f7ab231bc14f71159c0
       url: "https://pub.dev"
     source: hosted
-    version: "7.5.2"
+    version: "7.5.6"
   args:
     dependency: transitive
     description:
@@ -85,10 +85,10 @@ packages:
     dependency: transitive
     description:
       name: coverage
-      sha256: aa07dbe5f2294c827b7edb9a87bba44a9c15a3cc81bc8da2ca19b37322d30080
+      sha256: "5da775aa218eaf2151c721b16c01c7676fbfdd99cebba2bf64e8b807a28ff94d"
       url: "https://pub.dev"
     source: hosted
-    version: "1.14.1"
+    version: "1.15.0"
   crypto:
     dependency: transitive
     description:
@@ -250,7 +250,7 @@ packages:
     source: hosted
     version: "0.1.1"
   path:
-    dependency: transitive
+    dependency: "direct dev"
     description:
       name: path
       sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"

dart/pubspec.yaml

@@ -16,3 +16,4 @@ dev_dependencies:
   fake_async: ^1.3.3
   convert: ^3.1.2
   meta: ^1.16.0
+  path: ^1.9.1

dart/test/utils/native_test_utils.dart

@@ -12,6 +12,11 @@ const libPath = '../target/debug';
 var didLoadExtension = false;
 
 void applyOpenOverride() {
+  if (Platform.environment['CORE_TEST_SQLITE'] case final sqlite?) {
+    sqlite_open.open
+        .overrideForAll(() => DynamicLibrary.open(p.absolute(sqlite)));
+  }
+
   sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () {
     return DynamicLibrary.open('libsqlite3.so.0');
   });

dart/tool/download_sqlite3.dart

@@ -0,0 +1,93 @@
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+
+typedef SqliteVersion = ({String version, String year});
+
+const SqliteVersion latest = (version: '3500200', year: '2025');
+const SqliteVersion minimum = (version: '3440000', year: '2023');
+
+Future<void> main(List<String> args) async {
+  if (args.contains('version')) {
+    print(latest.version);
+    exit(0);
+  }
+
+  await _downloadAndCompile('latest', latest, force: args.contains('--force'));
+  await _downloadAndCompile('minimum', minimum,
+      force: args.contains('--force'));
+}
+
+extension on SqliteVersion {
+  String get autoconfUrl =>
+      'https://sqlite.org/$year/sqlite-autoconf-$version.tar.gz';
+}
+
+Future<void> _downloadAndCompile(String name, SqliteVersion version,
+    {bool force = false}) async {
+  final dartDirectory = p.dirname(p.dirname(Platform.script.toFilePath()));
+  final target = p.join(dartDirectory, '.dart_tool', 'sqlite3', name);
+  final versionFile = File(p.join(target, 'version'));
+
+  final needsDownload = force ||
+      !versionFile.existsSync() ||
+      versionFile.readAsStringSync() != version.version;
+
+  if (!needsDownload) {
+    print(
+      'Not downloading sqlite3 $name as it has already been downloaded. Use '
+      '--force to re-compile it.',
+    );
+    return;
+  }
+
+  print('Downloading and compiling sqlite3 $name (${version.version})');
+  final targetDirectory = Directory(target);
+
+  if (!targetDirectory.existsSync()) {
+    targetDirectory.createSync(recursive: true);
+  }
+
+  final temporaryDir =
+      await Directory.systemTemp.createTemp('powersync-core-compile-sqlite3');
+  final temporaryDirPath = temporaryDir.path;
+
+  await _run('curl -L ${version.autoconfUrl} --output sqlite.tar.gz',
+      workingDirectory: temporaryDirPath);
+  await _run('tar zxvf sqlite.tar.gz', workingDirectory: temporaryDirPath);
+
+  final sqlitePath =
+      p.join(temporaryDirPath, 'sqlite-autoconf-${version.version}');
+
+  await _run('./configure', workingDirectory: sqlitePath);
+  await _run('make -j', workingDirectory: sqlitePath);
+
+  await File(p.join(sqlitePath, 'sqlite3')).copy(p.join(target, 'sqlite3'));
+  final libsPath = name == 'latest' ? sqlitePath : p.join(sqlitePath, '.libs');
+
+  if (Platform.isLinux) {
+    await File(p.join(libsPath, 'libsqlite3.so'))
+        .copy(p.join(target, 'libsqlite3.so'));
+  } else if (Platform.isMacOS) {
+    await File(p.join(libsPath, 'libsqlite3.dylib'))
+        .copy(p.join(target, 'libsqlite3.dylib'));
+  }
+
+  await File(p.join(target, 'version')).writeAsString(version.version);
+}
+
+Future<void> _run(String command, {String? workingDirectory}) async {
+  print('Running $command');
+
+  final proc = await Process.start(
+    'sh',
+    ['-c', command],
+    mode: ProcessStartMode.inheritStdio,
+    workingDirectory: workingDirectory,
+  );
+  final exitCode = await proc.exitCode;
+
+  if (exitCode != 0) {
+    exit(exitCode);
+  }
+}