Move worker VFS into own directory

Author: simolus3

sqlite3/build.yaml

@@ -32,6 +32,8 @@ targets:
     builders:
       build_web_compilers:entrypoint:
         generate_for:
+          include:
+            - "example/web/**"
           # This one is compiled in the other target
           exclude:
             - "example/web/worker.dart"

sqlite3/example/web/worker.dart

@@ -5,8 +5,8 @@ import 'package:js/js.dart';
 import 'package:js/js_util.dart';
 import 'package:sqlite3/src/constants.dart';
 import 'package:sqlite3/src/vfs.dart';
-import 'package:sqlite3/src/wasm/vfs/client.dart';
-import 'package:sqlite3/src/wasm/vfs/worker.dart';
+import 'package:sqlite3/src/wasm/vfs/async_opfs/client.dart';
+import 'package:sqlite3/src/wasm/vfs/async_opfs/worker.dart';
 
 @JS()
 external bool get crossOriginIsolated;
@@ -34,9 +34,10 @@ void main() {
       await worker.onMessage.first;
 
       final vfs = WasmVfs(workerOptions: options);
-      final (file: file, outFlags: flags) =
+      final opened =
           vfs.xOpen(Sqlite3Filename('/test'), SqlFlag.SQLITE_OPEN_CREATE);
-      print('opened file $file, outflags $flags');
+      final file = opened.file;
+      print('opened file $file, outflags ${opened.outFlags}');
 
       final buffer = Uint8List.fromList([1, 2, 3, 4, 5, 6]);
       file.xWrite(buffer, 0);

sqlite3/lib/src/ffi/bindings.dart

@@ -3,8 +3,6 @@ import 'dart:convert';
 import 'dart:ffi';
 import 'dart:typed_data';
 
-import 'package:sqlite3/src/vfs.dart';
-
 import '../constants.dart';
 import '../functions.dart';
 import '../implementation/bindings.dart';

sqlite3/lib/src/ffi/sqlite3.g.dart

@@ -1183,16 +1183,16 @@ class Bindings {
       .asFunction<int Function(ffi.Pointer<ffi.Void>)>();
 }
 
-final class sqlite3_char extends ffi.Opaque {}
+class sqlite3_char extends ffi.Opaque {}
 
-final class sqlite3 extends ffi.Opaque {}
+class sqlite3 extends ffi.Opaque {}
 
-final class sqlite3_stmt extends ffi.Opaque {}
+class sqlite3_stmt extends ffi.Opaque {}
 
-final class sqlite3_backup extends ffi.Opaque {}
+class sqlite3_backup extends ffi.Opaque {}
 
-final class sqlite3_api_routines extends ffi.Opaque {}
+class sqlite3_api_routines extends ffi.Opaque {}
 
-final class sqlite3_value extends ffi.Opaque {}
+class sqlite3_value extends ffi.Opaque {}
 
-final class sqlite3_context extends ffi.Opaque {}
+class sqlite3_context extends ffi.Opaque {}

sqlite3/lib/src/vfs.dart

@@ -78,7 +78,13 @@ abstract class VirtualFileSystem {
 }
 
 /// The result of [VirtualFileSystem.xOpen].
-typedef XOpenResult = ({int outFlags, VirtualFileSystemFile file});
+class XOpenResult {
+  final int outFlags;
+  final VirtualFileSystemFile file;
+
+  XOpenResult({required this.outFlags, required this.file});
+  // todo: Turn into record
+}
 
 /// A file implemented by a VFS author and returned by [VirtualFileSystem.xOpen].
 ///

sqlite3/lib/src/wasm/file_system/indexed_db.dart

@@ -626,7 +626,7 @@ class IndexedDbFileSystem implements FileSystem {
   }
 }
 
-sealed class _IndexedDbWorkItem with LinkedListEntry<_IndexedDbWorkItem> {
+abstract class _IndexedDbWorkItem with LinkedListEntry<_IndexedDbWorkItem> {
   final Completer<void> completer = Completer.sync();
 
   /// Insert this item into the [pending] list, returning whether the item was
@@ -647,7 +647,7 @@ sealed class _IndexedDbWorkItem with LinkedListEntry<_IndexedDbWorkItem> {
   FutureOr<void> run();
 }
 
-final class _FunctionWorkItem extends _IndexedDbWorkItem {
+class _FunctionWorkItem extends _IndexedDbWorkItem {
   final FutureOr<void> Function() work;
   final String description;
 
@@ -657,7 +657,7 @@ final class _FunctionWorkItem extends _IndexedDbWorkItem {
   FutureOr<void> run() => work();
 }
 
-final class _DeleteFileWorkItem extends _IndexedDbWorkItem {
+class _DeleteFileWorkItem extends _IndexedDbWorkItem {
   final IndexedDbFileSystem fileSystem;
   final String path;
 
@@ -720,7 +720,7 @@ final class _DeleteFileWorkItem extends _IndexedDbWorkItem {
   }
 }
 
-final class _CreateFileWorkItem extends _IndexedDbWorkItem {
+class _CreateFileWorkItem extends _IndexedDbWorkItem {
   final IndexedDbFileSystem fileSystem;
   final String path;
 
@@ -733,7 +733,7 @@ final class _CreateFileWorkItem extends _IndexedDbWorkItem {
   }
 }
 
-final class _WriteFileWorkItem extends _IndexedDbWorkItem {
+class _WriteFileWorkItem extends _IndexedDbWorkItem {
   final IndexedDbFileSystem fileSystem;
   final String path;
 

sqlite3/lib/src/wasm/vfs/client.dart renamed to sqlite3/lib/src/wasm/vfs/async_opfs/client.dart

@@ -5,9 +5,9 @@ import 'dart:typed_data';
 import 'package:js/js_util.dart';
 import 'package:path/path.dart' as p;
 
-import '../../constants.dart';
-import '../../vfs.dart';
-import '../js_interop.dart';
+import '../../../constants.dart';
+import '../../../vfs.dart';
+import '../../js_interop.dart';
 import 'sync_channel.dart';
 import 'worker.dart';
 
@@ -71,7 +71,7 @@ class WasmVfs extends BaseVirtualFileSystem {
 
     final outFlags = result.flag0;
     final fd = result.flag1;
-    return (outFlags: outFlags, file: WasmFile(this, fd));
+    return XOpenResult(outFlags: outFlags, file: WasmFile(this, fd));
   }
 
   @override

sqlite3/lib/src/wasm/vfs/sync_channel.dart renamed to sqlite3/lib/src/wasm/vfs/async_opfs/sync_channel.dart

@@ -2,20 +2,26 @@ import 'dart:convert';
 import 'dart:html';
 import 'dart:typed_data';
 
-import '../js_interop.dart';
+import '../../js_interop.dart';
 
 const protocolVersion = 1;
 const asyncIdleWaitTimeMs = 150;
 const asyncIdleWaitTime = Duration(milliseconds: asyncIdleWaitTimeMs);
 
+/// Implements a synchronous mechanism to wait for requests and responses.
 class RequestResponseSynchronizer {
   static const _requestIndex = 0;
   static const _responseIndex = 1;
 
   // 2 32-bit slots for the int32 array
   static const byteLength = 2 * 4;
 
+  /// The shared array buffer used with atomics for synchronization.
+  ///
+  /// It must have a length of [byteLength].
   final SharedArrayBuffer buffer;
+
+  /// A int32 view over [buffer], required for atomics to work.
   final Int32List int32View;
 
   RequestResponseSynchronizer._(this.buffer) : int32View = buffer.asInt32List();
@@ -29,6 +35,8 @@ class RequestResponseSynchronizer {
         buffer ?? SharedArrayBuffer(byteLength));
   }
 
+  /// Send a request with the given [opcode], wait for the remote worker to
+  /// process it and returns the response code.
   int requestAndWaitForResponse(int opcode) {
     Atomics.store(int32View, _responseIndex, -1);
     Atomics.store(int32View, _requestIndex, opcode);

sqlite3/lib/src/wasm/vfs/worker.dart renamed to sqlite3/lib/src/wasm/vfs/async_opfs/worker.dart

@@ -3,9 +3,9 @@ import 'dart:html';
 import 'package:js/js.dart';
 import 'package:path/path.dart' as p show url;
 
-import '../../constants.dart';
-import '../../vfs.dart';
-import '../js_interop.dart';
+import '../../../constants.dart';
+import '../../../vfs.dart';
+import '../../js_interop.dart';
 import 'sync_channel.dart';
 
 const _workerDebugLog =
@@ -45,6 +45,19 @@ class WorkerOptions {
   }
 }
 
+class _ResolvedPath {
+  final String fullPath;
+
+  final FileSystemDirectoryHandle directory;
+  final String filename;
+
+  _ResolvedPath(this.fullPath, this.directory, this.filename);
+
+  Future<FileSystemFileHandle> openFile({bool create = false}) {
+    return directory.openFile(filename, create: create);
+  }
+}
+
 class VfsWorker {
   final RequestResponseSynchronizer synchronizer;
   final MessageSerializer messages;
@@ -55,8 +68,6 @@ class VfsWorker {
   final Map<int, _OpenedFileHandle> _openFiles = {};
   final Set<_OpenedFileHandle> _implicitlyHeldLocks = {};
 
-  var _shutdownRequested = false;
-
   VfsWorker._(WorkerOptions options, this.root)
       : synchronizer =
             RequestResponseSynchronizer(options.synchronizationBuffer),
@@ -73,30 +84,30 @@ class VfsWorker {
     return VfsWorker._(options, root);
   }
 
-  Future<(FileSystemDirectoryHandle, String, String)> _resolvePath(
-      String absolutePath,
+  Future<_ResolvedPath> _resolvePath(String absolutePath,
       {bool createDirectories = false}) async {
     final fullPath = p.url.relative(absolutePath, from: '/');
-    final [...dir, file] = p.url.split(fullPath);
+    final directories = p.url.split(fullPath);
+    final file = directories.removeLast();
 
     var dirHandle = root;
-    for (final entry in dir) {
+    for (final entry in directories) {
       dirHandle =
           await dirHandle.getDirectory(entry, create: createDirectories);
     }
 
-    return (dirHandle, fullPath, file);
+    return _ResolvedPath(fullPath, dirHandle, file);
   }
 
   Future<Flags> _xAccess(NameAndInt32Flags flags) async {
     var rc = 0;
 
     try {
-      final (dir, _, fileName) = await _resolvePath(flags.name);
+      final resolved = await _resolvePath(flags.name);
 
       // If we can open the file, it exists. For OPFS, that means that it's both
       // readable and writable.
-      await dir.openFile(fileName);
+      await resolved.openFile();
     } catch (e) {
       rc = SqlError.SQLITE_IOERR;
     }
@@ -105,9 +116,9 @@ class VfsWorker {
   }
 
   Future<void> _xDelete(NameAndInt32Flags options) async {
-    final (dirHandle, _, basename) = await _resolvePath(options.name);
+    final resolved = await _resolvePath(options.name);
     try {
-      await dirHandle.removeEntry(basename);
+      await resolved.directory.removeEntry(resolved.filename);
     } catch (e) {
       _log('Could not delete entry: $e');
       throw const VfsException(SqlExtendedError.SQLITE_IOERR_DELETE);
@@ -118,24 +129,22 @@ class VfsWorker {
     final flags = req.flag0;
     final create = (flags & SqlFlag.SQLITE_OPEN_CREATE) != 0;
 
-    FileSystemDirectoryHandle directory;
-    String fullPath, fileName;
+    _ResolvedPath resolved;
 
     try {
-      (directory, fullPath, fileName) =
-          await _resolvePath(req.name, createDirectories: create);
+      resolved = await _resolvePath(req.name, createDirectories: create);
     } catch (e) {
       // Error traversing the path
       throw VfsException(SqlError.SQLITE_NOTFOUND);
     }
 
-    final fileHandle = await directory.openFile(fileName, create: create);
+    final fileHandle = await resolved.openFile(create: create);
     final readonly = !create && (flags & SqlFlag.SQLITE_OPEN_READONLY) != 0;
     final opened = _OpenedFileHandle(
       fd: _fdCounter++,
-      directory: directory,
-      fullPath: fullPath,
-      filename: fileName,
+      directory: resolved.directory,
+      fullPath: resolved.fullPath,
+      filename: resolved.filename,
       file: fileHandle,
       deleteOnClose: (flags & SqlFlag.SQLITE_OPEN_DELETEONCLOSE) != 0,
       readonly: readonly,
@@ -268,7 +277,7 @@ class VfsWorker {
   }
 
   Future<void> start() async {
-    while (!_shutdownRequested) {
+    while (true) {
       final waitResult = synchronizer.waitForRequest();
       if (waitResult == Atomics.timedOut) {
         // No requests for some time, transition to idle

sqlite3/pubspec.yaml

@@ -5,7 +5,7 @@ homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3
 issue_tracker: https://github.com/simolus3/sqlite3.dart/issues
 
 environment:
-  sdk: '>=2.3.0 <4.0.0'
+  sdk: '>=3.0.0 <4.0.0'
 
 # This package supports all platforms listed below.
 platforms:
@@ -32,7 +32,7 @@ dev_dependencies:
   build_daemon: ^4.0.0
   build_runner: ^2.1.7
   build_web_compilers: ^4.0.3
-  ffigen: ^8.0.0-dev
+  ffigen: ^7.0.0
   http: ^0.13.4
   lints: ^2.0.1
   shelf: ^1.4.0