Merge libsql#156

156: sqld: extract ffi bindings to a separate crate r=MarinPostma a=psarna

That's done to avoid cyclic Cargo dependencies - bottomless depends on the FFI bindings, but sqld is going to depdend on bottomless soon.

Co-authored-by: Piotr Sarna <[email protected]>

Author: bors[bot]

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "libsql-client",
     "sqlc",
     "sqld",
+    "sqld-libsql-bindings",
 ]
 
 [patch.crates-io]

bottomless/Cargo.toml

@@ -16,7 +16,7 @@ aws-sdk-s3 = { version = "0.22.0" }
 bytes = "1"
 crc = "3.0.0"
 futures = { version = "0.3.25" }
-sqld = { path = "../sqld" }
+sqld-libsql-bindings = { version = "0", path = "../sqld-libsql-bindings" }
 tokio = { version = "1.22.0", features = ["full"] }
 tracing = "0.1.37"
 tracing-subscriber = "0.3.16"

bottomless/src/ffi.rs

@@ -1,4 +1,4 @@
-pub use sqld::libsql::ffi::{
+pub use sqld_libsql_bindings::ffi::{
     libsql_wal_methods, sqlite3, sqlite3_file, sqlite3_vfs, PageHdrIter, PgHdr, Wal, WalIndexHdr,
     SQLITE_CANTOPEN, SQLITE_CHECKPOINT_TRUNCATE, SQLITE_IOERR_WRITE, SQLITE_OK,
 };

sqld-libsql-bindings/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "sqld-libsql-bindings"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.66"
+libsql-wasmtime-bindings = "0"
+mvfs = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
+mwal = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
+rusqlite = { git = "https://github.com/psarna/rusqlite", rev = "4df75df0b0f5ecea2714a8b21bf9f2a0fcfd5905", default-features = false, features = [
+    "buildtime_bindgen",
+    "bundled-libsql",
+    "column_decltype"
+] }
+tracing = "0.1.37"
+
+[features]
+mwal_backend = ["mvfs", "mwal"]

sqld/src/libsql/ffi/mod.rs renamed to sqld-libsql-bindings/src/ffi/mod.rs

@@ -12,14 +12,14 @@ pub use wblibsql::{
 use anyhow::ensure;
 use rusqlite::Connection;
 
-use crate::libsql::{ffi::libsql_wal_methods_register, wal_hook::WalMethodsHook};
+use crate::{ffi::libsql_wal_methods_register, wal_hook::WalMethodsHook};
 
 use self::{
     ffi::{libsql_wal_methods, libsql_wal_methods_find},
     wal_hook::WalHook,
 };
 
-fn get_orig_wal_methods() -> anyhow::Result<*mut libsql_wal_methods> {
+pub fn get_orig_wal_methods() -> anyhow::Result<*mut libsql_wal_methods> {
     let orig: *mut libsql_wal_methods = unsafe { libsql_wal_methods_find(std::ptr::null()) };
     if orig.is_null() {
         anyhow::bail!("no underlying methods");
@@ -28,7 +28,7 @@ fn get_orig_wal_methods() -> anyhow::Result<*mut libsql_wal_methods> {
     Ok(orig)
 }
 
-pub(crate) fn open_with_regular_wal(
+pub fn open_with_regular_wal(
     path: impl AsRef<std::path::Path>,
     flags: rusqlite::OpenFlags,
     wal_hook: impl WalHook + 'static,

sqld/src/libsql/ffi/types.rs renamed to sqld-libsql-bindings/src/ffi/types.rs

@@ -2,7 +2,9 @@
 
 use std::sync::{Arc, Mutex};
 
-pub(crate) fn open_with_virtual_wal(
+pub use mwal::ffi;
+
+pub fn open_with_virtual_wal(
     path: impl AsRef<std::path::Path>,
     flags: rusqlite::OpenFlags,
     vwal_methods: Arc<Mutex<mwal::ffi::libsql_wal_methods>>,

sqld/src/libsql/mod.rs renamed to sqld-libsql-bindings/src/lib.rs

@@ -19,10 +19,6 @@ hex = "0.4.3"
 hyper = { version = "0.14.23", features = ["http2"] }
 itertools = "0.10.5"
 jsonwebtoken = "8.2.0"
-libsql-wasmtime-bindings = "0"
-# Regular mvfs prevents users from enabling WAL mode
-mvfs = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
-mwal = { git = "https://github.com/psarna/mvsqlite", branch = "mwal", optional = true }
 once_cell = "1.17.0"
 parking_lot = "0.12.1"
 pgwire = "0.7.0"
@@ -38,6 +34,7 @@ rusqlite = { git = "https://github.com/psarna/rusqlite", rev = "4df75df0b0f5ecea
 serde = { version = "1.0.149", features = ["derive"] }
 serde_json = "1.0.91"
 smallvec = "1.10.0"
+sqld-libsql-bindings = { version = "0", path = "../sqld-libsql-bindings" }
 sqlite3-parser = { git = "https://github.com/gwenn/lemon-rs.git", rev = "be6d0a96c4424fada2e068b51c25d1fdc9f983f4", default-features = false, features = [
     "YYNOERRORRECOVERY"
 ] }
@@ -65,4 +62,4 @@ tonic-build = "0.8.4"
 vergen = "6"
 
 [features]
-mwal_backend = ["mvfs", "mwal"]
+mwal_backend = ["sqld-libsql-bindings/mwal_backend"]

sqld/src/libsql/mwal/mod.rs renamed to sqld-libsql-bindings/src/mwal/mod.rs

@@ -133,7 +133,7 @@ macro_rules! ok_or_exit {
 fn open_db(
     path: impl AsRef<Path> + Send + 'static,
     #[cfg(feature = "mwal_backend")] vwal_methods: Option<
-        Arc<Mutex<mwal::ffi::libsql_wal_methods>>,
+        Arc<Mutex<sqld_libsql_bindings::mwal::ffi::libsql_wal_methods>>,
     >,
     wal_hook: impl WalHook + Send + Clone + 'static,
 ) -> anyhow::Result<rusqlite::Connection> {
@@ -198,7 +198,7 @@ impl LibSqlDb {
     pub fn new(
         path: impl AsRef<Path> + Send + 'static,
         #[cfg(feature = "mwal_backend")] vwal_methods: Option<
-            Arc<Mutex<mwal::ffi::libsql_wal_methods>>,
+            Arc<Mutex<sqld_libsql_bindings::mwal::ffi::libsql_wal_methods>>,
         >,
         wal_hook: impl WalHook + Send + Clone + 'static,
     ) -> crate::Result<Self> {

sqld/src/libsql/wal_hook.rs renamed to sqld-libsql-bindings/src/wal_hook.rs

@@ -28,7 +28,8 @@ pub struct WriteProxyDbFactory {
     write_proxy: ProxyClient<Channel>,
     db_path: PathBuf,
     #[cfg(feature = "mwal_backend")]
-    vwal_methods: Option<Arc<std::sync::Mutex<mwal::ffi::libsql_wal_methods>>>,
+    vwal_methods:
+        Option<Arc<std::sync::Mutex<sqld_libsql_bindings::mwal::ffi::libsql_wal_methods>>>,
     /// abort handle: abort db update loop on drop
     _abort_handle: crossbeam::channel::Sender<()>,
 }
@@ -42,7 +43,7 @@ impl WriteProxyDbFactory {
         ca_cert_path: Option<PathBuf>,
         db_path: PathBuf,
         #[cfg(feature = "mwal_backend")] vwal_methods: Option<
-            Arc<std::sync::Mutex<mwal::ffi::libsql_wal_methods>>,
+            Arc<std::sync::Mutex<sqld_libsql_bindings::mwal::ffi::libsql_wal_methods>>,
         >,
     ) -> anyhow::Result<Self> {
         let mut endpoint = Channel::from_shared(addr.to_string())?;
@@ -115,7 +116,7 @@ impl WriteProxyDatabase {
         write_proxy: ProxyClient<Channel>,
         path: PathBuf,
         #[cfg(feature = "mwal_backend")] vwal_methods: Option<
-            Arc<std::sync::Mutex<mwal::ffi::libsql_wal_methods>>,
+            Arc<std::sync::Mutex<sqld_libsql_bindings::mwal::ffi::libsql_wal_methods>>,
         >,
     ) -> Result<Self> {
         let read_db = LibSqlDb::new(

sqld/Cargo.toml

@@ -22,10 +22,11 @@ use crate::error::Error;
 use crate::postgres::service::PgConnectionFactory;
 use crate::server::Server;
 
+pub use sqld_libsql_bindings as libsql;
+
 mod database;
 mod error;
 mod http;
-pub mod libsql;
 mod postgres;
 mod query;
 mod query_analysis;
@@ -137,10 +138,11 @@ pub async fn run_server(config: Config) -> anyhow::Result<()> {
     }
 
     #[cfg(feature = "mwal_backend")]
-    let vwal_methods = config
-        .mwal_addr
-        .as_ref()
-        .map(|_| Arc::new(Mutex::new(mwal::ffi::libsql_wal_methods::new())));
+    let vwal_methods = config.mwal_addr.as_ref().map(|_| {
+        Arc::new(Mutex::new(
+            sqld_libsql_bindings::mwal::ffi::libsql_wal_methods::new(),
+        ))
+    });
 
     match config.writer_rpc_addr {
         Some(ref addr) => {