Add wasm pre-compilation and move plugins to server #73

Author: Polleps

Cargo.lock

@@ -21,9 +21,6 @@ rmp-serde = { version = "1.3.0", optional = true }
 # Needed for migration to messagepack
 bincode1 = { package = "bincode", version = "1", optional = true }
 directories = { version = ">= 2, < 5", optional = true }
-html2md = { version = "0.2.14", optional = true }
-kuchikiki = { version = "0.8.2", optional = true }
-lol_html = { version = "1", optional = true }
 rand = { version = "0.8" }
 regex = "1"
 ring = "0.17.14"
@@ -40,11 +37,6 @@ url = "2"
 urlencoding = "2"
 ulid = "1.1.3"
 yrs = "0.24.0"
-wasmtime = { version = "39.0.1", optional = true, features = [
-  "component-model",
-] }
-wasmtime-wasi = { version = "39.0.1", optional = true, features = ["p2"] }
-wasmtime-wasi-http = "39.0.1"
 tokio = { version = "1", features = ["rt", "macros"] }
 async-trait = "0.1.89"
 futures = "0.3.31"
@@ -57,7 +49,5 @@ ntest = "0.9"
 
 [features]
 config = ["directories", "toml"]
-db = ["sled", "rmp-serde", "bincode1", "wasm-plugins"]
-html = ["kuchikiki", "lol_html", "html2md"]
+db = ["sled", "rmp-serde", "bincode1"]
 rdf = ["rio_api", "rio_turtle"]
-wasm-plugins = ["wasmtime", "wasmtime-wasi"]

lib/Cargo.toml

@@ -30,7 +30,6 @@ use crate::{
     },
     endpoints::{Endpoint, HandleGetContext},
     errors::{AtomicError, AtomicResult},
-    plugins::{plugins, wasm},
     resources::PropVals,
     storelike::{Query, QueryResult, ResourceResponse, Storelike},
     values::SortableValue,
@@ -74,7 +73,7 @@ pub struct Db {
     /// Try not to use this directly, but use the Trees.
     db: sled::Db,
     default_agent: Arc<Mutex<Option<crate::agents::Agent>>>,
-    /// Stores all resources. The Key is the Subject as a `string.as_bytes()`, the value a [PropVals]. Propvals must be serialized using [bincode].
+    /// Stores all resources. The Key is the Subject as a `string.as_bytes()`, the value a [PropVals]. Propvals must be serialized using messagepack.
     resources: sled::Tree,
     /// [Tree::ValPropSub]
     reference_index: sled::Tree,
@@ -109,9 +108,8 @@ impl Db {
         let query_index = db.open_tree(Tree::QueryMembers)?;
         let prop_val_sub_index = db.open_tree(Tree::PropValSub)?;
         let watched_queries = db.open_tree(Tree::WatchedQueries)?;
-        let class_extenders = plugins::default_class_extenders();
 
-        let mut store = Db {
+        let store = Db {
             path: path.into(),
             db,
             default_agent: Arc::new(Mutex::new(None)),
@@ -121,14 +119,11 @@ impl Db {
             prop_val_sub_index,
             server_url,
             watched_queries,
-            endpoints: plugins::default_endpoints(),
-            class_extenders,
+            endpoints: vec![],
+            class_extenders: vec![],
             on_commit: None,
         };
 
-        let extenders = wasm::load_wasm_class_extenders(path, &store).await;
-        store.class_extenders.extend(extenders);
-
         migrate_maybe(&store).map(|e| format!("Error during migration of database: {:?}", e))?;
         crate::populate::populate_base_models(&store)
             .await
@@ -152,6 +147,20 @@ impl Db {
         Ok(store)
     }
 
+    pub fn add_class_extender(&mut self, class_extender: ClassExtender) -> AtomicResult<()> {
+        self.class_extenders.push(class_extender);
+        Ok(())
+    }
+
+    pub fn add_endpoint(&mut self, endpoint: Endpoint) -> AtomicResult<()> {
+        self.endpoints.push(endpoint);
+        Ok(())
+    }
+
+    pub fn get_endpoints(&self) -> &Vec<Endpoint> {
+        &self.endpoints
+    }
+
     #[instrument(skip(self))]
     fn add_atom_to_index(
         &self,

lib/src/db.rs

@@ -203,17 +203,6 @@ impl From<std::string::FromUtf8Error> for AtomicError {
     }
 }
 
-#[cfg(feature = "wasm-plugins")]
-impl From<wasmtime::Error> for AtomicError {
-    fn from(error: wasmtime::Error) -> Self {
-        AtomicError {
-            message: error.to_string(),
-            error_type: AtomicErrorType::OtherError,
-            subject: None,
-        }
-    }
-}
-
 impl From<ParseFloatError> for AtomicError {
     fn from(error: ParseFloatError) -> Self {
         AtomicError {

lib/src/errors.rs

@@ -77,8 +77,7 @@ pub mod errors;
 pub mod hierarchy;
 pub mod mapping;
 pub mod parse;
-#[cfg(feature = "db")]
-pub mod plugins;
+
 pub mod populate;
 pub mod resources;
 pub mod schema;

lib/src/lib.rs

@@ -318,7 +318,7 @@ pub async fn populate_collections(store: &impl Storelike) -> AtomicResult<()> {
 /// Adds default Endpoints (versioning) to the Db.
 /// Makes sure they are fetchable
 pub async fn populate_endpoints(store: &crate::Db) -> AtomicResult<()> {
-    let endpoints = crate::plugins::plugins::default_endpoints();
+    let endpoints = store.get_endpoints();
     let endpoints_collection = format!("{}/endpoints", store.get_server_url()?);
     for endpoint in endpoints {
         let mut resource = endpoint.to_resource(store).await?;

lib/src/plugins/plugins.rs

@@ -54,6 +54,15 @@ ureq = "2"
 urlencoding = "2"
 ring = "0.17.14"
 yrs = "0.24.0"
+wasmtime = { version = "39.0.1", optional = true, features = [
+    "component-model",
+] }
+wasmtime-wasi = { version = "39.0.1", optional = true, features = ["p2"] }
+wasmtime-wasi-http = { version = "39.0.1", optional = true }
+url = "2.5.7"
+html2md = { version = "0.2.14" }
+kuchikiki = { version = "0.8.2" }
+lol_html = { version = "1" }
 
 [dependencies.instant-acme]
 optional = true
@@ -95,7 +104,7 @@ version = "4.4"
 version = ">= 4.0.1"
 
 [dependencies.atomic_lib]
-features = ["config", "db", "rdf", "html"]
+features = ["config", "db", "rdf"]
 path = "../lib"
 version = "0.40.0"
 
@@ -135,6 +144,7 @@ actix-rt = "2"
 assert_cmd = "2"
 
 [features]
+default = ["https", "telemetry", "img", "wasm-plugins"]
 telemetry = [
     "tracing-opentelemetry",
     "opentelemetry",
@@ -144,8 +154,8 @@ telemetry = [
     "opentelemetry_sdk/rt-tokio",    # 👈 important for batching
 ]
 img = ["webp", "image"]
-default = ["https", "telemetry", "img"]
 https = ["rustls", "instant-acme", "rcgen", "rustls-pemfile"]
+wasm-plugins = ["wasmtime", "wasmtime-wasi", "wasmtime-wasi-http"]
 
 [lib]
 name = "atomic_server_lib"

lib/src/populate.rs

@@ -3,6 +3,7 @@ use crate::{
     commit_monitor::CommitMonitor,
     config::Config,
     errors::AtomicServerResult,
+    plugins,
     search::SearchState,
     y_sync_broadcaster::{self, YSyncBroadcaster},
 };
@@ -13,6 +14,7 @@ use atomic_lib::{
     Storelike,
 };
 
+use crate::plugins::wasm;
 /// The AppState contains all the relevant Context for the server.
 /// This data object is available to all handlers and actors.
 /// Contains the store, configuration and addresses for Actix Actors, such as for the [CommitMonitor].
@@ -47,6 +49,36 @@ impl AppState {
         }
 
         let mut store = atomic_lib::Db::init(&config.store_path, config.server_url.clone()).await?;
+
+        // Register all built-in class extenders
+        store.add_class_extender(plugins::collections::build_collection_extender())?;
+        store.add_class_extender(plugins::chatroom::build_chatroom_extender())?;
+        store.add_class_extender(plugins::chatroom::build_message_extender())?;
+        store.add_class_extender(plugins::invite::build_invite_extender())?;
+
+        // Register all built-in endpoints
+        store.add_endpoint(plugins::versioning::version_endpoint())?;
+        store.add_endpoint(plugins::versioning::all_versions_endpoint())?;
+        store.add_endpoint(plugins::bookmark::bookmark_endpoint())?;
+        store.add_endpoint(plugins::files::upload_endpoint())?;
+        store.add_endpoint(plugins::files::download_endpoint())?;
+        store.add_endpoint(plugins::export::export_endpoint())?;
+        store.add_endpoint(plugins::path::path_endpoint())?;
+        store.add_endpoint(plugins::importer::import_endpoint())?;
+        #[cfg(debug_assertions)]
+        store.add_endpoint(plugins::prunetests::prune_tests_endpoint())?;
+        store.add_endpoint(plugins::query::query_endpoint())?;
+        store.add_endpoint(plugins::search::search_endpoint())?;
+
+        // Get and register Wasm class extender plugins
+        let extenders =
+            wasm::load_wasm_class_extenders(&config.plugin_path, &config.plugin_cache_path, &store)
+                .await;
+
+        for extender in extenders {
+            store.add_class_extender(extender)?;
+        }
+
         let no_server_resource = store.get_resource(&config.server_url).await.is_err();
         if no_server_resource {
             tracing::warn!("Server URL resource not found. This is likely because the server URL has changed. Initializing a new database...");

server/Cargo.toml

@@ -13,6 +13,7 @@ mod helpers;
 #[cfg(feature = "https")]
 mod https;
 mod jsonerrors;
+pub mod plugins;
 mod routes;
 pub mod serve;
 mod y_sync_broadcaster;