Merge pull request #150 from wfraser/async-mode

implement optional async support and remove nested result type in response

.github/workflows/cargo-test.yml

@@ -29,10 +29,10 @@ jobs:
         run: if [[ ! -z `git status --porcelain=v1` ]]; then echo "::error::Workspace is dirty after running generator. Did you remember to check in the generated files?"; exit 1; fi
 
       - name: Install MSRV toolchain
-        run: rustup install 1.65.0 --profile minimal
+        run: rustup install 1.75.0 --profile minimal
 
       - name: Run cargo test
-        run: rustup run 1.65.0 cargo test
+        run: rustup run 1.75.0 cargo test
 
       - name: Install nightly toolchain
         run: |
@@ -41,3 +41,9 @@ jobs:
 
       - name: Run clippy
         run: rustup run nightly cargo clippy --all-targets --all-features -- --deny warnings
+
+      - name: Run clippy with minimal features (sync)
+        run: rustup run nightly cargo clippy --no-default-features --features sync_routes,dbx_files
+
+      - name: Run clippy with minimal features (async)
+        run: rustup run nightly cargo clippy --no-default-features --features default_async_client,dbx_files

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dropbox-sdk"
-version = "0.18.1"
+version = "0.19.0-beta1"
 authors = ["Bill Fraser <[email protected]>"]
 edition = "2018"
 description = "Rust bindings to the Dropbox API, generated by Stone from the official spec."
@@ -12,20 +12,34 @@ readme = "README.md"
 
 [package.metadata]
 # Keep this at least 1 year old.
-msrv = "1.65.0" # Nov 3, 2022
+# (or not... 1.75 is required for "-> impl Trait" sadly)
+msrv = "1.75.0" # Dec 28, 2023
 
 [dependencies]
+async-lock = "3.3.0"
 atty = "0.2.14"
 base64 = "0.22"
+bytes = "1.6.0"
 log = "0.4"
 ring = "0.17"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"
 url = "2.1"
 
+[dependencies.futures]
+version = "0.3.30"
+default-features = false
+features = ["std"]
+
+[dependencies.reqwest]
+version = "0.12.2"
+optional = true
+default-features = false
+features = ["http2", "rustls-tls", "stream"]
+
 [dependencies.ureq]
-version = "2.0.0"
+version = "2.5.0"
 optional = true
 default-features = false
 features = ["tls"]
@@ -36,6 +50,15 @@ chrono = "0.4"
 parallel_reader = "0.1"
 threadpool = "1.8"
 
+[dev-dependencies.tokio]
+version = "1.37.0"
+features = ["rt-multi-thread", "macros", "io-std"]
+
+[dev-dependencies.tokio-util]
+version = "0.7.10"
+default-features = false
+features = ["compat"]
+
 [[example]]
 name = "demo"
 required-features = ["dbx_files", "default_client"]
@@ -44,6 +67,10 @@ required-features = ["dbx_files", "default_client"]
 name = "large-file-upload"
 required-features = ["dbx_files", "default_client"]
 
+[[example]]
+name = "demo-async"
+required-features = ["dbx_files", "default_async_client"]
+
 [features]
 # dbx_* features each correspond to one Stone spec file.
 # The lists of dependencies must be kept in sync with the 'import' statements in them.
@@ -68,12 +95,24 @@ dbx_team_policies = []
 dbx_users = ["dbx_common", "dbx_team_common", "dbx_team_policies", "dbx_users_common"]
 dbx_users_common = ["dbx_common"]
 
-default_client = ["ureq"]
+default_async_client = ["async_routes", "dep:reqwest"]
+default_client = ["sync_routes", "sync_routes_default", "dep:ureq"]
 
 # Enable unstable ("preview") API routes.
 unstable = []
 
-# Include everything by default.
+# Enable sync routes under `dropbox_sdk::routes::{namespace}`
+sync_routes = []
+
+# Enable async routes under `dropbox_sdk::async_routes::{namespace}`
+async_routes = []
+
+# Re-export the sync routes as `dropbox_sdk::{namsepace}` directly (matches pre-v0.19 structure).
+# If disabled, export the async routes there instead.
+sync_routes_default = ["sync_routes"]
+
+# Include all namespaces by default.
+# Enable sync default client, sync routes, and make the sync routes default, to match pre-v0.19.
 default = [
     "dbx_account",
     "dbx_async",
@@ -96,6 +135,8 @@ default = [
     "dbx_users",
     "dbx_users_common",
     "default_client",
+    "sync_routes",
+    "sync_routes_default",
     ]
 
 [package.metadata.docs.rs]

README.md

@@ -26,6 +26,20 @@ However, that said,
 * We are happy to get feedback and/or pull requests from the community! See
 [contributing](CONTRIBUTING.md) for more information.
 
+## Sync and Async
+
+The routes (functions) come in two variants: sync, which do blocking network
+I/O and return their value directly; and async, which return futures.
+
+The sync routes (in the `dropbox_sdk::sync_routes` module) are enabled by
+default or with the `sync_routes` feature, and likewise the async ones are
+in the `dropbox_sdk::async_routes` module and can be enabled with the
+`async_routes` feature.
+
+Additionally, if the `sync_routes_default` feature is on (as it is by
+default), the sync routes are available directly as `dropbox_sdk::{namespace}`,
+which matches the original structure before the async routes were added.
+
 ## HTTP Client
 
 To actually use the API calls, you need a HTTP client -- all functions take a
@@ -35,13 +49,18 @@ pass it as the client argument.
 
 If you don't want to implement your own, this SDK comes with an optional
 default client that uses `ureq` and `rustls`.  To use it, build with the
-`default_client` feature flag, and then there will be a set of clents in the
+`default_client` feature flag, and then there will be a set of clients in the
 `dropbox_sdk::default_client` module that you can use, corresponding to each of
 the authentication types Dropbox uses (see below). The default client needs a
 Dropbox API token; how you get one is up to you and your program. See the
 programs under [examples/](examples/) for examples, and see the helper code in
 the [oauth2](src/oauth2.rs) module.
 
+Async clients can be implemented using a parallel set of traits located in the
+`dropbox_sdk::async_client_trait` module. A default implementation (which uses
+`reqwest` can be enabled with the `default_async_client` feature and is located
+at `dropbox_sdk::default_async_client`.
+
 ## Authentication Types
 
 The Dropbox API has a number of different [authentication types]. Each route
@@ -109,11 +128,9 @@ Some implementation notes, limitations, and TODOs:
  * Stone allows structures to inherit from other structures and be polymorphic.
    Rust doesn't have these paradigms, so instead this SDK represents
    polymorphic parent structs as enums, and the inherited fields are put in all
-   variants.  See `dropbox_sdk::files::Metadata` for an example.
- * This crate only supports synchronous I/O. Eventually we probably want to
-   support async I/O, which will require making incompatible changes to the
-   types returned by routes. This should probably wait until the futures
-   ecosystem and async/await have stabilized some more.
+   variants.  See `dropbox_sdk::files::Metadata` for an example. Upcasting is
+   supported using generated `From` implementations which either construct the
+   right enum variant or copy the subset of common fields.
  * This code does not use `serde_derive` for the most part, and instead uses
    manually-emitted serialization code. Previous work on this crate did attempt
    to use `serde_derive`, but the way the Dropbox API serializes unions
@@ -125,5 +142,10 @@ Some implementation notes, limitations, and TODOs:
  * Types with constraints (such as strings with patterns or min/max lengths, or
    integers with a range) do not check that the data being stored in them meets
    the constraints.
+ * The sync routes and clients are actually implemented in terms of the async
+   client interfaces, but all the futures returned are `std::future::ready()`,
+   which is then removed using `now_or_never()` before being returned to
+   callers. Even though futures are passed around and async functions are used,
+   no executor is actually needed because of this.
 
 ## Happy Dropboxing!

RELEASE_NOTES.md

@@ -1,3 +1,18 @@
+# v0.19.0-beta1
+xxxx-yy-zz
+* **BIG CHANGE: async support added**
+  * HTTP client traits completely redesigned
+    * Should actually be much simpler to implement now, as the work of setting the right headers has been extracted out
+      into client_helpers code, and implementations now just need to provide a generic HttpRequest type which can set
+      any header.
+  * The default sync client is still the default enabled feature.
+  * To switch to the async mode, enable the `default_async_client` feature (and disable the `default_client`, `sync_routes`, `sync_routes_default` features).
+* **BIG CHANGE: no more nested Results**
+  * Functions which used to return `Result<Result<T, E>, dropbox_sdk::Error>` now return `Result<T, dropbox_sdk::Error<E>>`.
+    * in other words, `Ok(Err(e))` is now written `Err(dropbox_sdk::Error::Api(e))` and `Ok(Ok(v))` is just `Ok(v)`.
+    * `dropbox_sdk::Error` now has a type parameter which differs depending on the function being called.
+* MSRV raised to 1.71.0
+
 # v0.18.1
 2024-05-06
 * fixed bug when using oauth2 refresh tokens using client secret instead of PKCE:

examples/demo-async.rs

@@ -0,0 +1,177 @@
+#![deny(rust_2018_idioms)]
+
+//! This example illustrates a few basic Dropbox API operations: getting an OAuth2 token, listing
+//! the contents of a folder recursively, and fetching a file given its path.
+
+use tokio_util::compat::FuturesAsyncReadCompatExt;
+use dropbox_sdk::default_async_client::{NoauthDefaultClient, UserAuthDefaultClient};
+use dropbox_sdk::async_routes::files;
+
+enum Operation {
+    Usage,
+    List(String),
+    Download(String),
+    Stat(String),
+}
+
+fn parse_args() -> Operation {
+    let mut ctor: Option<fn(String) -> Operation> = None;
+    for arg in std::env::args().skip(1) {
+        match arg.as_str() {
+            "--help" | "-h" => return Operation::Usage,
+            "--list" => {
+                ctor = Some(Operation::List);
+            }
+            "--download" => {
+                ctor = Some(Operation::Download);
+            }
+            "--stat" => {
+                ctor = Some(Operation::Stat);
+            }
+            path if path.starts_with('/') => {
+                return if let Some(ctor) = ctor {
+                    ctor(arg)
+                } else {
+                    eprintln!("Either --download or --list must be specified");
+                    Operation::Usage
+                };
+            }
+            _ => {
+                eprintln!("Unrecognized option {arg:?}");
+                eprintln!();
+                return Operation::Usage;
+            }
+        }
+    }
+    Operation::Usage
+}
+
+#[tokio::main]
+async fn main() {
+    env_logger::init();
+
+    let op = parse_args();
+
+    if let Operation::Usage = op {
+        eprintln!("usage: {} [option]", std::env::args().next().unwrap());
+        eprintln!("    options:");
+        eprintln!("        --help | -h          view this text");
+        eprintln!("        --download <path>    copy the contents of <path> to stdout");
+        eprintln!("        --list <path>        recursively list all files under <path>");
+        eprintln!("        --stat <path>        list all metadata of <path>");
+        eprintln!();
+        eprintln!("    If a Dropbox OAuth token is given in the environment variable");
+        eprintln!("    DBX_OAUTH_TOKEN, it will be used, otherwise you will be prompted for");
+        eprintln!("    authentication interactively.");
+        std::process::exit(1);
+    }
+
+    let mut auth = dropbox_sdk::oauth2::get_auth_from_env_or_prompt();
+    if auth.save().is_none() {
+        auth.obtain_access_token_async(NoauthDefaultClient::default()).await.unwrap();
+        eprintln!("Next time set these environment variables to reuse this authorization:");
+        eprintln!("  DBX_CLIENT_ID={}", auth.client_id());
+        eprintln!("  DBX_OAUTH={}", auth.save().unwrap());
+    }
+    let client = UserAuthDefaultClient::new(auth);
+
+    match op {
+        Operation::Usage => (), // handled above
+        Operation::Download(path) => {
+            eprintln!("Copying file to stdout: {}", path);
+            eprintln!();
+
+            match files::download(&client, &files::DownloadArg::new(path), None, None).await {
+                Ok(result) => {
+                    match tokio::io::copy(
+                        &mut result.body.expect("there must be a response body")
+                            .compat(),
+                        &mut tokio::io::stdout(),
+                    ).await {
+                        Ok(n) => {
+                            eprintln!("Downloaded {n} bytes");
+                        }
+                        Err(e) => {
+                            eprintln!("I/O error: {e}");
+                        }
+                    }
+                }
+                Err(e) => {
+                    eprintln!("Error from files/download: {e}");
+                }
+            }
+        }
+        Operation::List(mut path) => {
+            eprintln!("Listing recursively: {path}");
+
+            // Special case: the root folder is empty string. All other paths need to start with '/'.
+            if path == "/" {
+                path.clear();
+            }
+
+            let mut result = match files::list_folder(
+                &client,
+                &files::ListFolderArg::new(path).with_recursive(true),
+            ).await {
+                Ok(result) => result,
+                Err(e) => {
+                    eprintln!("Error from files/list_folder: {e}");
+                    return;
+                }
+            };
+
+            let mut num_entries = result.entries.len();
+            let mut num_pages = 1;
+
+            loop {
+                for entry in result.entries {
+                    match entry {
+                        files::Metadata::Folder(entry) => {
+                            println!("Folder: {}", entry.path_display.unwrap_or(entry.name));
+                        }
+                        files::Metadata::File(entry) => {
+                            println!("File: {}", entry.path_display.unwrap_or(entry.name));
+                        }
+                        files::Metadata::Deleted(entry) => {
+                            panic!("unexpected deleted entry: {:?}", entry);
+                        }
+                    }
+                }
+
+                if !result.has_more {
+                    break;
+                }
+
+                result = match files::list_folder_continue(
+                    &client,
+                    &files::ListFolderContinueArg::new(result.cursor),
+                ).await {
+                    Ok(result) => {
+                        num_pages += 1;
+                        num_entries += result.entries.len();
+                        result
+                    }
+                    Err(e) => {
+                        eprintln!("Error from files/list_folder_continue: {e}");
+                        break;
+                    }
+                }
+            }
+
+            eprintln!("{num_entries} entries from {num_pages} result pages");
+        }
+        Operation::Stat(path) => {
+            eprintln!("listing metadata for: {path}");
+
+            let arg = files::GetMetadataArg::new(path)
+                .with_include_media_info(true)
+                .with_include_deleted(true)
+                .with_include_has_explicit_shared_members(true);
+
+            match files::get_metadata(&client, &arg).await {
+                Ok(result) => println!("{result:#?}"),
+                Err(e) => eprintln!("Error from files/get_metadata: {e}"),
+            }
+        }
+    }
+}