storage: choose the backend through the configuration

Author: pietroalbini

docker-compose.yml

@@ -18,6 +18,7 @@ services:
         environment:
             CRATESFYI_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
             CRATESFYI_DATABASE_URL: postgresql://cratesfyi:password@db
+            DOCSRS_STORAGE_BACKEND: s3
             S3_ENDPOINT: http://s3:9000
             AWS_ACCESS_KEY_ID: cratesfyi
             AWS_SECRET_ACCESS_KEY: secret_key

src/config.rs

@@ -1,3 +1,4 @@
+use crate::storage::StorageBackendKind;
 use failure::{bail, format_err, Error, Fail, ResultExt};
 use std::env::VarError;
 use std::path::PathBuf;
@@ -16,6 +17,9 @@ pub struct Config {
     pub(crate) max_pool_size: u32,
     pub(crate) min_pool_idle: u32,
 
+    // Storage params
+    pub(crate) storage_backend: StorageBackendKind,
+
     // S3 params
     pub(crate) s3_bucket: String,
     #[cfg(test)]
@@ -48,6 +52,8 @@ impl Config {
             max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?,
             min_pool_idle: env("DOCSRS_MIN_POOL_IDLE", 10)?,
 
+            storage_backend: env("DOCSRS_STORAGE_BACKEND", StorageBackendKind::Database)?,
+
             s3_bucket: env("DOCSRS_S3_BUCKET", "rust-docs-rs".to_string())?,
             // DO NOT CONFIGURE THIS THROUGH AN ENVIRONMENT VARIABLE!
             // Accidentally turning this on outside of the test suite might cause data loss in the

src/storage/mod.rs

@@ -67,6 +67,28 @@ pub fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>, Error> {
     Ok(files)
 }
 
+#[derive(Debug, failure::Fail)]
+#[fail(display = "invalid storage backend")]
+pub(crate) struct InvalidStorageBackendError;
+
+#[derive(Debug)]
+pub(crate) enum StorageBackendKind {
+    Database,
+    S3,
+}
+
+impl std::str::FromStr for StorageBackendKind {
+    type Err = InvalidStorageBackendError;
+
+    fn from_str(input: &str) -> Result<Self, Self::Err> {
+        match input {
+            "database" => Ok(StorageBackendKind::Database),
+            "s3" => Ok(StorageBackendKind::S3),
+            _ => Err(InvalidStorageBackendError),
+        }
+    }
+}
+
 enum StorageBackend {
     Database(DatabaseBackend),
     S3(Box<S3Backend>),
@@ -78,29 +100,17 @@ pub struct Storage {
 
 impl Storage {
     pub fn new(pool: Pool, metrics: Arc<Metrics>, config: &Config) -> Result<Self, Error> {
-        let backend = if let Some(c) = s3::s3_client() {
-            StorageBackend::S3(Box::new(S3Backend::new(c, metrics, config)?))
-        } else {
-            StorageBackend::Database(DatabaseBackend::new(pool, metrics))
-        };
-        Ok(Storage { backend })
-    }
-
-    #[cfg(test)]
-    pub(crate) fn temp_new_s3(metrics: Arc<Metrics>, config: &Config) -> Result<Self, Error> {
         Ok(Storage {
-            backend: StorageBackend::S3(Box::new(S3Backend::new(
-                s3::s3_client().unwrap(),
-                metrics,
-                config,
-            )?)),
-        })
-    }
-
-    #[cfg(test)]
-    pub(crate) fn temp_new_db(pool: Pool, metrics: Arc<Metrics>) -> Result<Self, Error> {
-        Ok(Storage {
-            backend: StorageBackend::Database(DatabaseBackend::new(pool, metrics)),
+            backend: match config.storage_backend {
+                StorageBackendKind::Database => {
+                    StorageBackend::Database(DatabaseBackend::new(pool, metrics))
+                }
+                StorageBackendKind::S3 => StorageBackend::S3(Box::new(S3Backend::new(
+                    s3::s3_client().unwrap(),
+                    metrics,
+                    config,
+                )?)),
+            },
         })
     }
 
@@ -564,18 +574,21 @@ mod backend_tests {
 
     macro_rules! backend_tests {
         (
-            backends($env:ident) { $($backend:ident => $create:expr,)* }
+            backends { $($backend:ident => $config:expr,)* }
             tests $tests:tt
             tests_with_metrics $tests_with_metrics:tt
         ) => {
             $(
                 mod $backend {
                     use crate::test::TestEnvironment;
-                    use crate::storage::Storage;
+                    use crate::storage::{Storage, StorageBackendKind};
                     use std::sync::Arc;
 
-                    fn get_storage($env: &TestEnvironment) -> Arc<Storage> {
-                        $create
+                    fn get_storage(env: &TestEnvironment) -> Arc<Storage> {
+                        env.override_config(|config| {
+                            config.storage_backend = $config;
+                        });
+                        env.storage()
                     }
 
                     backend_tests!(@tests $tests);
@@ -606,9 +619,9 @@ mod backend_tests {
     }
 
     backend_tests! {
-        backends(env) {
-            s3 => env.s3(),
-            database => env.temp_storage_db(),
+        backends {
+            s3 => StorageBackendKind::S3,
+            database => StorageBackendKind::Database,
         }
 
         tests {

src/test/mod.rs

@@ -98,8 +98,6 @@ pub(crate) struct TestEnvironment {
     storage: OnceCell<Arc<Storage>>,
     metrics: OnceCell<Arc<Metrics>>,
     frontend: OnceCell<TestFrontend>,
-    s3: OnceCell<Arc<Storage>>,
-    storage_db: OnceCell<Arc<Storage>>,
 }
 
 pub(crate) fn init_logger() {
@@ -119,8 +117,6 @@ impl TestEnvironment {
             storage: OnceCell::new(),
             metrics: OnceCell::new(),
             frontend: OnceCell::new(),
-            s3: OnceCell::new(),
-            storage_db: OnceCell::new(),
         }
     }
 
@@ -203,28 +199,6 @@ impl TestEnvironment {
         self.frontend.get_or_init(|| TestFrontend::new(&*self))
     }
 
-    pub(crate) fn s3(&self) -> Arc<Storage> {
-        self.s3
-            .get_or_init(|| {
-                Arc::new(
-                    Storage::temp_new_s3(self.metrics(), &*self.config())
-                        .expect("failed to initialize the storage"),
-                )
-            })
-            .clone()
-    }
-
-    pub(crate) fn temp_storage_db(&self) -> Arc<Storage> {
-        self.storage_db
-            .get_or_init(|| {
-                Arc::new(
-                    Storage::temp_new_db(self.db().pool(), self.metrics())
-                        .expect("failed to initialize the storage"),
-                )
-            })
-            .clone()
-    }
-
     pub(crate) fn fake_release(&self) -> fakes::FakeRelease {
         fakes::FakeRelease::new(self.db(), self.storage())
     }