diff --git a/Cargo.lock b/Cargo.lock index 9aac5f7d4..09e33d067 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,6 +64,17 @@ dependencies = [ "backtrace", ] +[[package]] +name = "assert-json-diff" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259cbe96513d2f1073027a259fc2ca917feb3026a5a8d984e3628e490255cc0" +dependencies = [ + "extend", + "serde", + "serde_json", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -186,6 +197,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-cloudfront" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99b3b9423539dd0479f7a107c9411b865f1f5143f49f54165db424071e41cfe" +dependencies = [ + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "tokio-stream", + "tower", +] + [[package]] name = "aws-sdk-s3" version = "0.18.0" @@ -334,6 +367,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-tower", + "aws-smithy-protocol-test", "aws-smithy-types", "bytes", "fastrand", @@ -343,6 +377,7 @@ dependencies = [ "hyper-rustls", "lazy_static", "pin-project-lite", + "serde", "tokio", "tower", "tracing", @@ -405,6 +440,21 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a615c2f5d1df9a970e5221bd05fddf4a89d1cb6e1e8b2282b0bd9cb5c9b4e444" +dependencies = [ + "assert-json-diff 1.1.0", + "http", + "pretty_assertions", + "regex", + "roxmltree", + "serde_json", + "thiserror", +] + [[package]] name = "aws-smithy-query" version = "0.48.0" @@ -928,6 +978,16 @@ dependencies = [ "sct", ] +[[package]] +name = "ctor" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "dashmap" version = "5.4.0" @@ -970,6 +1030,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690" +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.5" @@ -1007,7 +1073,10 @@ version = "0.6.0" dependencies = [ "anyhow", "aws-config", + "aws-sdk-cloudfront", "aws-sdk-s3", + "aws-smithy-client", + "aws-smithy-http", "aws-smithy-types-convert", "backtrace", "base64 0.13.0", @@ -1076,6 +1145,7 @@ dependencies = [ "tokio", "toml", "url 2.3.1", + "uuid", "walkdir", "zip", "zstd", @@ -1160,6 +1230,18 @@ dependencies = [ "libc", ] +[[package]] +name = "extend" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "failure" version = "0.1.8" @@ -2105,7 +2187,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "401edc088069634afaa5f4a29617b36dba683c0c16fe4435a86debad23fa2f1a" dependencies = [ - "assert-json-diff", + "assert-json-diff 2.0.2", "colored", "httparse", "lazy_static", @@ -2307,6 +2389,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "output_vt100" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" +dependencies = [ + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2741,6 +2832,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "pretty_assertions" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +dependencies = [ + "ctor", + "diff", + "output_vt100", + "yansi", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3193,6 +3296,15 @@ dependencies = [ "url 1.7.2", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -4029,9 +4141,21 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2", + "tokio-macros", "winapi", ] +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.3.0" @@ -4670,6 +4794,12 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index defc759c3..f5ddfa0e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,8 +73,10 @@ tokio = { version = "1.0", features = ["rt-multi-thread"] } futures-util = "0.3.5" aws-config = "0.48.0" aws-sdk-s3 = "0.18.0" +aws-sdk-cloudfront = "0.18.0" aws-smithy-types-convert = { version = "0.48.0", features = ["convert-chrono"] } http = "0.2.6" +uuid = "1.1.2" # Data serialization and deserialization serde = { version = "1.0", features = ["derive"] } @@ -114,6 +116,9 @@ rand = "0.8" mockito = "0.31.0" test-case = "2.0.0" fn-error-context = "0.2.0" +aws-smithy-client = { version = "0.48.0", features = ["test-util"]} +aws-smithy-http = "0.48.0" +tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] } [build-dependencies] time = "0.3" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 0a402cfb4..71abdeda5 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{anyhow, Context as _, Error, Result}; +use docs_rs::cdn::CdnBackend; use docs_rs::db::{self, add_path_into_database, Pool, PoolClient}; use docs_rs::repositories::RepositoryStatsUpdater; use docs_rs::utils::{ @@ -16,6 +17,7 @@ use once_cell::sync::OnceCell; use sentry_log::SentryLogger; use structopt::StructOpt; use strum::VariantNames; +use tokio::runtime::Runtime; fn main() { let _sentry_guard = if let Ok(sentry_dsn) = env::var("SENTRY_DSN") { @@ -558,11 +560,13 @@ enum DeleteSubcommand { struct BinContext { build_queue: OnceCell>, storage: OnceCell>, + cdn: OnceCell>, config: OnceCell>, pool: OnceCell, metrics: OnceCell>, index: OnceCell>, repository_stats_updater: OnceCell>, + runtime: OnceCell>, } impl BinContext { @@ -570,11 +574,13 @@ impl BinContext { Self { build_queue: OnceCell::new(), storage: OnceCell::new(), + cdn: OnceCell::new(), config: OnceCell::new(), pool: OnceCell::new(), metrics: OnceCell::new(), index: OnceCell::new(), repository_stats_updater: OnceCell::new(), + runtime: OnceCell::new(), } } @@ -606,9 +612,12 @@ impl Context for BinContext { self.pool()?, self.metrics()?, self.config()?, + self.runtime()?, )?; + fn cdn(self) -> CdnBackend = CdnBackend::new(&self.config()?, &self.runtime()?); fn config(self) -> Config = Config::from_env()?; fn metrics(self) -> Metrics = Metrics::new()?; + fn runtime(self) -> Runtime = Runtime::new()?; fn index(self) -> Index = { let config = self.config()?; let path = config.registry_index_path.clone(); diff --git a/src/cdn.rs b/src/cdn.rs new file mode 100644 index 000000000..241df3801 --- /dev/null +++ b/src/cdn.rs @@ -0,0 +1,224 @@ +use crate::Config; +use anyhow::{Error, Result}; +use aws_sdk_cloudfront::{ + model::{InvalidationBatch, Paths}, + Client, RetryConfig, +}; +use std::sync::{Arc, Mutex}; +use strum::EnumString; +use tokio::runtime::Runtime; +use uuid::Uuid; + +#[derive(Debug, EnumString)] +pub(crate) enum CdnKind { + #[strum(ascii_case_insensitive)] + Dummy, + + #[strum(ascii_case_insensitive)] + CloudFront, +} + +pub enum CdnBackend { + Dummy(Arc>>), + CloudFront { + runtime: Arc, + client: Client, + }, +} + +impl CdnBackend { + pub fn new(config: &Arc, runtime: &Arc) -> CdnBackend { + match config.cdn_backend { + CdnKind::CloudFront => { + let shared_config = runtime.block_on(aws_config::load_from_env()); + let config_builder = aws_sdk_cloudfront::config::Builder::from(&shared_config) + .retry_config(RetryConfig::new().with_max_attempts(3)); + + Self::CloudFront { + runtime: runtime.clone(), + client: Client::from_conf(config_builder.build()), + } + } + CdnKind::Dummy => Self::Dummy(Arc::new(Mutex::new(Vec::new()))), + } + } + /// create a Front invalidation request for a list of path patterns. + /// patterns can be + /// * `/filename.ext` (a specific path) + /// * `/directory-path/file-name.*` (delete these files, all extensions) + /// * `/directory-path/*` (invalidate all of the files in a directory, without subdirectories) + /// * `/directory-path*` (recursive directory delete, including subdirectories) + /// see https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#invalidation-specifying-objects + /// + /// Returns the caller reference that can be used to query the status of this + /// invalidation request. + pub(crate) fn create_invalidation( + &self, + distribution_id: &str, + path_patterns: &[&str], + ) -> Result { + let caller_reference = Uuid::new_v4(); + + match *self { + CdnBackend::CloudFront { + ref runtime, + ref client, + } => { + runtime.block_on(CdnBackend::cloudfront_invalidation( + client, + distribution_id, + &caller_reference.to_string(), + path_patterns, + ))?; + } + CdnBackend::Dummy(ref invalidation_requests) => { + let mut invalidation_requests = invalidation_requests + .lock() + .expect("could not lock mutex on dummy CDN"); + + invalidation_requests.extend( + path_patterns + .iter() + .map(|p| (distribution_id.to_owned(), (*p).to_owned())), + ); + } + } + + Ok(caller_reference) + } + + async fn cloudfront_invalidation( + client: &Client, + distribution_id: &str, + caller_reference: &str, + path_patterns: &[&str], + ) -> Result<(), Error> { + let path_patterns: Vec<_> = path_patterns.iter().cloned().map(String::from).collect(); + + client + .create_invalidation() + .distribution_id(distribution_id) + .invalidation_batch( + InvalidationBatch::builder() + .paths( + Paths::builder() + .quantity(path_patterns.len().try_into().unwrap()) + .set_items(Some(path_patterns)) + .build(), + ) + .caller_reference(caller_reference) + .build(), + ) + .send() + .await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test::wrapper; + + use aws_sdk_cloudfront::{Client, Config, Credentials, Region}; + use aws_smithy_client::{erase::DynConnector, test_connection::TestConnection}; + use aws_smithy_http::body::SdkBody; + + #[test] + fn create_cloudfront() { + wrapper(|env| { + env.override_config(|config| { + config.cdn_backend = CdnKind::CloudFront; + }); + + assert!(matches!(*env.cdn(), CdnBackend::CloudFront { .. })); + assert!(matches!( + CdnBackend::new(&env.config(), &env.runtime()), + CdnBackend::CloudFront { .. } + )); + + Ok(()) + }) + } + + #[test] + fn create_dummy() { + wrapper(|env| { + assert!(matches!(*env.cdn(), CdnBackend::Dummy { .. })); + assert!(matches!( + CdnBackend::new(&env.config(), &env.runtime()), + CdnBackend::Dummy { .. } + )); + + Ok(()) + }) + } + + async fn get_mock_config() -> aws_sdk_cloudfront::Config { + let cfg = aws_config::from_env() + .region(Region::new("eu-central-1")) + .credentials_provider(Credentials::new( + "accesskey", + "privatekey", + None, + None, + "dummy", + )) + .load() + .await; + + Config::new(&cfg) + } + + #[tokio::test] + async fn invalidate_path() { + let conn = TestConnection::new(vec![( + http::Request::builder() + .header("content-type", "application/xml") + .uri(http::uri::Uri::from_static( + "https://cloudfront.amazonaws.com/2020-05-31/distribution/some_distribution/invalidation", + )) + .body(SdkBody::from( + r#"2/some/path*/another/path/*some_reference"#, + )) + .unwrap(), + http::Response::builder() + .status(200) + .body(SdkBody::from( + r#" + + 2019-12-05T18:40:49.413Z + I2J0I21PCUYOIK + + some_reference + + + /some/path* + /another/path/* + + 2 + + + InProgress + + "#, + )) + .unwrap(), + )]); + let client = + Client::from_conf_conn(get_mock_config().await, DynConnector::new(conn.clone())); + + CdnBackend::cloudfront_invalidation( + &client, + "some_distribution", + "some_reference", + &["/some/path*", "/another/path/*"], + ) + .await + .expect("error creating invalidation"); + + assert_eq!(conn.requests().len(), 1); + conn.assert_requests_match(&[]); + } +} diff --git a/src/config.rs b/src/config.rs index 631d6672b..5e9d5a540 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -use crate::storage::StorageKind; +use crate::{cdn::CdnKind, storage::StorageKind}; use anyhow::{anyhow, bail, Context, Result}; use std::env::VarError; use std::error::Error; @@ -57,12 +57,18 @@ pub struct Config { // Content Security Policy pub(crate) csp_report_only: bool, - // Cache-Control header + // Cache-Control header, for versioned URLs. // If both are absent, don't generate the header. If only one is present, // generate just that directive. Values are in seconds. pub(crate) cache_control_stale_while_revalidate: Option, pub(crate) cache_control_max_age: Option, + pub(crate) cdn_backend: CdnKind, + + // CloudFront distribution ID for the web server. + // Will be used for invalidation-requests. + pub cloudfront_distribution_id_web: Option, + // Build params pub(crate) build_attempts: u16, pub(crate) rustwide_workspace: PathBuf, @@ -141,6 +147,10 @@ impl Config { )?, cache_control_max_age: maybe_env("CACHE_CONTROL_MAX_AGE")?, + cdn_backend: env("DOCSRS_CDN_BACKEND", CdnKind::Dummy)?, + + cloudfront_distribution_id_web: maybe_env("CLOUDFRONT_DISTRIBUTION_ID_WEB")?, + local_archive_cache_path: env( "DOCSRS_ARCHIVE_INDEX_CACHE_PATH", prefix.join("archive_cache"), diff --git a/src/context.rs b/src/context.rs index a2a54bf32..ff28681ae 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,15 +1,19 @@ +use crate::cdn::CdnBackend; use crate::db::Pool; use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; use crate::{BuildQueue, Config, Index, Metrics, Storage}; use std::sync::Arc; +use tokio::runtime::Runtime; pub trait Context { fn config(&self) -> Result>; fn build_queue(&self) -> Result>; fn storage(&self) -> Result>; + fn cdn(&self) -> Result>; fn pool(&self) -> Result; fn metrics(&self) -> Result>; fn index(&self) -> Result>; fn repository_stats_updater(&self) -> Result>; + fn runtime(&self) -> Result>; } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 97d653909..125fded44 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,3 +1,4 @@ +use crate::cdn::CdnBackend; use crate::db::file::add_path_into_database; use crate::db::{ add_build_into_database, add_doc_coverage, add_package_into_database, @@ -9,11 +10,12 @@ use crate::index::api::ReleaseData; use crate::repositories::RepositoryStatsUpdater; use crate::storage::{rustdoc_archive_path, source_archive_path}; use crate::utils::{ - copy_dir_all, parse_rustc_version, queue_builder, set_config, CargoMetadata, ConfigName, + copy_dir_all, parse_rustc_version, queue_builder, report_error, set_config, CargoMetadata, + ConfigName, }; use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; use crate::{Config, Context, Index, Metrics, Storage}; -use anyhow::{anyhow, bail, Error}; +use anyhow::{anyhow, bail, Context as _, Error}; use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET}; use failure::Error as FailureError; use log::{debug, info, warn, LevelFilter}; @@ -42,6 +44,7 @@ pub struct RustwideBuilder { config: Arc, db: Pool, storage: Arc, + cdn: Arc, metrics: Arc, index: Arc, rustc_version: String, @@ -80,6 +83,7 @@ impl RustwideBuilder { config, db: context.pool()?, storage: context.storage()?, + cdn: context.cdn()?, metrics: context.metrics()?, index: context.index()?, rustc_version: String::new(), @@ -500,6 +504,18 @@ impl RustwideBuilder { .purge_from_cache(&self.workspace) .map_err(FailureError::compat)?; local_storage.close()?; + if let Some(distribution_id) = self.config.cloudfront_distribution_id_web.as_ref() { + if let Err(err) = self + .cdn + .create_invalidation( + distribution_id, + &[&format!("/{}*", name), &format!("/crate/{}*", name)], + ) + .context("error creating CDN invalidation") + { + report_error(&err); + } + } Ok(successful) } @@ -806,7 +822,10 @@ pub(crate) struct BuildResult { #[cfg(test)] mod tests { use super::*; - use crate::test::{assert_redirect, assert_success, wrapper}; + use crate::{ + cdn::CdnKind, + test::{assert_redirect, assert_success, wrapper}, + }; use serde_json::Value; #[test] @@ -923,6 +942,12 @@ mod tests { } } + assert!(matches!(*env.cdn(), CdnBackend::Dummy(_))); + if let CdnBackend::Dummy(ref invalidation_requests) = *env.cdn() { + let ir = invalidation_requests.lock().unwrap(); + assert!(ir.is_empty()); + } + Ok(()) }) } @@ -1044,6 +1069,37 @@ mod tests { }); } + #[test] + #[ignore] + fn test_cdn_invalidation() { + wrapper(|env| { + env.override_config(|cfg| { + cfg.cdn_backend = CdnKind::Dummy; + cfg.cloudfront_distribution_id_web = Some("distribution_id".into()); + }); + + let mut builder = RustwideBuilder::init(env).unwrap(); + assert!(builder.build_package( + DUMMY_CRATE_NAME, + DUMMY_CRATE_VERSION, + PackageKind::CratesIo + )?); + + assert!(matches!(*env.cdn(), CdnBackend::Dummy(_))); + if let CdnBackend::Dummy(ref invalidation_requests) = *env.cdn() { + let ir = invalidation_requests.lock().unwrap(); + assert_eq!(ir.len(), 2); + assert_eq!(ir[0], ("distribution_id".into(), "/empty-library*".into())); + assert_eq!( + ir[1], + ("distribution_id".into(), "/crate/empty-library*".into()) + ); + } + + Ok(()) + }); + } + #[test] #[ignore] fn test_locked_fails_unlocked_needs_new_deps() { diff --git a/src/lib.rs b/src/lib.rs index 32075ee7e..d8c81791d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub use self::storage::Storage; pub use self::web::Server; mod build_queue; +pub mod cdn; mod config; mod context; pub mod db; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index bc4d0a568..1191be6d2 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -21,6 +21,7 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; +use tokio::runtime::Runtime; const MAX_CONCURRENT_UPLOADS: usize = 1000; @@ -113,14 +114,21 @@ pub struct Storage { } impl Storage { - pub fn new(pool: Pool, metrics: Arc, config: Arc) -> Result { + pub fn new( + pool: Pool, + metrics: Arc, + config: Arc, + runtime: Arc, + ) -> Result { Ok(Storage { config: config.clone(), backend: match config.storage_backend { StorageKind::Database => { StorageBackend::Database(DatabaseBackend::new(pool, metrics)) } - StorageKind::S3 => StorageBackend::S3(Box::new(S3Backend::new(metrics, &config)?)), + StorageKind::S3 => { + StorageBackend::S3(Box::new(S3Backend::new(metrics, &config, runtime)?)) + } }, }) } diff --git a/src/storage/s3.rs b/src/storage/s3.rs index 7a476fb27..c39d9c8ac 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -18,7 +18,7 @@ use tokio::runtime::Runtime; pub(super) struct S3Backend { client: Client, - runtime: Runtime, + runtime: Arc, bucket: String, metrics: Arc, #[cfg(test)] @@ -26,9 +26,11 @@ pub(super) struct S3Backend { } impl S3Backend { - pub(super) fn new(metrics: Arc, config: &Config) -> Result { - let runtime = Runtime::new()?; - + pub(super) fn new( + metrics: Arc, + config: &Config, + runtime: Arc, + ) -> Result { let shared_config = runtime.block_on(aws_config::load_from_env()); let mut config_builder = aws_sdk_s3::config::Builder::from(&shared_config) .retry_config(RetryConfig::new().with_max_attempts(3)) diff --git a/src/test/mod.rs b/src/test/mod.rs index aeb3baf9d..3cfb2c995 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -1,6 +1,7 @@ mod fakes; pub(crate) use self::fakes::FakeBuild; +use crate::cdn::CdnBackend; use crate::db::{Pool, PoolClient}; use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; @@ -17,6 +18,7 @@ use reqwest::{ Method, }; use std::{fs, net::SocketAddr, panic, sync::Arc, time::Duration}; +use tokio::runtime::Runtime; pub(crate) fn wrapper(f: impl FnOnce(&TestEnvironment) -> Result<()>) { let env = TestEnvironment::new(); @@ -113,7 +115,9 @@ pub(crate) struct TestEnvironment { config: OnceCell>, db: OnceCell, storage: OnceCell>, + cdn: OnceCell>, index: OnceCell>, + runtime: OnceCell>, metrics: OnceCell>, frontend: OnceCell, repository_stats_updater: OnceCell>, @@ -137,9 +141,11 @@ impl TestEnvironment { config: OnceCell::new(), db: OnceCell::new(), storage: OnceCell::new(), + cdn: OnceCell::new(), index: OnceCell::new(), metrics: OnceCell::new(), frontend: OnceCell::new(), + runtime: OnceCell::new(), repository_stats_updater: OnceCell::new(), } } @@ -206,6 +212,12 @@ impl TestEnvironment { .clone() } + pub(crate) fn cdn(&self) -> Arc { + self.cdn + .get_or_init(|| Arc::new(CdnBackend::new(&self.config(), &self.runtime()))) + .clone() + } + pub(crate) fn config(&self) -> Arc { self.config .get_or_init(|| Arc::new(self.base_config())) @@ -216,8 +228,13 @@ impl TestEnvironment { self.storage .get_or_init(|| { Arc::new( - Storage::new(self.db().pool(), self.metrics(), self.config()) - .expect("failed to initialize the storage"), + Storage::new( + self.db().pool(), + self.metrics(), + self.config(), + self.runtime(), + ) + .expect("failed to initialize the storage"), ) }) .clone() @@ -228,6 +245,11 @@ impl TestEnvironment { .get_or_init(|| Arc::new(Metrics::new().expect("failed to initialize the metrics"))) .clone() } + pub(crate) fn runtime(&self) -> Arc { + self.runtime + .get_or_init(|| Arc::new(Runtime::new().expect("failed to initialize runtime"))) + .clone() + } pub(crate) fn index(&self) -> Arc { self.index @@ -288,6 +310,10 @@ impl Context for TestEnvironment { Ok(TestEnvironment::storage(self)) } + fn cdn(&self) -> Result> { + Ok(TestEnvironment::cdn(self)) + } + fn pool(&self) -> Result { Ok(self.db().pool()) } @@ -303,6 +329,10 @@ impl Context for TestEnvironment { fn repository_stats_updater(&self) -> Result> { Ok(self.repository_stats_updater()) } + + fn runtime(&self) -> Result> { + Ok(self.runtime()) + } } pub(crate) struct TestDatabase { diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index c92e74b50..7da82ae5e 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -260,6 +260,7 @@ impl RustdocPage { let mut response = Response::with((Status::Ok, html)); response.headers.set(ContentType::html()); + if is_latest_url { response .headers @@ -888,7 +889,7 @@ mod test { } #[test] - fn cache_headers() { + fn cache_headers_on_version() { wrapper(|env| { env.override_config(|config| { config.cache_control_max_age = Some(600);