diff --git a/Cargo.lock b/Cargo.lock index a2d4c7fd7..76cbd659e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1425,7 +1425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2409,7 +2409,7 @@ checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi 0.4.0", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2608,7 +2608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2766,6 +2766,7 @@ dependencies = [ "tilejson", "tokio", "tokio-postgres-rustls", + "tokio-retry2", "url", ] @@ -3725,7 +3726,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4110,7 +4111,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5019,7 +5020,7 @@ dependencies = [ "getrandom 0.2.15", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5316,6 +5317,17 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "tokio-retry2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1264d076dd34560544a2799e40e457bd07c43d30f4a845686b031bcd8455c84f" +dependencies = [ + "pin-project", + "rand 0.9.0", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.1" @@ -5912,7 +5924,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0acf6e357..0860ef397 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ tokio = { version = "1", features = ["macros"] } tokio-postgres-rustls = "0.13" url = "2.5" xxhash-rust = { version = "0.8", features = ["xxh3"] } +tokio-retry2 = { version = "0.5.7", features = ["jitter"] } [profile.dev.package] # See https://github.com/launchbadge/sqlx#compile-time-verification diff --git a/martin/Cargo.toml b/martin/Cargo.toml index d756837e8..22fc81ed9 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -116,6 +116,7 @@ tilejson.workspace = true tokio = { workspace = true, features = ["io-std"] } tokio-postgres-rustls = { workspace = true, optional = true } url.workspace = true +tokio-retry2.workspace = true [build-dependencies] static-files = { workspace = true, optional = true } diff --git a/martin/src/pg/config.rs b/martin/src/pg/config.rs index b25f5e7ec..31b81c654 100644 --- a/martin/src/pg/config.rs +++ b/martin/src/pg/config.rs @@ -17,6 +17,8 @@ use crate::pg::PgResult; use crate::source::TileInfoSources; use crate::utils::{IdResolver, OptBoolObj, OptOneMany}; use crate::MartinResult; +use tokio_retry2::strategy::{jitter_range, FixedInterval}; +use tokio_retry2::{Retry, RetryError}; pub trait PgInfo { fn format_id(&self) -> String; @@ -133,7 +135,18 @@ impl PgConfig { } pub async fn resolve(&mut self, id_resolver: IdResolver) -> MartinResult { - let pg = PgBuilder::new(self, id_resolver).await?; + // waits a maximum of 20s..25s before failing permanently + let retry_strategy = FixedInterval::from_millis(500) + .map(jitter_range(0.8, 1.0)) // jitter 400ms..500ms => no thundering herd + .take(50); + + let pg = Retry::spawn(retry_strategy, || async { + PgBuilder::new(self, id_resolver.clone()) + .await + .map_err(RetryError::transient) + }) + .await?; + let inst_tables = on_slow( pg.instantiate_tables(), // warn only if default bounds timeout has already passed