From 10d28824766da1804e0dcce236b70bbe0d38ec7d Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 28 Feb 2024 20:38:40 +0800 Subject: [PATCH 1/6] Use ST_EstimatedExtent for quick bounds calc --- martin/src/pg/query_tables.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 23a13c15a..ab7e89ab3 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -117,7 +117,8 @@ pub async fn table_to_query( BoundsCalcType::Skip => {} BoundsCalcType::Calc => { debug!("Computing {} table bounds for {id}", info.format_id()); - info.bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid).await?; + info.bounds = + calc_bounds(&pool, &schema, &table, &geometry_column, srid, false).await?; } BoundsCalcType::Quick => { debug!( @@ -125,7 +126,7 @@ pub async fn table_to_query( info.format_id(), DEFAULT_BOUNDS_TIMEOUT.as_secs() ); - let bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid); + let bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid, true); pin_mut!(bounds); if let Ok(bounds) = timeout(DEFAULT_BOUNDS_TIMEOUT, &mut bounds).await { info.bounds = bounds?; @@ -217,10 +218,12 @@ async fn calc_bounds( table: &str, geometry_column: &str, srid: i32, + is_quick: bool, ) -> PgResult> { - Ok(pool.get() - .await? - .query_one(&format!( + let sql = if is_quick { + format!("SELECT ST_EstimatedExtent('{schema}', '{table}', '{geometry_column}') as bounds") + } else { + format!( r#" WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}), {srid}) AS rb FROM {schema}.{table}) SELECT ST_Transform( @@ -232,7 +235,14 @@ SELECT ST_Transform( 4326 ) AS bounds FROM {schema}.{table}; - "#), &[]) + "# + ) + }; + + Ok(pool + .get() + .await? + .query_one(&sql, &[]) .await .map_err(|e| PostgresError(e, "querying table bounds"))? .get::<_, Option>("bounds") From a186362612466653c494c379452b13fe7bf342ff Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Tue, 12 Mar 2024 11:36:34 +0800 Subject: [PATCH 2/6] wip --- martin/src/pg/query_tables.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index ab7e89ab3..124c2267e 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -98,6 +98,7 @@ fn escape_with_alias(mapping: &HashMap, field: &str) -> String { } } +#[allow(clippy::too_many_lines)] /// Generate a query to fetch tiles from a table. /// The function is async because it may need to query the database for the table bounds (could be very slow). pub async fn table_to_query( @@ -117,8 +118,15 @@ pub async fn table_to_query( BoundsCalcType::Skip => {} BoundsCalcType::Calc => { debug!("Computing {} table bounds for {id}", info.format_id()); - info.bounds = - calc_bounds(&pool, &schema, &table, &geometry_column, srid, false).await?; + info.bounds = calc_bounds( + &pool, + &info.schema, + &info.table, + &info.geometry_column, + srid, + false, + ) + .await?; } BoundsCalcType::Quick => { debug!( @@ -126,7 +134,14 @@ pub async fn table_to_query( info.format_id(), DEFAULT_BOUNDS_TIMEOUT.as_secs() ); - let bounds = calc_bounds(&pool, &schema, &table, &geometry_column, srid, true); + let bounds = calc_bounds( + &pool, + &info.schema, + &info.table, + &info.geometry_column, + srid, + true, + ); pin_mut!(bounds); if let Ok(bounds) = timeout(DEFAULT_BOUNDS_TIMEOUT, &mut bounds).await { info.bounds = bounds?; @@ -221,8 +236,14 @@ async fn calc_bounds( is_quick: bool, ) -> PgResult> { let sql = if is_quick { - format!("SELECT ST_EstimatedExtent('{schema}', '{table}', '{geometry_column}') as bounds") + let schema = escape_literal(schema); + let table = escape_literal(table); + let geometry_column = escape_literal(geometry_column); + format!("SELECT ST_EstimatedExtent({schema}, {table}, {geometry_column}) as bounds") } else { + let schema = escape_identifier(schema); + let table = escape_identifier(table); + let geometry_column = escape_identifier(geometry_column); format!( r#" WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}), {srid}) AS rb FROM {schema}.{table}) From 9bd57983b69080e1656962f276e4028fb363f53b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sun, 17 Mar 2024 00:25:53 -0400 Subject: [PATCH 3/6] fix ST_EstimatedExtent computation --- martin/src/pg/query_tables.rs | 69 ++++++++++++------- tests/expected/configured/save_config.yaml | 24 +++---- .../expected/configured/tbl_comment_cfg.json | 8 +-- 3 files changed, 59 insertions(+), 42 deletions(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 124c2267e..7882b9d9a 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -108,9 +108,6 @@ pub async fn table_to_query( bounds_type: BoundsCalcType, max_feature_count: Option, ) -> PgResult<(String, PgSqlInfo, TableInfo)> { - let schema = escape_identifier(&info.schema); - let table = escape_identifier(&info.table); - let geometry_column = escape_identifier(&info.geometry_column); let srid = info.srid; if info.bounds.is_none() { @@ -200,6 +197,9 @@ pub async fn table_to_query( let limit_clause = max_feature_count.map_or(String::new(), |v| format!("LIMIT {v}")); let layer_id = escape_literal(info.layer_id.as_ref().unwrap_or(&id)); let clip_geom = info.clip_geom.unwrap_or(DEFAULT_CLIP_GEOM); + let schema = escape_identifier(&info.schema); + let table = escape_identifier(&info.table); + let geometry_column = escape_identifier(&info.geometry_column); let query = format!( r#" SELECT @@ -233,19 +233,28 @@ async fn calc_bounds( table: &str, geometry_column: &str, srid: i32, - is_quick: bool, + mut is_quick: bool, ) -> PgResult> { - let sql = if is_quick { - let schema = escape_literal(schema); - let table = escape_literal(table); - let geometry_column = escape_literal(geometry_column); - format!("SELECT ST_EstimatedExtent({schema}, {table}, {geometry_column}) as bounds") - } else { - let schema = escape_identifier(schema); - let table = escape_identifier(table); - let geometry_column = escape_identifier(geometry_column); - format!( - r#" + let schema = escape_identifier(schema); + let table = escape_identifier(table); + + let cn = pool.get().await?; + loop { + let query = if is_quick { + // This method is faster but less accurate, and can fail in a number of cases (returns NULL) + cn.query_one( + "SELECT ST_Transform(ST_SetSRID(ST_EstimatedExtent($1, $2, $3)::geometry, $4), 4326) as bounds", + &[ + &&schema[1..schema.len() - 1], + &&table[1..table.len() - 1], + &geometry_column, + &srid, + ], + ).await + } else { + let geometry_column = escape_identifier(geometry_column); + cn.query_one( + &format!(r#" WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}), {srid}) AS rb FROM {schema}.{table}) SELECT ST_Transform( CASE @@ -256,16 +265,24 @@ SELECT ST_Transform( 4326 ) AS bounds FROM {schema}.{table}; - "# - ) - }; + "#), + &[] + ).await + }; - Ok(pool - .get() - .await? - .query_one(&sql, &[]) - .await - .map_err(|e| PostgresError(e, "querying table bounds"))? - .get::<_, Option>("bounds") - .and_then(|p| polygon_to_bbox(&p))) + if let Some(bounds) = query + .map_err(|e| PostgresError(e, "querying table bounds"))? + .get::<_, Option>("bounds") + { + return Ok(polygon_to_bbox(&bounds)); + } + if is_quick { + // ST_EstimatedExtent failed probably because there is no index or statistics or if it's a view + // This can only happen once if we are in quick mode + is_quick = false; + warn!("ST_EstimatedExtent on {schema}.{table}.{geometry_column} failed, trying slower method to compute bounds"); + } else { + return Ok(None); + } + } } diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 308490c10..585ae2ad3 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -23,10 +23,10 @@ postgres: geometry_column: Geom id_column: giD bounds: - - -170.94984639004662 - - -84.20025580733805 - - 167.70892858284475 - - 74.23573284753762 + - -170.94985961914062 + - -84.20025634765625 + - 167.7089385986328 + - 74.23573303222656 geometry_type: POINT properties: taBLe: text @@ -37,10 +37,10 @@ postgres: geometry_column: geom id_column: feat_id bounds: - - -166.87107126230424 - - -53.44747249115674 - - 168.14061220360549 - - 84.22411861475385 + - -166.87107849121094 + - -53.44747543334961 + - 168.140625 + - 84.22412109375 extent: 9000 buffer: 3 clip_geom: false @@ -54,10 +54,10 @@ postgres: geometry_column: geom id_column: big_feat_id bounds: - - -174.89475564568033 - - -77.2579745396886 - - 174.72753224514435 - - 73.80785950599903 + - -174.89476013183594 + - -77.25798034667969 + - 174.7275390625 + - 73.807861328125 extent: 9000 buffer: 3 clip_geom: false diff --git a/tests/expected/configured/tbl_comment_cfg.json b/tests/expected/configured/tbl_comment_cfg.json index 167c242dc..6b3644f7e 100644 --- a/tests/expected/configured/tbl_comment_cfg.json +++ b/tests/expected/configured/tbl_comment_cfg.json @@ -13,10 +13,10 @@ } ], "bounds": [ - -170.94984639004662, - -84.20025580733805, - 167.70892858284475, - 74.23573284753762 + -170.94985961914062, + -84.20025634765625, + 167.7089385986328, + 74.23573303222656 ], "description": "a description from comment on table", "name": "MixPoints" From 3cdd8b58564e28f6c1c8c73c7be97321fc553b48 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Wed, 5 Mar 2025 22:49:41 +0100 Subject: [PATCH 4/6] Update martin/src/pg/query_tables.rs Co-authored-by: Yuri Astrakhan --- martin/src/pg/query_tables.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 96b2bb796..00b9c5f5d 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -118,9 +118,7 @@ pub async fn table_to_query( debug!("Computing {} table bounds for {id}", info.format_id()); info.bounds = calc_bounds( &pool, - &info.schema, - &info.table, - &info.geometry_column, + &info srid, false, ) From daae7f2023a76735205a2ae0c4f2544e9afdc01d Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Thu, 6 Mar 2025 02:02:56 +0100 Subject: [PATCH 5/6] Revert "Update martin/src/pg/query_tables.rs" This reverts commit 3cdd8b58564e28f6c1c8c73c7be97321fc553b48. --- martin/src/pg/query_tables.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 00b9c5f5d..96b2bb796 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -118,7 +118,9 @@ pub async fn table_to_query( debug!("Computing {} table bounds for {id}", info.format_id()); info.bounds = calc_bounds( &pool, - &info + &info.schema, + &info.table, + &info.geometry_column, srid, false, ) From 9ceac57a452a25f6c4a7baa01ed61cbce6219e56 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 6 Mar 2025 13:59:43 -0500 Subject: [PATCH 6/6] Optimize quick bounds calc a bit --- martin/src/pg/query_tables.rs | 41 ++++++++++++----------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 96b2bb796..376dd8119 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -116,15 +116,7 @@ pub async fn table_to_query( BoundsCalcType::Skip => {} BoundsCalcType::Calc => { debug!("Computing {} table bounds for {id}", info.format_id()); - info.bounds = calc_bounds( - &pool, - &info.schema, - &info.table, - &info.geometry_column, - srid, - false, - ) - .await?; + info.bounds = calc_bounds(&pool, &info, srid, false).await?; } BoundsCalcType::Quick => { debug!( @@ -132,16 +124,13 @@ pub async fn table_to_query( info.format_id(), DEFAULT_BOUNDS_TIMEOUT.as_secs() ); - let bounds = calc_bounds( - &pool, - &info.schema, - &info.table, - &info.geometry_column, - srid, - true, - ); - pin_mut!(bounds); - if let Ok(bounds) = timeout(DEFAULT_BOUNDS_TIMEOUT, &mut bounds).await { + let bounds = { + let bounds = calc_bounds(&pool, &info, srid, true); + pin_mut!(bounds); + timeout(DEFAULT_BOUNDS_TIMEOUT, &mut bounds).await + }; + + if let Ok(bounds) = bounds { info.bounds = bounds?; } else { warn!( @@ -230,14 +219,12 @@ FROM ( /// Compute the bounds of a table. This could be slow if the table is large or has no geo index. async fn calc_bounds( pool: &PgPool, - schema: &str, - table: &str, - geometry_column: &str, + info: &TableInfo, srid: i32, mut is_quick: bool, ) -> PgResult> { - let schema = escape_identifier(schema); - let table = escape_identifier(table); + let schema = escape_identifier(&info.schema); + let table = escape_identifier(&info.table); let cn = pool.get().await?; loop { @@ -248,12 +235,12 @@ async fn calc_bounds( &[ &&schema[1..schema.len() - 1], &&table[1..table.len() - 1], - &geometry_column, + &info.geometry_column, &srid, ], ).await } else { - let geometry_column = escape_identifier(geometry_column); + let geometry_column = escape_identifier(&info.geometry_column); cn.query_one( &format!(r" WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}::geometry), {srid}) AS rb FROM {schema}.{table}) @@ -280,7 +267,7 @@ FROM {schema}.{table};"), // ST_EstimatedExtent failed probably because there is no index or statistics or if it's a view // This can only happen once if we are in quick mode is_quick = false; - warn!("ST_EstimatedExtent on {schema}.{table}.{geometry_column} failed, trying slower method to compute bounds"); + warn!("ST_EstimatedExtent on {schema}.{table}.{} failed, trying slower method to compute bounds", info.geometry_column); } else { return Ok(None); }