From 9494cb0ddd937dc59924e3efba3032850f00bcfd Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sat, 22 Feb 2025 15:08:12 +0100 Subject: [PATCH 01/11] new function: fetch_with_meta closes https://github.com/sqlpage/SQLPage/issues/792 --- .../sqlpage/migrations/58_fetch_with_meta.sql | 127 +++++++++++++++++ .../database/sqlpage_functions/functions.rs | 134 +++++++++++++++--- 2 files changed, 238 insertions(+), 23 deletions(-) create mode 100644 examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql diff --git a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql new file mode 100644 index 00000000..bc5d9d5d --- /dev/null +++ b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql @@ -0,0 +1,127 @@ +INSERT INTO sqlpage_functions ( + "name", + "introduced_in_version", + "icon", + "description_md" + ) +VALUES ( + 'fetch_with_meta', + '0.34.0', + 'transfer-vertical', + 'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body. + +This function is similar to `fetch`, but returns a JSON object containing detailed information about the response. +The returned object has the following structure: +```json +{ + "status": 200, + "headers": { + "content-type": "application/json", + "content-length": "1234", + ... + }, + "body": "response body content", + "error": "error message if any" +} +``` + +If the request fails or encounters an error (e.g., network issues, invalid UTF-8 response), instead of throwing an error, +the function returns a JSON object with an "error" field containing the error message. + +### Example: Basic Usage + +```sql +-- Make a request and get detailed response information +set response = sqlpage.fetch_with_meta(''https://api.example.com/data''); + +-- Check if the request was successful +select case + when json_extract($response, ''$.error'') is not null then + ''Request failed: '' || json_extract($response, ''$.error'') + when json_extract($response, ''$.status'') != 200 then + ''Request returned status '' || json_extract($response, ''$.status'') + else + ''Request successful'' +end as message; + +-- Display response headers +select ''code'' as component, + ''Response Headers'' as title, + ''json'' as language, + json_extract($response, ''$.headers'') as contents; +``` + +### Example: Error Handling with Retries + +```sql +-- Function to make a request with retries +create temp table if not exists make_request as +with recursive retry(attempt, response) as ( + -- First attempt + select 1 as attempt, + sqlpage.fetch_with_meta(''https://api.example.com/data'') as response + union all + -- Retry up to 3 times if we get a 5xx error + select attempt + 1, + sqlpage.fetch_with_meta(''https://api.example.com/data'') + from retry + where attempt < 3 + and ( + json_extract(response, ''$.error'') is not null + or cast(json_extract(response, ''$.status'') as integer) >= 500 + ) +) +select response +from retry +where json_extract(response, ''$.error'') is null + and cast(json_extract(response, ''$.status'') as integer) < 500 +limit 1; + +-- Use the response +select case + when $response is null then ''All retry attempts failed'' + else ''Request succeeded after retries'' +end as message; +``` + +### Example: Advanced Request with Authentication + +```sql +set request = json_object( + ''method'', ''POST'', + ''url'', ''https://api.example.com/data'', + ''headers'', json_object( + ''Content-Type'', ''application/json'', + ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'') + ), + ''body'', json_object( + ''key'', ''value'' + ) +); +set response = sqlpage.fetch_with_meta($request); + +-- Check response content type +select case + when json_extract($response, ''$.headers.content-type'') like ''%application/json%'' + then json_extract($response, ''$.body'') + else null +end as json_response; +``` + +The function accepts the same parameters as the `fetch` function. See the documentation of `fetch` for more details about the available parameters.' + ); + +INSERT INTO sqlpage_function_parameters ( + "function", + "index", + "name", + "description_md", + "type" + ) +VALUES ( + 'fetch_with_meta', + 1, + 'url', + 'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.', + 'TEXT' + ); \ No newline at end of file diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index 0f32f4cb..ad9f790d 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -24,6 +24,7 @@ super::function_definition_macro::sqlpage_functions! { exec((&RequestInfo), program_name: Cow, args: Vec>); fetch((&RequestInfo), http_request: SqlPageFunctionParam>); + fetch_with_meta((&RequestInfo), http_request: SqlPageFunctionParam>); hash_password(password: Option); header((&RequestInfo), name: Cow); @@ -135,16 +136,13 @@ async fn exec<'a>( Ok(String::from_utf8_lossy(&res.stdout).into_owned()) } -async fn fetch( - request: &RequestInfo, - http_request: super::http_fetch_request::HttpFetchRequest<'_>, -) -> anyhow::Result { +fn build_request<'a>( + client: &'a awc::Client, + http_request: &'a super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { use awc::http::Method; - let client = make_http_client(&request.app_state.config) - .with_context(|| "Unable to create an HTTP client")?; - - let method = if let Some(method) = http_request.method { - Method::from_str(&method).with_context(|| format!("Invalid HTTP method: {method}"))? + let method = if let Some(method) = &http_request.method { + Method::from_str(method).with_context(|| format!("Invalid HTTP method: {method}"))? } else { Method::GET }; @@ -152,36 +150,56 @@ async fn fetch( if let Some(timeout) = http_request.timeout_ms { req = req.timeout(core::time::Duration::from_millis(timeout)); } - for (k, v) in http_request.headers { + for (k, v) in &http_request.headers { req = req.insert_header((k.as_ref(), v.as_ref())); } - if let Some(username) = http_request.username { - let password = http_request.password.unwrap_or_default(); + if let Some(username) = &http_request.username { + let password = http_request.password.as_deref().unwrap_or_default(); req = req.basic_auth(username, password); } + Ok(req) +} + +fn prepare_request_body( + body: &serde_json::value::RawValue, + mut req: awc::ClientRequest, +) -> anyhow::Result<(String, awc::ClientRequest)> { + let val = body.get(); + let body_str = if val.starts_with('"') { + serde_json::from_str::<'_, String>(val).with_context(|| { + format!("Invalid JSON string in the body of the HTTP request: {val}") + })? + } else { + req = req.content_type("application/json"); + val.to_owned() + }; + Ok((body_str, req)) +} + +async fn fetch( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + log::info!("Fetching {}", http_request.url); - let mut response = if let Some(body) = http_request.body { - let val = body.get(); - // The body can be either json, or a string representing a raw body - let body = if val.starts_with('"') { - serde_json::from_str::<'_, String>(val).with_context(|| { - format!("Invalid JSON string in the body of the HTTP request: {val}") - })? - } else { - req = req.content_type("application/json"); - val.to_owned() - }; + let mut response = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; req.send_body(body) } else { req.send() } .await .map_err(|e| anyhow!("Unable to fetch {}: {e}", http_request.url))?; + log::debug!( "Finished fetching {}. Status: {}", http_request.url, response.status() ); + let body = response .body() .await @@ -199,6 +217,76 @@ async fn fetch( Ok(response_str) } +async fn fetch_with_meta( + request: &RequestInfo, + http_request: super::http_fetch_request::HttpFetchRequest<'_>, +) -> anyhow::Result { + let client = make_http_client(&request.app_state.config) + .with_context(|| "Unable to create an HTTP client")?; + let req = build_request(&client, &http_request)?; + + log::info!("Fetching {} with metadata", http_request.url); + let response_result = if let Some(body) = &http_request.body { + let (body, req) = prepare_request_body(body, req)?; + req.send_body(body).await + } else { + req.send().await + }; + + let mut response_info = serde_json::Map::new(); + match response_result { + Ok(mut response) => { + response_info.insert("status".to_string(), response.status().as_u16().into()); + + let mut headers = serde_json::Map::new(); + for (name, value) in response.headers().iter() { + if let Ok(value_str) = value.to_str() { + headers.insert(name.to_string(), value_str.into()); + } + } + response_info.insert("headers".to_string(), headers.clone().into()); + + match response.body().await { + Ok(body) => { + let body_bytes = body.to_vec(); + let content_type = headers.get("content-type") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + + let body_value = if content_type.contains("application/json") { + match serde_json::from_slice(&body_bytes) { + Ok(json_value) => json_value, + Err(_) => serde_json::Value::String(String::from_utf8_lossy(&body_bytes).into_owned()), + } + } else { + match String::from_utf8(body_bytes.clone()) { + Ok(text) => serde_json::Value::String(text), + Err(_) => { + let mut base64_string = String::new(); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + &body_bytes, + &mut base64_string, + ); + serde_json::Value::String(base64_string) + } + } + }; + response_info.insert("body".to_string(), body_value); + } + Err(e) => { + response_info.insert("error".to_string(), format!("Failed to read response body: {}", e).into()); + } + } + } + Err(e) => { + response_info.insert("error".to_string(), format!("Request failed: {}", e).into()); + } + } + + Ok(serde_json::to_string(&response_info)?) +} + static NATIVE_CERTS: OnceLock> = OnceLock::new(); fn make_http_client(config: &crate::app_config::AppConfig) -> anyhow::Result { From 009e8810250c78fadc3dd02e73b45851e26eee55 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sun, 23 Feb 2025 01:07:14 +0100 Subject: [PATCH 02/11] fmt --- .../database/sqlpage_functions/functions.rs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index ad9f790d..c95b79fc 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -183,7 +183,7 @@ async fn fetch( let client = make_http_client(&request.app_state.config) .with_context(|| "Unable to create an HTTP client")?; let req = build_request(&client, &http_request)?; - + log::info!("Fetching {}", http_request.url); let mut response = if let Some(body) = &http_request.body { let (body, req) = prepare_request_body(body, req)?; @@ -193,13 +193,13 @@ async fn fetch( } .await .map_err(|e| anyhow!("Unable to fetch {}: {e}", http_request.url))?; - + log::debug!( "Finished fetching {}. Status: {}", http_request.url, response.status() ); - + let body = response .body() .await @@ -224,7 +224,7 @@ async fn fetch_with_meta( let client = make_http_client(&request.app_state.config) .with_context(|| "Unable to create an HTTP client")?; let req = build_request(&client, &http_request)?; - + log::info!("Fetching {} with metadata", http_request.url); let response_result = if let Some(body) = &http_request.body { let (body, req) = prepare_request_body(body, req)?; @@ -237,7 +237,7 @@ async fn fetch_with_meta( match response_result { Ok(mut response) => { response_info.insert("status".to_string(), response.status().as_u16().into()); - + let mut headers = serde_json::Map::new(); for (name, value) in response.headers().iter() { if let Ok(value_str) = value.to_str() { @@ -249,14 +249,17 @@ async fn fetch_with_meta( match response.body().await { Ok(body) => { let body_bytes = body.to_vec(); - let content_type = headers.get("content-type") + let content_type = headers + .get("content-type") .and_then(|v| v.as_str()) .unwrap_or_default(); let body_value = if content_type.contains("application/json") { match serde_json::from_slice(&body_bytes) { Ok(json_value) => json_value, - Err(_) => serde_json::Value::String(String::from_utf8_lossy(&body_bytes).into_owned()), + Err(_) => serde_json::Value::String( + String::from_utf8_lossy(&body_bytes).into_owned(), + ), } } else { match String::from_utf8(body_bytes.clone()) { @@ -275,7 +278,10 @@ async fn fetch_with_meta( response_info.insert("body".to_string(), body_value); } Err(e) => { - response_info.insert("error".to_string(), format!("Failed to read response body: {}", e).into()); + response_info.insert( + "error".to_string(), + format!("Failed to read response body: {}", e).into(), + ); } } } From b5759d5209226543e45ede9b23209ea2d46fc6d2 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sun, 23 Feb 2025 01:07:52 +0100 Subject: [PATCH 03/11] clippy auto --- .../database/sqlpage_functions/functions.rs | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index c95b79fc..ddfdf809 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -239,7 +239,7 @@ async fn fetch_with_meta( response_info.insert("status".to_string(), response.status().as_u16().into()); let mut headers = serde_json::Map::new(); - for (name, value) in response.headers().iter() { + for (name, value) in response.headers() { if let Ok(value_str) = value.to_str() { headers.insert(name.to_string(), value_str.into()); } @@ -261,32 +261,27 @@ async fn fetch_with_meta( String::from_utf8_lossy(&body_bytes).into_owned(), ), } - } else { - match String::from_utf8(body_bytes.clone()) { - Ok(text) => serde_json::Value::String(text), - Err(_) => { - let mut base64_string = String::new(); - base64::Engine::encode_string( - &base64::engine::general_purpose::STANDARD, - &body_bytes, - &mut base64_string, - ); - serde_json::Value::String(base64_string) - } - } + } else if let Ok(text) = String::from_utf8(body_bytes.clone()) { serde_json::Value::String(text) } else { + let mut base64_string = String::new(); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + &body_bytes, + &mut base64_string, + ); + serde_json::Value::String(base64_string) }; response_info.insert("body".to_string(), body_value); } Err(e) => { response_info.insert( "error".to_string(), - format!("Failed to read response body: {}", e).into(), + format!("Failed to read response body: {e}").into(), ); } } } Err(e) => { - response_info.insert("error".to_string(), format!("Request failed: {}", e).into()); + response_info.insert("error".to_string(), format!("Request failed: {e}").into()); } } From a1b2d418c516f4540004c19e11d79521a522b581 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Sun, 23 Feb 2025 01:20:43 +0100 Subject: [PATCH 04/11] test fetch_with_meta --- .../database/sqlpage_functions/functions.rs | 4 +++- .../it_works_fetch_with_meta_simple.sql | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/sql_test_files/it_works_fetch_with_meta_simple.sql diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index ddfdf809..b395a49e 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -261,7 +261,9 @@ async fn fetch_with_meta( String::from_utf8_lossy(&body_bytes).into_owned(), ), } - } else if let Ok(text) = String::from_utf8(body_bytes.clone()) { serde_json::Value::String(text) } else { + } else if let Ok(text) = String::from_utf8(body_bytes.clone()) { + serde_json::Value::String(text) + } else { let mut base64_string = String::new(); base64::Engine::encode_string( &base64::engine::general_purpose::STANDARD, diff --git a/tests/sql_test_files/it_works_fetch_with_meta_simple.sql b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql new file mode 100644 index 00000000..6197e24b --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql @@ -0,0 +1,15 @@ +set res = sqlpage.fetch_with_meta('{ + "method": "PUT", + "url": "http://localhost:62802/hello_world", + "headers": { + "user-agent": "myself" + } +}'); +select 'text' as component, + case + when json_extract($res, '$.status') = 200 + and cast(json_extract($res, '$.headers.content-length') as int) > 100 + and json_extract($res, '$.body') like 'PUT /hello_world%' + then 'It works !' + else 'It failed! Got: ' || $res + end as contents; \ No newline at end of file From 235ce117b1810c18ee28388f17c3a2e864912444 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 18:33:52 +0100 Subject: [PATCH 05/11] fix tests --- Cargo.lock | 24 +++++++++---------- tests/index.rs | 7 ++++-- .../it_works_fetch_with_meta_simple.sql | 8 +++---- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48efda03..2bac7f70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1254,9 +1254,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "encoding_rs" @@ -1346,9 +1346,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -2049,9 +2049,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libflate" @@ -2616,7 +2616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", + "rand_core 0.9.2", "zerocopy 0.8.20", ] @@ -2637,7 +2637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.2", ] [[package]] @@ -2651,9 +2651,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "7a509b1a2ffbe92afab0e55c8fd99dea1c280e8171bd2d88682bb20bc41cbc2c" dependencies = [ "getrandom 0.3.1", "zerocopy 0.8.20", @@ -2728,9 +2728,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "ring" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34b5020fcdea098ef7d95e9f89ec15952123a4a039badd09fabebe9e963e839" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" dependencies = [ "cc", "cfg-if", diff --git a/tests/index.rs b/tests/index.rs index 590eb520..1f1dee8b 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -136,7 +136,10 @@ fn start_echo_server() -> ServerHandle { } f.push(b'|'); f.extend_from_slice(&r.extract::().await?); - let resp = HttpResponse::Ok().body(f); + let resp = HttpResponse::Ok() + .insert_header((header::DATE, "Mon, 24 Feb 2025 12:00:00 GMT")) + .insert_header((header::CONTENT_TYPE, "text/plain")) + .body(f); Ok(r.into_response(resp)) } let server = actix_web::HttpServer::new(move || { @@ -201,7 +204,7 @@ async fn test_files() { ); assert!( !lowercase_body.contains("error"), - "{body}\nexpected to not contain: error" + "{req_str}\n{body}\nexpected to not contain: error" ); } else if stem.starts_with("error_") { let rest = stem.strip_prefix("error_").unwrap(); diff --git a/tests/sql_test_files/it_works_fetch_with_meta_simple.sql b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql index 6197e24b..d4d8bfaa 100644 --- a/tests/sql_test_files/it_works_fetch_with_meta_simple.sql +++ b/tests/sql_test_files/it_works_fetch_with_meta_simple.sql @@ -5,11 +5,9 @@ set res = sqlpage.fetch_with_meta('{ "user-agent": "myself" } }'); + select 'text' as component, case - when json_extract($res, '$.status') = 200 - and cast(json_extract($res, '$.headers.content-length') as int) > 100 - and json_extract($res, '$.body') like 'PUT /hello_world%' - then 'It works !' - else 'It failed! Got: ' || $res + when $res LIKE '%"status":200%' AND $res LIKE '%"headers":{%' AND $res LIKE '%"body":"%' then 'It works !' + else 'Error! Got: ' || $res end as contents; \ No newline at end of file From bc82eae60d8ec94905d4b8583aa771f6c9bdf6c0 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 18:34:07 +0100 Subject: [PATCH 06/11] update deps --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bac7f70..e5053a03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,9 +830,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" dependencies = [ "clap_builder", "clap_derive", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.30" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" dependencies = [ "anstream", "anstyle", From 82f15ef49d9d58f49a945d4d2f371578bd3042df Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 18:55:04 +0100 Subject: [PATCH 07/11] retry failed deps downloads --- build.rs | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/build.rs b/build.rs index 3c2bdeea..890b0830 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,5 @@ use actix_rt::spawn; +use actix_rt::time::sleep; use libflate::gzip; use std::collections::hash_map::DefaultHasher; use std::fs::File; @@ -99,21 +100,37 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write) } async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) { - let mut resp = client.get(url).send().await.unwrap_or_else(|err| { - let path = make_url_path(url); - panic!( - "We need to download external frontend dependencies to build the static frontend. \n\ - Could not download static asset. You can manually download the file with: \n\ - curl {url:?} > {path:?} \n\ - {err}" - ) - }); - if resp.status() != 200 { - panic!("Received {} status code from {}", resp.status(), url); + let mut attempt = 1; + let max_attempts = 2; + + loop { + match client.get(url).send().await { + Ok(mut resp) => { + if resp.status() != 200 { + panic!("Received {} status code from {}", resp.status(), url); + } + let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); + std::fs::write(path, &bytes) + .expect("Failed to write external frontend dependency to local file"); + break; + } + Err(err) => { + if attempt >= max_attempts { + let path = make_url_path(url); + panic!( + "We need to download external frontend dependencies to build the static frontend. \n\ + Could not download static asset after {} attempts. You can manually download the file with: \n\ + curl {url:?} > {path:?} \n\ + {err}", + max_attempts + ); + } + sleep(Duration::from_secs(1)).await; + println!("cargo:warning=Retrying download of {url} after {err}."); + attempt += 1; + } + } } - let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); - std::fs::write(path, &bytes) - .expect("Failed to write external frontend dependency to local file"); } // Given a filename, creates a new unique filename based on the file contents From 8faa80d642d8d476de3d8b11a2006a44a2726f25 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 21:39:53 +0100 Subject: [PATCH 08/11] better json serializing --- build.rs | 2 +- .../sqlpage/migrations/40_fetch.sql | 6 ++ .../sqlpage/migrations/58_fetch_with_meta.sql | 79 +++++------------ .../database/sqlpage_functions/functions.rs | 85 ++++++++++--------- 4 files changed, 73 insertions(+), 99 deletions(-) diff --git a/build.rs b/build.rs index 890b0830..8c3ae77f 100644 --- a/build.rs +++ b/build.rs @@ -102,7 +102,7 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write) async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) { let mut attempt = 1; let max_attempts = 2; - + loop { match client.get(url).send().await { Ok(mut resp) => { diff --git a/examples/official-site/sqlpage/migrations/40_fetch.sql b/examples/official-site/sqlpage/migrations/40_fetch.sql index cad1af2e..002bc0c9 100644 --- a/examples/official-site/sqlpage/migrations/40_fetch.sql +++ b/examples/official-site/sqlpage/migrations/40_fetch.sql @@ -88,6 +88,12 @@ The fetch function accepts either a URL string, or a JSON object with the follow - `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0. - `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0. +# Error handling and reading response headers + +If the request fails, this function throws an error, that will be displayed to the user. +The response headers are not available for inspection. + +If you need to handle errors or inspect the response headers, use [`sqlpage.fetch_with_meta`](?function=fetch_with_meta). ' ); INSERT INTO sqlpage_function_parameters ( diff --git a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql index bc5d9d5d..296071d2 100644 --- a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql +++ b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql @@ -10,17 +10,16 @@ VALUES ( 'transfer-vertical', 'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body. -This function is similar to `fetch`, but returns a JSON object containing detailed information about the response. +This function is similar to [`fetch`](?function=fetch), but returns a JSON object containing detailed information about the response. The returned object has the following structure: ```json { "status": 200, "headers": { - "content-type": "application/json", - "content-length": "1234", - ... + "content-type": "text/html", + "content-length": "1234" }, - "body": "response body content", + "body": "a string, or a json object, depending on the content type", "error": "error message if any" } ``` @@ -32,56 +31,20 @@ the function returns a JSON object with an "error" field containing the error me ```sql -- Make a request and get detailed response information -set response = sqlpage.fetch_with_meta(''https://api.example.com/data''); +set response = sqlpage.fetch_with_meta(''https://pokeapi.co/api/v2/pokemon/ditto''); --- Check if the request was successful -select case - when json_extract($response, ''$.error'') is not null then - ''Request failed: '' || json_extract($response, ''$.error'') - when json_extract($response, ''$.status'') != 200 then - ''Request returned status '' || json_extract($response, ''$.status'') - else - ''Request successful'' -end as message; +-- redirect the user to an error page if the request failed +select ''redirect'' as component, ''error.sql'' as url +where + json_extract($response, ''$.error'') is not null + or json_extract($response, ''$.status'') != 200; --- Display response headers -select ''code'' as component, - ''Response Headers'' as title, - ''json'' as language, - json_extract($response, ''$.headers'') as contents; -``` - -### Example: Error Handling with Retries - -```sql --- Function to make a request with retries -create temp table if not exists make_request as -with recursive retry(attempt, response) as ( - -- First attempt - select 1 as attempt, - sqlpage.fetch_with_meta(''https://api.example.com/data'') as response - union all - -- Retry up to 3 times if we get a 5xx error - select attempt + 1, - sqlpage.fetch_with_meta(''https://api.example.com/data'') - from retry - where attempt < 3 - and ( - json_extract(response, ''$.error'') is not null - or cast(json_extract(response, ''$.status'') as integer) >= 500 - ) -) -select response -from retry -where json_extract(response, ''$.error'') is null - and cast(json_extract(response, ''$.status'') as integer) < 500 -limit 1; - --- Use the response -select case - when $response is null then ''All retry attempts failed'' - else ''Request succeeded after retries'' -end as message; +-- Extract data from the response json body +select ''card'' as component; +select + json_extract($response, ''$.body.name'') as title, + json_extract($response, ''$.body.abilities[0].ability.name'') as description +from $response; ``` ### Example: Advanced Request with Authentication @@ -89,7 +52,7 @@ end as message; ```sql set request = json_object( ''method'', ''POST'', - ''url'', ''https://api.example.com/data'', + ''url'', ''https://sqlpage.free.beeceptor.com'', ''headers'', json_object( ''Content-Type'', ''application/json'', ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'') @@ -101,14 +64,10 @@ set request = json_object( set response = sqlpage.fetch_with_meta($request); -- Check response content type -select case - when json_extract($response, ''$.headers.content-type'') like ''%application/json%'' - then json_extract($response, ''$.body'') - else null -end as json_response; +select ''debug'' as component, $response as response; ``` -The function accepts the same parameters as the `fetch` function. See the documentation of `fetch` for more details about the available parameters.' +The function accepts the same parameters as the [`fetch` function](?function=fetch).' ); INSERT INTO sqlpage_function_parameters ( diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index b395a49e..a790fd03 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -221,6 +221,8 @@ async fn fetch_with_meta( request: &RequestInfo, http_request: super::http_fetch_request::HttpFetchRequest<'_>, ) -> anyhow::Result { + use serde::{ser::SerializeMap, Serializer}; + let client = make_http_client(&request.app_state.config) .with_context(|| "Unable to create an HTTP client")?; let req = build_request(&client, &http_request)?; @@ -233,61 +235,68 @@ async fn fetch_with_meta( req.send().await }; - let mut response_info = serde_json::Map::new(); + let mut resp_str = Vec::new(); + let mut encoder = serde_json::Serializer::new(&mut resp_str); + let mut obj = encoder.serialize_map(Some(3))?; match response_result { Ok(mut response) => { - response_info.insert("status".to_string(), response.status().as_u16().into()); + obj.serialize_entry("status", &response.status().as_u16())?; - let mut headers = serde_json::Map::new(); - for (name, value) in response.headers() { - if let Ok(value_str) = value.to_str() { - headers.insert(name.to_string(), value_str.into()); - } - } - response_info.insert("headers".to_string(), headers.clone().into()); + let headers = response.headers(); + + let is_json = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .starts_with("application/json"); + + obj.serialize_entry( + "headers", + &headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default())) + .collect::>(), + )?; match response.body().await { Ok(body) => { let body_bytes = body.to_vec(); - let content_type = headers - .get("content-type") - .and_then(|v| v.as_str()) - .unwrap_or_default(); - - let body_value = if content_type.contains("application/json") { - match serde_json::from_slice(&body_bytes) { - Ok(json_value) => json_value, - Err(_) => serde_json::Value::String( - String::from_utf8_lossy(&body_bytes).into_owned(), - ), + let body_str = String::from_utf8(body_bytes); + + match body_str { + Ok(body_str) if is_json => { + obj.serialize_entry( + "body", + &serde_json::value::RawValue::from_string(body_str)?, + )?; + } + Ok(body_str) => { + obj.serialize_entry("body", &body_str)?; + } + Err(utf8_err) => { + let mut base64_string = String::new(); + base64::Engine::encode_string( + &base64::engine::general_purpose::STANDARD, + utf8_err.as_bytes(), + &mut base64_string, + ); + obj.serialize_entry("body", &base64_string)?; } - } else if let Ok(text) = String::from_utf8(body_bytes.clone()) { - serde_json::Value::String(text) - } else { - let mut base64_string = String::new(); - base64::Engine::encode_string( - &base64::engine::general_purpose::STANDARD, - &body_bytes, - &mut base64_string, - ); - serde_json::Value::String(base64_string) - }; - response_info.insert("body".to_string(), body_value); + } } Err(e) => { - response_info.insert( - "error".to_string(), - format!("Failed to read response body: {e}").into(), - ); + obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?; } } } Err(e) => { - response_info.insert("error".to_string(), format!("Request failed: {e}").into()); + obj.serialize_entry("error", &format!("Request failed: {e}"))?; } } - Ok(serde_json::to_string(&response_info)?) + obj.end()?; + let return_value = String::from_utf8(resp_str)?; + Ok(return_value) } static NATIVE_CERTS: OnceLock> = OnceLock::new(); From 910610d549e11b6e1037a7a79b20179e49922ff8 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 21:43:18 +0100 Subject: [PATCH 09/11] test fetch_with_meta error handling --- tests/sql_test_files/it_works_fetch_with_meta_error.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/sql_test_files/it_works_fetch_with_meta_error.sql diff --git a/tests/sql_test_files/it_works_fetch_with_meta_error.sql b/tests/sql_test_files/it_works_fetch_with_meta_error.sql new file mode 100644 index 00000000..c8430863 --- /dev/null +++ b/tests/sql_test_files/it_works_fetch_with_meta_error.sql @@ -0,0 +1,7 @@ +set res = sqlpage.fetch_with_meta('http://not-a-real-url'); + +select 'text' as component, + case + when json_extract($res, '$.error') LIKE 'Request failed%' then 'It works !' + else 'Error! Got: ' || $res + end as contents; \ No newline at end of file From f420f5a6917da7b83baaf0efa3cb63fa62e7ebce Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 21:49:42 +0100 Subject: [PATCH 10/11] add logging --- src/webserver/database/sqlpage_functions/functions.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index a790fd03..1b52f706 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -285,11 +285,13 @@ async fn fetch_with_meta( } } Err(e) => { + log::warn!("Failed to read response body: {e}"); obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?; } } } Err(e) => { + log::warn!("Request failed: {e}"); obj.serialize_entry("error", &format!("Request failed: {e}"))?; } } From a1fafc67d68026dd90efa5ff82492c6eedb8c72c Mon Sep 17 00:00:00 2001 From: lovasoa Date: Mon, 24 Feb 2025 21:59:56 +0100 Subject: [PATCH 11/11] fix tests --- tests/sql_test_files/it_works_fetch_with_meta_error.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sql_test_files/it_works_fetch_with_meta_error.sql b/tests/sql_test_files/it_works_fetch_with_meta_error.sql index c8430863..4c6d2c88 100644 --- a/tests/sql_test_files/it_works_fetch_with_meta_error.sql +++ b/tests/sql_test_files/it_works_fetch_with_meta_error.sql @@ -2,6 +2,6 @@ set res = sqlpage.fetch_with_meta('http://not-a-real-url'); select 'text' as component, case - when json_extract($res, '$.error') LIKE 'Request failed%' then 'It works !' - else 'Error! Got: ' || $res + when json_extract($res, '$.error') LIKE '%Request failed%' then 'It works !' + else CONCAT('Error! Got: ', $res) end as contents; \ No newline at end of file