new function: fetch_with_meta

lovasoa · lovasoa · commit 9494cb0ddd93 · 2025-02-22T15:08:12.000+01:00
closes #792
diff --git a/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql b/examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql
@@ -0,0 +1,127 @@
+INSERT INTO sqlpage_functions (
+        "name",
+        "introduced_in_version",
+        "icon",
+        "description_md"
+    )
+VALUES (
+        'fetch_with_meta',
+        '0.34.0',
+        'transfer-vertical',
+        'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body.
+
+This function is similar to `fetch`, but returns a JSON object containing detailed information about the response.
+The returned object has the following structure:
+```json
+{
+    "status": 200,
+    "headers": {
+        "content-type": "application/json",
+        "content-length": "1234",
+        ...
+    },
+    "body": "response body content",
+    "error": "error message if any"
+}
+```
+
+If the request fails or encounters an error (e.g., network issues, invalid UTF-8 response), instead of throwing an error,
+the function returns a JSON object with an "error" field containing the error message.
+
+### Example: Basic Usage
+
+```sql
+-- Make a request and get detailed response information
+set response = sqlpage.fetch_with_meta(''https://api.example.com/data'');
+
+-- Check if the request was successful
+select case 
+    when json_extract($response, ''$.error'') is not null then
+        ''Request failed: '' || json_extract($response, ''$.error'')
+    when json_extract($response, ''$.status'') != 200 then
+        ''Request returned status '' || json_extract($response, ''$.status'')
+    else
+        ''Request successful''
+end as message;
+
+-- Display response headers
+select ''code'' as component,
+    ''Response Headers'' as title,
+    ''json'' as language,
+    json_extract($response, ''$.headers'') as contents;
+```
+
+### Example: Error Handling with Retries
+
+```sql
+-- Function to make a request with retries
+create temp table if not exists make_request as
+with recursive retry(attempt, response) as (
+    -- First attempt
+    select 1 as attempt,
+           sqlpage.fetch_with_meta(''https://api.example.com/data'') as response
+    union all
+    -- Retry up to 3 times if we get a 5xx error
+    select attempt + 1,
+           sqlpage.fetch_with_meta(''https://api.example.com/data'')
+    from retry
+    where attempt < 3
+    and (
+        json_extract(response, ''$.error'') is not null
+        or cast(json_extract(response, ''$.status'') as integer) >= 500
+    )
+)
+select response
+from retry
+where json_extract(response, ''$.error'') is null
+  and cast(json_extract(response, ''$.status'') as integer) < 500
+limit 1;
+
+-- Use the response
+select case
+    when $response is null then ''All retry attempts failed''
+    else ''Request succeeded after retries''
+end as message;
+```
+
+### Example: Advanced Request with Authentication
+
+```sql
+set request = json_object(
+    ''method'', ''POST'',
+    ''url'', ''https://api.example.com/data'',
+    ''headers'', json_object(
+        ''Content-Type'', ''application/json'',
+        ''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'')
+    ),
+    ''body'', json_object(
+        ''key'', ''value''
+    )
+);
+set response = sqlpage.fetch_with_meta($request);
+
+-- Check response content type
+select case
+    when json_extract($response, ''$.headers.content-type'') like ''%application/json%''
+    then json_extract($response, ''$.body'')
+    else null
+end as json_response;
+```
+
+The function accepts the same parameters as the `fetch` function. See the documentation of `fetch` for more details about the available parameters.'
+    );
+
+INSERT INTO sqlpage_function_parameters (
+        "function",
+        "index",
+        "name",
+        "description_md",
+        "type"
+    )
+VALUES (
+        'fetch_with_meta',
+        1,
+        'url',
+        'Either a string containing an URL to request, or a json object in the standard format of the request interface of the web fetch API.',
+        'TEXT'
+    ); 
diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs
@@ -24,6 +24,7 @@ super::function_definition_macro::sqlpage_functions! {
     exec((&RequestInfo), program_name: Cow<str>, args: Vec<Cow<str>>);
 
     fetch((&RequestInfo), http_request: SqlPageFunctionParam<super::http_fetch_request::HttpFetchRequest<'_>>);
+    fetch_with_meta((&RequestInfo), http_request: SqlPageFunctionParam<super::http_fetch_request::HttpFetchRequest<'_>>);
 
     hash_password(password: Option<String>);
     header((&RequestInfo), name: Cow<str>);
@@ -135,53 +136,70 @@ async fn exec<'a>(
     Ok(String::from_utf8_lossy(&res.stdout).into_owned())
 }
 
-async fn fetch(
-    request: &RequestInfo,
-    http_request: super::http_fetch_request::HttpFetchRequest<'_>,
-) -> anyhow::Result<String> {
+fn build_request<'a>(
+    client: &'a awc::Client,
+    http_request: &'a super::http_fetch_request::HttpFetchRequest<'_>,
+) -> anyhow::Result<awc::ClientRequest> {
     use awc::http::Method;
-    let client = make_http_client(&request.app_state.config)
-        .with_context(|| "Unable to create an HTTP client")?;
-
-    let method = if let Some(method) = http_request.method {
-        Method::from_str(&method).with_context(|| format!("Invalid HTTP method: {method}"))?
+    let method = if let Some(method) = &http_request.method {
+        Method::from_str(method).with_context(|| format!("Invalid HTTP method: {method}"))?
     } else {
         Method::GET
     };
     let mut req = client.request(method, http_request.url.as_ref());
     if let Some(timeout) = http_request.timeout_ms {
         req = req.timeout(core::time::Duration::from_millis(timeout));
     }
-    for (k, v) in http_request.headers {
+    for (k, v) in &http_request.headers {
         req = req.insert_header((k.as_ref(), v.as_ref()));
     }
-    if let Some(username) = http_request.username {
-        let password = http_request.password.unwrap_or_default();
+    if let Some(username) = &http_request.username {
+        let password = http_request.password.as_deref().unwrap_or_default();
         req = req.basic_auth(username, password);
     }
+    Ok(req)
+}
+
+fn prepare_request_body(
+    body: &serde_json::value::RawValue,
+    mut req: awc::ClientRequest,
+) -> anyhow::Result<(String, awc::ClientRequest)> {
+    let val = body.get();
+    let body_str = if val.starts_with('"') {
+        serde_json::from_str::<'_, String>(val).with_context(|| {
+            format!("Invalid JSON string in the body of the HTTP request: {val}")
+        })?
+    } else {
+        req = req.content_type("application/json");
+        val.to_owned()
+    };
+    Ok((body_str, req))
+}
+
+async fn fetch(
+    request: &RequestInfo,
+    http_request: super::http_fetch_request::HttpFetchRequest<'_>,
+) -> anyhow::Result<String> {
+    let client = make_http_client(&request.app_state.config)
+        .with_context(|| "Unable to create an HTTP client")?;
+    let req = build_request(&client, &http_request)?;
+    
     log::info!("Fetching {}", http_request.url);
-    let mut response = if let Some(body) = http_request.body {
-        let val = body.get();
-        // The body can be either json, or a string representing a raw body
-        let body = if val.starts_with('"') {
-            serde_json::from_str::<'_, String>(val).with_context(|| {
-                format!("Invalid JSON string in the body of the HTTP request: {val}")
-            })?
-        } else {
-            req = req.content_type("application/json");
-            val.to_owned()
-        };
+    let mut response = if let Some(body) = &http_request.body {
+        let (body, req) = prepare_request_body(body, req)?;
         req.send_body(body)
     } else {
         req.send()
     }
     .await
     .map_err(|e| anyhow!("Unable to fetch {}: {e}", http_request.url))?;
+    
     log::debug!(
         "Finished fetching {}. Status: {}",
         http_request.url,
         response.status()
     );
+    
     let body = response
         .body()
         .await
@@ -199,6 +217,76 @@ async fn fetch(
     Ok(response_str)
 }
 
+async fn fetch_with_meta(
+    request: &RequestInfo,
+    http_request: super::http_fetch_request::HttpFetchRequest<'_>,
+) -> anyhow::Result<String> {
+    let client = make_http_client(&request.app_state.config)
+        .with_context(|| "Unable to create an HTTP client")?;
+    let req = build_request(&client, &http_request)?;
+    
+    log::info!("Fetching {} with metadata", http_request.url);
+    let response_result = if let Some(body) = &http_request.body {
+        let (body, req) = prepare_request_body(body, req)?;
+        req.send_body(body).await
+    } else {
+        req.send().await
+    };
+
+    let mut response_info = serde_json::Map::new();
+    match response_result {
+        Ok(mut response) => {
+            response_info.insert("status".to_string(), response.status().as_u16().into());
+            
+            let mut headers = serde_json::Map::new();
+            for (name, value) in response.headers().iter() {
+                if let Ok(value_str) = value.to_str() {
+                    headers.insert(name.to_string(), value_str.into());
+                }
+            }
+            response_info.insert("headers".to_string(), headers.clone().into());
+
+            match response.body().await {
+                Ok(body) => {
+                    let body_bytes = body.to_vec();
+                    let content_type = headers.get("content-type")
+                        .and_then(|v| v.as_str())
+                        .unwrap_or_default();
+
+                    let body_value = if content_type.contains("application/json") {
+                        match serde_json::from_slice(&body_bytes) {
+                            Ok(json_value) => json_value,
+                            Err(_) => serde_json::Value::String(String::from_utf8_lossy(&body_bytes).into_owned()),
+                        }
+                    } else {
+                        match String::from_utf8(body_bytes.clone()) {
+                            Ok(text) => serde_json::Value::String(text),
+                            Err(_) => {
+                                let mut base64_string = String::new();
+                                base64::Engine::encode_string(
+                                    &base64::engine::general_purpose::STANDARD,
+                                    &body_bytes,
+                                    &mut base64_string,
+                                );
+                                serde_json::Value::String(base64_string)
+                            }
+                        }
+                    };
+                    response_info.insert("body".to_string(), body_value);
+                }
+                Err(e) => {
+                    response_info.insert("error".to_string(), format!("Failed to read response body: {}", e).into());
+                }
+            }
+        }
+        Err(e) => {
+            response_info.insert("error".to_string(), format!("Request failed: {}", e).into());
+        }
+    }
+
+    Ok(serde_json::to_string(&response_info)?)
+}
+
 static NATIVE_CERTS: OnceLock<anyhow::Result<rustls::RootCertStore>> = OnceLock::new();
 
 fn make_http_client(config: &crate::app_config::AppConfig) -> anyhow::Result<awc::Client> {
