Skip to content

Commit 8faa80d

Browse files
lovasoalovasoa
committed
better json serializing
1 parent 82f15ef commit 8faa80d

File tree

4 files changed

+73
-99
lines changed

4 files changed

+73
-99
lines changed

‎build.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write)
102102
async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) {
103103
let mut attempt = 1;
104104
let max_attempts = 2;
105-
105+
106106
loop {
107107
match client.get(url).send().await {
108108
Ok(mut resp) => {

‎examples/official-site/sqlpage/migrations/40_fetch.sql‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ The fetch function accepts either a URL string, or a JSON object with the follow
8888
- `username`: Optional username for HTTP Basic Authentication. Introduced in version 0.33.0.
8989
- `password`: Optional password for HTTP Basic Authentication. Only used if username is provided. Introduced in version 0.33.0.
9090
91+
# Error handling and reading response headers
92+
93+
If the request fails, this function throws an error, that will be displayed to the user.
94+
The response headers are not available for inspection.
95+
96+
If you need to handle errors or inspect the response headers, use [`sqlpage.fetch_with_meta`](?function=fetch_with_meta).
9197
'
9298
);
9399
INSERT INTO sqlpage_function_parameters (

‎examples/official-site/sqlpage/migrations/58_fetch_with_meta.sql‎

Lines changed: 19 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,16 @@ VALUES (
1010
'transfer-vertical',
1111
'Sends an HTTP request and returns detailed metadata about the response, including status code, headers, and body.
1212
13-
This function is similar to `fetch`, but returns a JSON object containing detailed information about the response.
13+
This function is similar to [`fetch`](?function=fetch), but returns a JSON object containing detailed information about the response.
1414
The returned object has the following structure:
1515
```json
1616
{
1717
"status": 200,
1818
"headers": {
19-
"content-type": "application/json",
20-
"content-length": "1234",
21-
...
19+
"content-type": "text/html",
20+
"content-length": "1234"
2221
},
23-
"body": "response body content",
22+
"body": "a string, or a json object, depending on the content type",
2423
"error": "error message if any"
2524
}
2625
```
@@ -32,64 +31,28 @@ the function returns a JSON object with an "error" field containing the error me
3231
3332
```sql
3433
-- Make a request and get detailed response information
35-
set response = sqlpage.fetch_with_meta(''https://api.example.com/data'');
34+
set response = sqlpage.fetch_with_meta(''https://pokeapi.co/api/v2/pokemon/ditto'');
3635
37-
-- Check if the request was successful
38-
select case
39-
when json_extract($response, ''$.error'') is not null then
40-
''Request failed: '' || json_extract($response, ''$.error'')
41-
when json_extract($response, ''$.status'') != 200 then
42-
''Request returned status '' || json_extract($response, ''$.status'')
43-
else
44-
''Request successful''
45-
end as message;
36+
-- redirect the user to an error page if the request failed
37+
select ''redirect'' as component, ''error.sql'' as url
38+
where
39+
json_extract($response, ''$.error'') is not null
40+
or json_extract($response, ''$.status'') != 200;
4641
47-
-- Display response headers
48-
select ''code'' as component,
49-
''Response Headers'' as title,
50-
''json'' as language,
51-
json_extract($response, ''$.headers'') as contents;
52-
```
53-
54-
### Example: Error Handling with Retries
55-
56-
```sql
57-
-- Function to make a request with retries
58-
create temp table if not exists make_request as
59-
with recursive retry(attempt, response) as (
60-
-- First attempt
61-
select 1 as attempt,
62-
sqlpage.fetch_with_meta(''https://api.example.com/data'') as response
63-
union all
64-
-- Retry up to 3 times if we get a 5xx error
65-
select attempt + 1,
66-
sqlpage.fetch_with_meta(''https://api.example.com/data'')
67-
from retry
68-
where attempt < 3
69-
and (
70-
json_extract(response, ''$.error'') is not null
71-
or cast(json_extract(response, ''$.status'') as integer) >= 500
72-
)
73-
)
74-
select response
75-
from retry
76-
where json_extract(response, ''$.error'') is null
77-
and cast(json_extract(response, ''$.status'') as integer) < 500
78-
limit 1;
79-
80-
-- Use the response
81-
select case
82-
when $response is null then ''All retry attempts failed''
83-
else ''Request succeeded after retries''
84-
end as message;
42+
-- Extract data from the response json body
43+
select ''card'' as component;
44+
select
45+
json_extract($response, ''$.body.name'') as title,
46+
json_extract($response, ''$.body.abilities[0].ability.name'') as description
47+
from $response;
8548
```
8649
8750
### Example: Advanced Request with Authentication
8851
8952
```sql
9053
set request = json_object(
9154
''method'', ''POST'',
92-
''url'', ''https://api.example.com/data'',
55+
''url'', ''https://sqlpage.free.beeceptor.com'',
9356
''headers'', json_object(
9457
''Content-Type'', ''application/json'',
9558
''Authorization'', ''Bearer '' || sqlpage.environment_variable(''API_TOKEN'')
@@ -101,14 +64,10 @@ set request = json_object(
10164
set response = sqlpage.fetch_with_meta($request);
10265
10366
-- Check response content type
104-
select case
105-
when json_extract($response, ''$.headers.content-type'') like ''%application/json%''
106-
then json_extract($response, ''$.body'')
107-
else null
108-
end as json_response;
67+
select ''debug'' as component, $response as response;
10968
```
11069
111-
The function accepts the same parameters as the `fetch` function. See the documentation of `fetch` for more details about the available parameters.'
70+
The function accepts the same parameters as the [`fetch` function](?function=fetch).'
11271
);
11372

11473
INSERT INTO sqlpage_function_parameters (

‎src/webserver/database/sqlpage_functions/functions.rs‎

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@ async fn fetch_with_meta(
221221
request: &RequestInfo,
222222
http_request: super::http_fetch_request::HttpFetchRequest<'_>,
223223
) -> anyhow::Result<String> {
224+
use serde::{ser::SerializeMap, Serializer};
225+
224226
let client = make_http_client(&request.app_state.config)
225227
.with_context(|| "Unable to create an HTTP client")?;
226228
let req = build_request(&client, &http_request)?;
@@ -233,61 +235,68 @@ async fn fetch_with_meta(
233235
req.send().await
234236
};
235237

236-
let mut response_info = serde_json::Map::new();
238+
let mut resp_str = Vec::new();
239+
let mut encoder = serde_json::Serializer::new(&mut resp_str);
240+
let mut obj = encoder.serialize_map(Some(3))?;
237241
match response_result {
238242
Ok(mut response) => {
239-
response_info.insert("status".to_string(), response.status().as_u16().into());
243+
obj.serialize_entry("status", &response.status().as_u16())?;
240244

241-
let mut headers = serde_json::Map::new();
242-
for (name, value) in response.headers() {
243-
if let Ok(value_str) = value.to_str() {
244-
headers.insert(name.to_string(), value_str.into());
245-
}
246-
}
247-
response_info.insert("headers".to_string(), headers.clone().into());
245+
let headers = response.headers();
246+
247+
let is_json = headers
248+
.get("content-type")
249+
.and_then(|v| v.to_str().ok())
250+
.unwrap_or_default()
251+
.starts_with("application/json");
252+
253+
obj.serialize_entry(
254+
"headers",
255+
&headers
256+
.iter()
257+
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or_default()))
258+
.collect::<std::collections::HashMap<_, _>>(),
259+
)?;
248260

249261
match response.body().await {
250262
Ok(body) => {
251263
let body_bytes = body.to_vec();
252-
let content_type = headers
253-
.get("content-type")
254-
.and_then(|v| v.as_str())
255-
.unwrap_or_default();
256-
257-
let body_value = if content_type.contains("application/json") {
258-
match serde_json::from_slice(&body_bytes) {
259-
Ok(json_value) => json_value,
260-
Err(_) => serde_json::Value::String(
261-
String::from_utf8_lossy(&body_bytes).into_owned(),
262-
),
264+
let body_str = String::from_utf8(body_bytes);
265+
266+
match body_str {
267+
Ok(body_str) if is_json => {
268+
obj.serialize_entry(
269+
"body",
270+
&serde_json::value::RawValue::from_string(body_str)?,
271+
)?;
272+
}
273+
Ok(body_str) => {
274+
obj.serialize_entry("body", &body_str)?;
275+
}
276+
Err(utf8_err) => {
277+
let mut base64_string = String::new();
278+
base64::Engine::encode_string(
279+
&base64::engine::general_purpose::STANDARD,
280+
utf8_err.as_bytes(),
281+
&mut base64_string,
282+
);
283+
obj.serialize_entry("body", &base64_string)?;
263284
}
264-
} else if let Ok(text) = String::from_utf8(body_bytes.clone()) {
265-
serde_json::Value::String(text)
266-
} else {
267-
let mut base64_string = String::new();
268-
base64::Engine::encode_string(
269-
&base64::engine::general_purpose::STANDARD,
270-
&body_bytes,
271-
&mut base64_string,
272-
);
273-
serde_json::Value::String(base64_string)
274-
};
275-
response_info.insert("body".to_string(), body_value);
285+
}
276286
}
277287
Err(e) => {
278-
response_info.insert(
279-
"error".to_string(),
280-
format!("Failed to read response body: {e}").into(),
281-
);
288+
obj.serialize_entry("error", &format!("Failed to read response body: {e}"))?;
282289
}
283290
}
284291
}
285292
Err(e) => {
286-
response_info.insert("error".to_string(), format!("Request failed: {e}").into());
293+
obj.serialize_entry("error", &format!("Request failed: {e}"))?;
287294
}
288295
}
289296

290-
Ok(serde_json::to_string(&response_info)?)
297+
obj.end()?;
298+
let return_value = String::from_utf8(resp_str)?;
299+
Ok(return_value)
291300
}
292301

293302
static NATIVE_CERTS: OnceLock<anyhow::Result<rustls::RootCertStore>> = OnceLock::new();

0 commit comments

Comments
 (0)