From e0e2423e30162fc3ea9e6ab4f421fb12a91ef72e Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Wed, 28 Jul 2021 01:46:20 -0500 Subject: [PATCH 1/2] Change response body from text to bytea --- sql/pg_net--0.1.sql | 2 +- src/worker.c | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/sql/pg_net--0.1.sql b/sql/pg_net--0.1.sql index a8c24ff..d6a0fa7 100644 --- a/sql/pg_net--0.1.sql +++ b/sql/pg_net--0.1.sql @@ -29,7 +29,7 @@ create table net._http_response( status_code integer, content_type text, headers jsonb, - content text, + body bytea, timed_out bool, error_msg text ); diff --git a/src/worker.c b/src/worker.c index c0a2752..99817b1 100644 --- a/src/worker.c +++ b/src/worker.c @@ -56,7 +56,7 @@ body_cb(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; StringInfo si = (StringInfo)userp; - appendBinaryStringInfo(si, (const char*)contents, (int)realsize); + appendBinaryStringInfoNT(si, (const char*)contents, (int)realsize); return realsize; } @@ -353,6 +353,8 @@ worker_main(Datum main_arg) false, 1) != SPI_OK_INSERT) { elog(ERROR, "SPI_exec failed: %s", query_insert_response_bad.data); + + } } else { int argCount = 6; @@ -362,6 +364,7 @@ worker_main(Datum main_arg) CurlData *cdata = NULL; char *contentType = NULL; bool timedOut = false; + bytea *body_data = NULL; curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code); curl_easy_getinfo(eh, CURLINFO_CONTENT_TYPE, &contentType); @@ -379,12 +382,14 @@ worker_main(Datum main_arg) argValues[1] = Int32GetDatum(http_status_code); nulls[1] = ' '; - argTypes[2] = CSTRINGOID; - argValues[2] = CStringGetDatum(cdata->body->data); - if(cdata->body->data[0] == '\0') - nulls[2] = 'n'; - else - nulls[2] = ' '; + argTypes[2] = BYTEAOID; + + body_data = (bytea *) palloc(cdata->body->len + VARHDRSZ); + SET_VARSIZE(body_data, cdata->body->len + VARHDRSZ); + memcpy(VARDATA(body_data), cdata->body->data, cdata->body->len); + + argValues[2] = PointerGetDatum(body_data); + nulls[2] = ' '; argTypes[3] = JSONBOID; argValues[3] = JsonbPGetDatum(JsonbValueToJsonb(pushJsonbValue(&cdata->headers, WJB_END_OBJECT, NULL))); @@ -409,6 +414,7 @@ worker_main(Datum main_arg) pfree(cdata->body->data); pfree(cdata->body); + pfree(body_data); } curl_multi_remove_handle(cm, eh); From 3e41e27beb2cbe7ea38b4a22a538ef2f1933c875 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 28 Jul 2021 06:41:26 -0500 Subject: [PATCH 2/2] pass bytea through public. test coverage for bytea body json response --- docs/api.md | 2 +- sql/pg_net--0.1.sql | 4 ++-- src/worker.c | 2 +- test/test_http_get_collect.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/api.md b/docs/api.md index a2fffbf..21a8b8e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -83,7 +83,7 @@ where `response` is a composite ```sql status_code integer headers jsonb -content text +body bytea ``` Possible values for `net.http_response_result.status` are `('PENDING', 'SUCCESS', 'ERROR')` diff --git a/sql/pg_net--0.1.sql b/sql/pg_net--0.1.sql index d6a0fa7..a35a37a 100644 --- a/sql/pg_net--0.1.sql +++ b/sql/pg_net--0.1.sql @@ -146,7 +146,7 @@ create type net.request_status as enum ('PENDING', 'SUCCESS', 'ERROR'); create type net.http_response AS ( status_code integer, headers jsonb, - content text + body bytea ); -- State wrapper around responses @@ -220,7 +220,7 @@ begin ( rec.status_code, rec.headers, - rec.content + rec.body )::net.http_response )::net.http_response_result; end; diff --git a/src/worker.c b/src/worker.c index 99817b1..4b7663c 100644 --- a/src/worker.c +++ b/src/worker.c @@ -318,7 +318,7 @@ worker_main(Datum main_arg) initStringInfo(&query_insert_response_ok); appendStringInfo(&query_insert_response_ok, "\ - insert into net._http_response(id, status_code, content, headers, content_type, timed_out) values ($1, $2, $3, $4, $5, $6)"); + insert into net._http_response(id, status_code, body, headers, content_type, timed_out) values ($1, $2, $3, $4, $5, $6)"); initStringInfo(&query_insert_response_bad); appendStringInfo(&query_insert_response_bad, "\ diff --git a/test/test_http_get_collect.py b/test/test_http_get_collect.py index 21cf1ef..7eac441 100644 --- a/test/test_http_get_collect.py +++ b/test/test_http_get_collect.py @@ -86,3 +86,33 @@ def test_http_collect_response_async_does_not_exist(sess): assert response[0] == "ERROR" assert "not found" in response[1] assert response[2] is None + + +def test_http_get_returns_jsonb_body(sess): + """Confirm bytea response body is deserializeable""" + + # Create a request + (request_id,) = sess.execute( + """ + select net.http_get(url:='https://httpbin.org/anything'); + """ + ).fetchone() + + # Commit so background worker can start + sess.commit() + + # Collect the response, waiting as needed + (body,) = sess.execute( + text( + """ + select + encode((x.response).body, 'escape')::jsonb + from + net.http_collect_response(:request_id, async:=false) x; + """ + ), + {"request_id": request_id}, + ).fetchone() + + assert isinstance(body, dict) + assert "url" in body