diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b71ec8..5fdffed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,9 @@ jobs: run: make vet - name: Test + run: make test + + - name: Test-Cover run: make test-cover - name: Upload coverage diff --git a/Makefile b/Makefile index 0283e6a..b3058cf 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ test: internal/sqlite/sqlite3.c @CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags="libsqlite3,sqlite_json1" ./... test-cover: - @CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags=$(TAGS) ./... -cover -covermode=count -coverprofile=coverage.out + @CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags="libsqlite3,sqlite_json1"./... -cover -covermode=count -coverprofile=coverage.out @CGO_LDFLAGS="${CGO_LDFLAGS}" go tool cover -html=coverage.out vet: internal/sqlite/sqlite3.c diff --git a/go.mod b/go.mod index 2c0acea..6ca50df 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8 + github.com/jmoiron/sqlx v1.3.4 github.com/mattn/go-sqlite3 v1.14.6 go.riyazali.net/sqlite v0.0.0-20210326190148-448ec1ab2454 gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index b9182f4..f3d2146 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,12 @@ github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8 h1:Dkj6EXcZhE github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8/go.mod h1:BKbu+b2ssBzOOxr8F3X1cVZa5K+cd452Ogc3qgcfIfY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= diff --git a/internal/readfile/readfile.go b/internal/file_read/file_read.go similarity index 97% rename from internal/readfile/readfile.go rename to internal/file_read/file_read.go index 90fbf7f..e91ae07 100644 --- a/internal/readfile/readfile.go +++ b/internal/file_read/file_read.go @@ -1,4 +1,4 @@ -package readfile +package file_read import ( "io/ioutil" diff --git a/internal/http/get.go b/internal/http/get.go index 38f0bd8..ed69a48 100644 --- a/internal/http/get.go +++ b/internal/http/get.go @@ -8,28 +8,35 @@ import ( "go.riyazali.net/sqlite" ) -type get struct{} - -// TODO add PUT and POST stuff +type get struct { + client *http.Client +} -func (m *get) Args() int { return -1 } -func (m *get) Deterministic() bool { return false } -func (m *get) Apply(ctx *sqlite.Context, values ...sqlite.Value) { +func (f *get) Args() int { return -1 } +func (f *get) Deterministic() bool { return false } +func (f *get) Apply(ctx *sqlite.Context, values ...sqlite.Value) { var ( - request string + url string + headers [][]string err error contents []byte - response *http.Response + request *http.Request ) if len(values) > 0 { - request = values[0].Text() + url = values[0].Text() + } else if len(values) > 1 { + heads := values[1].Text() + headers = ParseHeaders(heads) } else { - err := errors.New("input a single url as the argument to http get") + err := errors.New("input a single url as the argument to http get or a url with headers") ctx.ResultError(err) } - - response, err = http.Get(request) + request, err = HttpRequest(url, headers, "GET") + if err != nil { + ctx.ResultError(err) + } + response, err := f.client.Do(request) if err != nil { ctx.ResultError(err) } @@ -37,11 +44,10 @@ func (m *get) Apply(ctx *sqlite.Context, values ...sqlite.Value) { if err != nil { ctx.ResultError(err) } - ctx.ResultText(string(contents)) } // NewHTTPGet returns a sqlite function for reading the contents of a file func NewHTTPGet() sqlite.Function { - return &get{} + return &get{http.DefaultClient} } diff --git a/internal/http/helpers.go b/internal/http/helpers.go new file mode 100644 index 0000000..055249b --- /dev/null +++ b/internal/http/helpers.go @@ -0,0 +1,29 @@ +package http + +import ( + "net/http" + "strings" +) + +func ParseHeaders(headers string) [][]string { + headerList := strings.Split(headers, "|") + var kvHeaders [][]string + for _, s := range headerList { + st := strings.Split(s, ":") + for i := range st { + st[i] = strings.TrimSpace(st[i]) + } + kvHeaders = append(kvHeaders, st) + } + return kvHeaders +} +func HttpRequest(requestUrl string, headers [][]string, requestType string) (*http.Request, error) { + request, err := http.NewRequest(requestType, requestUrl, nil) + if err != nil { + return nil, err + } + for _, header := range headers { + request.Header.Add(header[0], header[1]) + } + return request, nil +} diff --git a/internal/http/helpers_test.go b/internal/http/helpers_test.go new file mode 100644 index 0000000..4645da9 --- /dev/null +++ b/internal/http/helpers_test.go @@ -0,0 +1,161 @@ +package http + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + _ "github.com/augmentable-dev/flite/internal/sqlite" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "go.riyazali.net/sqlite" +) + +func TestHeaderParser(t *testing.T) { + header := "Content-Type: application/json |Accept: application/json|accept-encoding: application/gzip|x-api-key: 09187304915uqiyoewue90832174109732y6985132" + expected := [][]string{{"Content-Type", "application/json"}, {"Accept", "application/json"}, {"accept-encoding", "application/gzip"}, {"x-api-key", "09187304915uqiyoewue90832174109732y6985132"}} + parsed := ParseHeaders(header) + for i := range expected { + if expected[i][0] != parsed[i][0] || expected[i][1] != parsed[i][1] { + t.Fatalf("expected %s,%s got %s,%s", expected[i][0], expected[i][1], parsed[i][0], parsed[i][1]) + } + } +} + +type mockRoundTripper struct { + f func(*http.Request) (*http.Response, error) +} + +func (m *mockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return m.f(req) +} +func newMockRoundTripper(f func(*http.Request) (*http.Response, error)) *mockRoundTripper { + return &mockRoundTripper{f: f} +} +func TestHTTPGet(t *testing.T) { + getFunc := NewHTTPGet() + f := getFunc.(*get) + url := "https://some-url.com/v1/some-endpoint.json" + body := "OK" + + f.client.Transport = newMockRoundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.String() != url { + t.Fatalf("expected request URL: %s, got: %s", url, req.URL.String()) + } + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + }, nil + }) + sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) { + if err := api.CreateFunction("http_get", getFunc); err != nil { + return sqlite.SQLITE_ERROR, err + } + return sqlite.SQLITE_OK, nil + }) + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + row := db.QueryRow("SELECT http_get($1)", url) + err = row.Err() + if err != nil { + t.Fatal(err) + } + var res string + err = row.Scan(&res) + if err != nil { + t.Fatal(err) + } + if res != body { + t.Fatalf("expected response: %s, got: %s", body, res) + } +} +func TestHTTPPost(t *testing.T) { + postFunc := NewHTTPpost() + f := postFunc.(*post) + url := "https://some-url.com/v1/some-endpoint.json" + body := "OK" + header := "Content-Type: application/json |Accept: application/json|accept-encoding: application/gzip|x-api-key: 09187304915uqiyoewue90832174109732y6985132" + expected := map[string][]string{"Content-Type": {"application/json"}, "Accept": {"application/json"}, "accept-encoding": {"application/gzip"}, "x-api-key": {"09187304915uqiyoewue90832174109732y6985132"}} + f.client.Transport = newMockRoundTripper(func(req *http.Request) (*http.Response, error) { + if req.URL.String() != url { + t.Fatalf("expected request URL: %s, got: %s", url, req.URL.String()) + } + if req.Header.Get("Content-Type") != expected["Content-Type"][0] { + t.Fatalf("expected Header Content-Type: %s got: %s", expected["Content-type"], req.Header.Get("Content-Type")) + } + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(body)), + Header: expected, + }, nil + }) + sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) { + if err := api.CreateFunction("http_post", postFunc); err != nil { + return sqlite.SQLITE_ERROR, err + } + return sqlite.SQLITE_OK, nil + }) + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + row := db.QueryRow("SELECT http_post($1,$2)", url, header) + err = row.Err() + if err != nil { + t.Fatal(err) + } + var res string + err = row.Scan(&res) + if err != nil { + t.Fatal(err) + } + if res != body { + t.Fatalf("expected response: %s, got: %s", body, res) + } +} +func TestHttpRequest(t *testing.T) { + method := "GET" + url := "http://api.citybik.es/v2/networks" + responseRecorder := httptest.NewRecorder() + headers := [][]string{{"Content-Type", "application/json"}} + req, err := HttpRequest(url, headers, method) + if err != nil { + t.Fatal(err) + } + handler := http.HandlerFunc(httpHandler) + handler.ServeHTTP(responseRecorder, req) + if status := responseRecorder.Code; status != http.StatusOK { + t.Fatal(err) + } + expected := `{"alive": true}` + if responseRecorder.Body.String() != expected { + t.Fatalf("received %s expected %s", responseRecorder.Body.String(), expected) + } + +} +func httpHandler(w http.ResponseWriter, r *http.Request) { + println(r.URL.String()) + if r.URL.String() != "http://api.citybik.es/v2/networks" { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"response": "incorrect url"`) + return + } + if r.Header.Get("Content-Type") != "application/json" { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"response": "incorrect header"`) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"alive": true}`) +} diff --git a/internal/http/post.go b/internal/http/post.go new file mode 100644 index 0000000..94ea8c7 --- /dev/null +++ b/internal/http/post.go @@ -0,0 +1,52 @@ +package http + +import ( + "errors" + "io/ioutil" + "net/http" + + "go.riyazali.net/sqlite" +) + +type post struct { + client *http.Client +} + + +func (f *post) Args() int { return -1 } +func (f *post) Deterministic() bool { return false } +func (f *post) Apply(ctx *sqlite.Context, values ...sqlite.Value) { + var ( + url string + headers [][]string + err error + contents []byte + request *http.Request + ) + if len(values) > 1 { + url = values[0].Text() + heads := values[1].Text() + headers = ParseHeaders(heads) + } else { + err := errors.New("input a url with headers as the argument to post") + ctx.ResultError(err) + } + request, err = HttpRequest(url, headers, "POST") + if err != nil { + ctx.ResultError(err) + } + response, err := f.client.Do(request) + if err != nil { + ctx.ResultError(err) + } + contents, err = ioutil.ReadAll(response.Body) + if err != nil { + ctx.ResultError(err) + } + ctx.ResultText(string(contents)) +} + +// NewHTTPpost returns a sqlite function for reading the contents of a file +func NewHTTPpost() sqlite.Function { + return &post{http.DefaultClient} +} diff --git a/pkg/ext/ext.go b/pkg/ext/ext.go index f4b8914..57e6b6b 100644 --- a/pkg/ext/ext.go +++ b/pkg/ext/ext.go @@ -1,9 +1,9 @@ package ext import ( + "github.com/augmentable-dev/flite/internal/file_read" "github.com/augmentable-dev/flite/internal/file_split" "github.com/augmentable-dev/flite/internal/http" - "github.com/augmentable-dev/flite/internal/readfile" "github.com/augmentable-dev/flite/internal/yaml" _ "github.com/mattn/go-sqlite3" "go.riyazali.net/sqlite" @@ -16,7 +16,7 @@ func init() { return sqlite.SQLITE_ERROR, err } - if err := api.CreateFunction("readfile", readfile.NewReadFile()); err != nil { + if err := api.CreateFunction("file_read", file_read.NewReadFile()); err != nil { return sqlite.SQLITE_ERROR, err }