diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..3d2ac0b --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.1.0" +} \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 312cf57..6804ffb 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 15 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/scrapegraphai%2Fscrapegraphai-969ebada41127057e4cda129b2e7206224743b5c7fd33aa8ae062ff71b775ac9.yml -openapi_spec_hash: 2b2c2c684e6f6885398efca5f2b1f854 -config_hash: 938d01575f4400810a85576c6324f29d +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/scrapegraphai%2Fscrapegraphai-633fdeab6abaefbe666099e8f86ce6b2acc9dacff1c33a80813bb04e8e437229.yml +openapi_spec_hash: f41ec90694ca8e7233bd20cc7ff1afbf +config_hash: 6889576ba0fdc14f2c71cea09a60a0f6 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b3a00b1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +## 0.1.0 (2025-09-26) + +Full Changelog: [v0.0.1...v0.1.0](https://github.com/ScrapeGraphAI/scrapegraphai-go/compare/v0.0.1...v0.1.0) + +### Features + +* add files ([97854ff](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/97854ff979b013c7efad31e34a3d69d79aa30324)) + + +### Bug Fixes + +* bugfix for setting JSON keys with special characters ([222be49](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/222be49bdaf83c8bc03cd4e3cac467e0053f564c)) +* close body before retrying ([1d46935](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/1d4693550bac9b3731c9d7d531e83db9630a0632)) +* **internal:** unmarshal correctly when there are multiple discriminators ([5468ee8](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/5468ee86e67c05f4785caa68a8406367a04e3bc7)) +* use slices.Concat instead of sometimes modifying r.Options ([82ca835](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/82ca835db2e7304e50a4af6a7042a531ebd8058f)) + + +### Chores + +* bump minimum go version to 1.22 ([a915799](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/a915799b8fef759a3a8d0f3282aa96e1b683a81d)) +* do not install brew dependencies in ./scripts/bootstrap by default ([3064991](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/3064991a415d952635d4556d3bf546701e2dbcd8)) +* **internal:** codegen related update ([5b87899](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/5b878999e0a8295a834430c7311db61bb4ecc285)) +* update more docs for 1.22 ([f6381a8](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/f6381a8ebbc32746beb76c123565b1d135ede2e6)) +* update SDK settings ([b3671c2](https://github.com/ScrapeGraphAI/scrapegraphai-go/commit/b3671c28676539d26390c214911188376fe48e93)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e3d792..06d021e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ $ ./scripts/lint This will install all the required dependencies and build the SDK. -You can also [install go 1.18+ manually](https://go.dev/doc/install). +You can also [install go 1.22+ manually](https://go.dev/doc/install). ## Modifying/Adding code @@ -41,7 +41,7 @@ To use a local version of this library from source in another project, edit the directive. This can be done through the CLI with the following: ```sh -$ go mod edit -replace github.com/stainless-sdks/scrapegraphai-go=/path/to/scrapegraphai-go +$ go mod edit -replace github.com/ScrapeGraphAI/scrapegraphai-go=/path/to/scrapegraphai-go ``` ## Running tests diff --git a/README.md b/README.md index 3d6528d..5b5d78e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Scrapegraphai Go API Library -Go Reference + + +Go Reference + + The Scrapegraphai Go library provides convenient access to the [Scrapegraphai REST API](https://scrapegraphai.com) from applications written in Go. @@ -9,21 +13,29 @@ It is generated with [Stainless](https://www.stainless.com/). ## Installation + + ```go import ( - "github.com/stainless-sdks/scrapegraphai-go" // imported as scrapegraphai + "github.com/ScrapeGraphAI/scrapegraphai-go" // imported as scrapegraphai ) ``` + + Or to pin the version: + + ```sh -go get -u 'github.com/stainless-sdks/scrapegraphai-go@v0.0.1' +go get -u 'github.com/ScrapeGraphAI/scrapegraphai-go@v0.1.0' ``` + + ## Requirements -This library requires Go 1.18+. +This library requires Go 1.22+. ## Usage @@ -36,8 +48,8 @@ import ( "context" "fmt" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func main() { @@ -267,7 +279,7 @@ client.Smartscraper.New(context.TODO(), ..., The request option `option.WithDebugLog(nil)` may be helpful while debugging. -See the [full list of request options](https://pkg.go.dev/github.com/stainless-sdks/scrapegraphai-go/option). +See the [full list of request options](https://pkg.go.dev/github.com/ScrapeGraphAI/scrapegraphai-go/option). ### Pagination @@ -481,7 +493,7 @@ This package generally follows [SemVer](https://semver.org/spec/v2.0.0.html) con We take backwards-compatibility seriously and work hard to ensure you can rely on a smooth upgrade experience. -We are keen for your feedback; please open an [issue](https://www.github.com/stainless-sdks/scrapegraphai-go/issues) with questions, bugs, or suggestions. +We are keen for your feedback; please open an [issue](https://www.github.com/ScrapeGraphAI/scrapegraphai-go/issues) with questions, bugs, or suggestions. ## Contributing diff --git a/aliases.go b/aliases.go index 1fd83b4..1176c2f 100644 --- a/aliases.go +++ b/aliases.go @@ -3,8 +3,8 @@ package scrapegraphai import ( - "github.com/stainless-sdks/scrapegraphai-go/internal/apierror" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apierror" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" ) // aliased to make [param.APIUnion] private when embedding diff --git a/api.md b/api.md index d88431e..8e73063 100644 --- a/api.md +++ b/api.md @@ -2,101 +2,101 @@ Response Types: -- scrapegraphai.CompletedSmartscraper -- scrapegraphai.FailedSmartscraper -- scrapegraphai.SmartscraperGetResponseUnion -- scrapegraphai.SmartscraperListResponseUnion +- scrapegraphai.CompletedSmartscraper +- scrapegraphai.FailedSmartscraper +- scrapegraphai.SmartscraperGetResponseUnion +- scrapegraphai.SmartscraperListResponseUnion Methods: -- client.Smartscraper.New(ctx context.Context, body scrapegraphai.SmartscraperNewParams) (scrapegraphai.CompletedSmartscraper, error) -- client.Smartscraper.Get(ctx context.Context, requestID string) (scrapegraphai.SmartscraperGetResponseUnion, error) -- client.Smartscraper.List(ctx context.Context) (scrapegraphai.SmartscraperListResponseUnion, error) +- client.Smartscraper.New(ctx context.Context, body scrapegraphai.SmartscraperNewParams) (scrapegraphai.CompletedSmartscraper, error) +- client.Smartscraper.Get(ctx context.Context, requestID string) (scrapegraphai.SmartscraperGetResponseUnion, error) +- client.Smartscraper.List(ctx context.Context) (scrapegraphai.SmartscraperListResponseUnion, error) # Markdownify Response Types: -- scrapegraphai.CompletedMarkdownify -- scrapegraphai.MarkdownifyGetStatusResponseUnion +- scrapegraphai.CompletedMarkdownify +- scrapegraphai.MarkdownifyGetStatusResponseUnion Methods: -- client.Markdownify.Convert(ctx context.Context, body scrapegraphai.MarkdownifyConvertParams) (scrapegraphai.CompletedMarkdownify, error) -- client.Markdownify.GetStatus(ctx context.Context, requestID string) (scrapegraphai.MarkdownifyGetStatusResponseUnion, error) +- client.Markdownify.Convert(ctx context.Context, body scrapegraphai.MarkdownifyConvertParams) (scrapegraphai.CompletedMarkdownify, error) +- client.Markdownify.GetStatus(ctx context.Context, requestID string) (scrapegraphai.MarkdownifyGetStatusResponseUnion, error) # Searchscraper Response Types: -- scrapegraphai.CompletedSearchScraper -- scrapegraphai.SearchscraperGetStatusResponseUnion +- scrapegraphai.CompletedSearchScraper +- scrapegraphai.SearchscraperGetStatusResponseUnion Methods: -- client.Searchscraper.New(ctx context.Context, body scrapegraphai.SearchscraperNewParams) (scrapegraphai.CompletedSearchScraper, error) -- client.Searchscraper.GetStatus(ctx context.Context, requestID string) (scrapegraphai.SearchscraperGetStatusResponseUnion, error) +- client.Searchscraper.New(ctx context.Context, body scrapegraphai.SearchscraperNewParams) (scrapegraphai.CompletedSearchScraper, error) +- client.Searchscraper.GetStatus(ctx context.Context, requestID string) (scrapegraphai.SearchscraperGetStatusResponseUnion, error) # GenerateSchema Response Types: -- scrapegraphai.GenerateSchemaNewResponse -- scrapegraphai.GenerateSchemaGetResponseUnion +- scrapegraphai.GenerateSchemaNewResponse +- scrapegraphai.GenerateSchemaGetResponseUnion Methods: -- client.GenerateSchema.New(ctx context.Context, body scrapegraphai.GenerateSchemaNewParams) (scrapegraphai.GenerateSchemaNewResponse, error) -- client.GenerateSchema.Get(ctx context.Context, requestID string) (scrapegraphai.GenerateSchemaGetResponseUnion, error) +- client.GenerateSchema.New(ctx context.Context, body scrapegraphai.GenerateSchemaNewParams) (scrapegraphai.GenerateSchemaNewResponse, error) +- client.GenerateSchema.Get(ctx context.Context, requestID string) (scrapegraphai.GenerateSchemaGetResponseUnion, error) # Crawl Response Types: -- scrapegraphai.CrawlGetResultsResponse -- scrapegraphai.CrawlStartResponse +- scrapegraphai.CrawlGetResultsResponse +- scrapegraphai.CrawlStartResponse Methods: -- client.Crawl.GetResults(ctx context.Context, taskID string) (scrapegraphai.CrawlGetResultsResponse, error) -- client.Crawl.Start(ctx context.Context, body scrapegraphai.CrawlStartParams) (scrapegraphai.CrawlStartResponse, error) +- client.Crawl.GetResults(ctx context.Context, taskID string) (scrapegraphai.CrawlGetResultsResponse, error) +- client.Crawl.Start(ctx context.Context, body scrapegraphai.CrawlStartParams) (scrapegraphai.CrawlStartResponse, error) # Credits Response Types: -- scrapegraphai.CreditGetResponse +- scrapegraphai.CreditGetResponse Methods: -- client.Credits.Get(ctx context.Context) (scrapegraphai.CreditGetResponse, error) +- client.Credits.Get(ctx context.Context) (scrapegraphai.CreditGetResponse, error) # Validate Response Types: -- scrapegraphai.ValidateAPIKeyResponse +- scrapegraphai.ValidateAPIKeyResponse Methods: -- client.Validate.APIKey(ctx context.Context) (scrapegraphai.ValidateAPIKeyResponse, error) +- client.Validate.APIKey(ctx context.Context) (scrapegraphai.ValidateAPIKeyResponse, error) # Feedback Response Types: -- scrapegraphai.FeedbackSubmitResponse +- scrapegraphai.FeedbackSubmitResponse Methods: -- client.Feedback.Submit(ctx context.Context, body scrapegraphai.FeedbackSubmitParams) (scrapegraphai.FeedbackSubmitResponse, error) +- client.Feedback.Submit(ctx context.Context, body scrapegraphai.FeedbackSubmitParams) (scrapegraphai.FeedbackSubmitResponse, error) # Healthz Response Types: -- scrapegraphai.HealthzCheckResponse +- scrapegraphai.HealthzCheckResponse Methods: -- client.Healthz.Check(ctx context.Context) (scrapegraphai.HealthzCheckResponse, error) +- client.Healthz.Check(ctx context.Context) (scrapegraphai.HealthzCheckResponse, error) diff --git a/client.go b/client.go index 40fa119..4d07d88 100644 --- a/client.go +++ b/client.go @@ -6,9 +6,10 @@ import ( "context" "net/http" "os" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) // Client creates a struct with services and top level methods that help with @@ -94,7 +95,7 @@ func NewClient(opts ...option.RequestOption) (r Client) { // For even greater flexibility, see [option.WithResponseInto] and // [option.WithResponseBodyInto]. func (r *Client) Execute(ctx context.Context, method string, path string, params any, res any, opts ...option.RequestOption) error { - opts = append(r.Options, opts...) + opts = slices.Concat(r.Options, opts) return requestconfig.ExecuteNewRequest(ctx, method, path, params, res, opts...) } diff --git a/client_test.go b/client_test.go index 654345f..5c2d398 100644 --- a/client_test.go +++ b/client_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) type closureTransport struct { diff --git a/crawl.go b/crawl.go index d224923..2624e5f 100644 --- a/crawl.go +++ b/crawl.go @@ -7,12 +7,13 @@ import ( "errors" "fmt" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // CrawlService contains methods and other services that help with interacting with @@ -36,7 +37,7 @@ func NewCrawlService(opts ...option.RequestOption) (r CrawlService) { // Retrieve the status and results of a crawling job func (r *CrawlService) GetResults(ctx context.Context, taskID string, opts ...option.RequestOption) (res *CrawlGetResultsResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) if taskID == "" { err = errors.New("missing required task_id parameter") return @@ -50,7 +51,7 @@ func (r *CrawlService) GetResults(ctx context.Context, taskID string, opts ...op // extraction mode and markdown conversion mode. Returns a task ID for async // processing. func (r *CrawlService) Start(ctx context.Context, body CrawlStartParams, opts ...option.RequestOption) (res *CrawlStartResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "crawl" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return diff --git a/crawl_test.go b/crawl_test.go index c5d219d..f5e2a56 100644 --- a/crawl_test.go +++ b/crawl_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestCrawlGetResults(t *testing.T) { diff --git a/credit.go b/credit.go index 1ac1c23..55826d7 100644 --- a/credit.go +++ b/credit.go @@ -5,11 +5,12 @@ package scrapegraphai import ( "context" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // CreditService contains methods and other services that help with interacting @@ -33,7 +34,7 @@ func NewCreditService(opts ...option.RequestOption) (r CreditService) { // Retrieve the current credit balance and usage for the authenticated user func (r *CreditService) Get(ctx context.Context, opts ...option.RequestOption) (res *CreditGetResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "credits" err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, nil, &res, opts...) return diff --git a/credit_test.go b/credit_test.go index 27bb292..c5e8f41 100644 --- a/credit_test.go +++ b/credit_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestCreditGet(t *testing.T) { diff --git a/feedback.go b/feedback.go index 6fe4c8d..6123ac2 100644 --- a/feedback.go +++ b/feedback.go @@ -5,13 +5,14 @@ package scrapegraphai import ( "context" "net/http" + "slices" "time" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // FeedbackService contains methods and other services that help with interacting @@ -35,7 +36,7 @@ func NewFeedbackService(opts ...option.RequestOption) (r FeedbackService) { // Submit feedback for a specific request func (r *FeedbackService) Submit(ctx context.Context, body FeedbackSubmitParams, opts ...option.RequestOption) (res *FeedbackSubmitResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "feedback" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return diff --git a/feedback_test.go b/feedback_test.go index 1b37015..6aa1c24 100644 --- a/feedback_test.go +++ b/feedback_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestFeedbackSubmitWithOptionalParams(t *testing.T) { diff --git a/field.go b/field.go index e84a943..084e4ca 100644 --- a/field.go +++ b/field.go @@ -1,7 +1,7 @@ package scrapegraphai import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "io" "time" ) diff --git a/generateschema.go b/generateschema.go index 07b7dc8..7b93390 100644 --- a/generateschema.go +++ b/generateschema.go @@ -8,12 +8,13 @@ import ( "errors" "fmt" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // GenerateSchemaService contains methods and other services that help with @@ -38,7 +39,7 @@ func NewGenerateSchemaService(opts ...option.RequestOption) (r GenerateSchemaSer // Generate or modify JSON schemas based on natural language descriptions. Can // create new schemas or extend existing ones. func (r *GenerateSchemaService) New(ctx context.Context, body GenerateSchemaNewParams, opts ...option.RequestOption) (res *GenerateSchemaNewResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "generate_schema" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return @@ -46,7 +47,7 @@ func (r *GenerateSchemaService) New(ctx context.Context, body GenerateSchemaNewP // Retrieve the status and results of a schema generation request func (r *GenerateSchemaService) Get(ctx context.Context, requestID string, opts ...option.RequestOption) (res *GenerateSchemaGetResponseUnion, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) if requestID == "" { err = errors.New("missing required request_id parameter") return diff --git a/generateschema_test.go b/generateschema_test.go index a3df646..7ee052e 100644 --- a/generateschema_test.go +++ b/generateschema_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestGenerateSchemaNewWithOptionalParams(t *testing.T) { diff --git a/go.mod b/go.mod index 61d6e0d..a3bf7b9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ -module github.com/stainless-sdks/scrapegraphai-go +module github.com/ScrapeGraphAI/scrapegraphai-go -go 1.21 +go 1.22 require ( github.com/tidwall/gjson v1.14.4 diff --git a/healthz.go b/healthz.go index fb90546..23100bd 100644 --- a/healthz.go +++ b/healthz.go @@ -5,11 +5,12 @@ package scrapegraphai import ( "context" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // HealthzService contains methods and other services that help with interacting @@ -33,7 +34,7 @@ func NewHealthzService(opts ...option.RequestOption) (r HealthzService) { // Check the health status of the service func (r *HealthzService) Check(ctx context.Context, opts ...option.RequestOption) (res *HealthzCheckResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "healthz" err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, nil, &res, opts...) return diff --git a/healthz_test.go b/healthz_test.go index 30a6b47..37d2f43 100644 --- a/healthz_test.go +++ b/healthz_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestHealthzCheck(t *testing.T) { diff --git a/internal/apierror/apierror.go b/internal/apierror/apierror.go index 07843c5..9c54cbd 100644 --- a/internal/apierror/apierror.go +++ b/internal/apierror/apierror.go @@ -7,8 +7,8 @@ import ( "net/http" "net/http/httputil" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // Error represents an error that originates from the API, i.e. when a request is diff --git a/internal/apiform/encoder.go b/internal/apiform/encoder.go index ff9a458..9cefa18 100644 --- a/internal/apiform/encoder.go +++ b/internal/apiform/encoder.go @@ -13,7 +13,7 @@ import ( "sync" "time" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" ) var encoders sync.Map // map[encoderEntry]encoderFunc diff --git a/internal/apiform/form_test.go b/internal/apiform/form_test.go index 3ee1fb6..833cfe9 100644 --- a/internal/apiform/form_test.go +++ b/internal/apiform/form_test.go @@ -2,7 +2,7 @@ package apiform import ( "bytes" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "io" "mime/multipart" "strings" diff --git a/internal/apiform/richparam.go b/internal/apiform/richparam.go index fe05bb7..632aeb9 100644 --- a/internal/apiform/richparam.go +++ b/internal/apiform/richparam.go @@ -1,7 +1,7 @@ package apiform import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "mime/multipart" "reflect" ) diff --git a/internal/apijson/decodeparam_test.go b/internal/apijson/decodeparam_test.go index b8213e0..31a7c8c 100644 --- a/internal/apijson/decodeparam_test.go +++ b/internal/apijson/decodeparam_test.go @@ -3,8 +3,8 @@ package apijson_test import ( "encoding/json" "fmt" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" "testing" ) @@ -351,6 +351,36 @@ func init() { }) } +type FooVariant struct { + Type string `json:"type,required"` + Value string `json:"value,required"` +} + +type BarVariant struct { + Type string `json:"type,required"` + Enable bool `json:"enable,required"` +} + +type MultiDiscriminatorUnion struct { + OfFoo *FooVariant `json:",inline"` + OfBar *BarVariant `json:",inline"` + + paramUnion +} + +func init() { + apijson.RegisterDiscriminatedUnion[MultiDiscriminatorUnion]("type", map[string]reflect.Type{ + "foo": reflect.TypeOf(FooVariant{}), + "foo_v2": reflect.TypeOf(FooVariant{}), + "bar": reflect.TypeOf(BarVariant{}), + "bar_legacy": reflect.TypeOf(BarVariant{}), + }) +} + +func (m *MultiDiscriminatorUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, m) +} + func (d *DiscriminatedUnion) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, d) } @@ -408,3 +438,61 @@ func TestDiscriminatedUnion(t *testing.T) { }) } } + +func TestMultiDiscriminatorUnion(t *testing.T) { + tests := map[string]struct { + raw string + target MultiDiscriminatorUnion + shouldFail bool + }{ + "foo_variant": { + raw: `{"type":"foo","value":"test"}`, + target: MultiDiscriminatorUnion{OfFoo: &FooVariant{ + Type: "foo", + Value: "test", + }}, + }, + "foo_v2_variant": { + raw: `{"type":"foo_v2","value":"test_v2"}`, + target: MultiDiscriminatorUnion{OfFoo: &FooVariant{ + Type: "foo_v2", + Value: "test_v2", + }}, + }, + "bar_variant": { + raw: `{"type":"bar","enable":true}`, + target: MultiDiscriminatorUnion{OfBar: &BarVariant{ + Type: "bar", + Enable: true, + }}, + }, + "bar_legacy_variant": { + raw: `{"type":"bar_legacy","enable":false}`, + target: MultiDiscriminatorUnion{OfBar: &BarVariant{ + Type: "bar_legacy", + Enable: false, + }}, + }, + "invalid_type": { + raw: `{"type":"unknown","value":"test"}`, + target: MultiDiscriminatorUnion{}, + shouldFail: true, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var dst MultiDiscriminatorUnion + err := json.Unmarshal([]byte(test.raw), &dst) + if err != nil && !test.shouldFail { + t.Fatalf("failed unmarshal with err: %v", err) + } + if err == nil && test.shouldFail { + t.Fatalf("expected unmarshal to fail but it succeeded") + } + if !reflect.DeepEqual(dst, test.target) { + t.Fatalf("failed equality, got %#v but expected %#v", dst, test.target) + } + }) + } +} diff --git a/internal/apijson/decoder.go b/internal/apijson/decoder.go index 3819d46..faff3ff 100644 --- a/internal/apijson/decoder.go +++ b/internal/apijson/decoder.go @@ -7,7 +7,7 @@ package apijson import ( "encoding/json" "fmt" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" "strconv" "sync" diff --git a/internal/apijson/decoderesp_test.go b/internal/apijson/decoderesp_test.go index f2eb8b8..3cec2a2 100644 --- a/internal/apijson/decoderesp_test.go +++ b/internal/apijson/decoderesp_test.go @@ -2,8 +2,8 @@ package apijson_test import ( "encoding/json" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" "testing" ) diff --git a/internal/apijson/encoder.go b/internal/apijson/encoder.go index 8358a2f..ab7a3c1 100644 --- a/internal/apijson/encoder.go +++ b/internal/apijson/encoder.go @@ -16,6 +16,10 @@ import ( var encoders sync.Map // map[encoderEntry]encoderFunc +// If we want to set a literal key value into JSON using sjson, we need to make sure it doesn't have +// special characters that sjson interprets as a path. +var EscapeSJSONKey = strings.NewReplacer("\\", "\\\\", "|", "\\|", "#", "\\#", "@", "\\@", "*", "\\*", ".", "\\.", ":", "\\:", "?", "\\?").Replace + func Marshal(value any) ([]byte, error) { e := &encoder{dateFormat: time.RFC3339} return e.marshal(value) @@ -270,7 +274,7 @@ func (e *encoder) newStructTypeEncoder(t reflect.Type) encoderFunc { if encoded == nil { continue } - json, err = sjson.SetRawBytes(json, ef.tag.name, encoded) + json, err = sjson.SetRawBytes(json, EscapeSJSONKey(ef.tag.name), encoded) if err != nil { return nil, err } @@ -348,7 +352,7 @@ func (e *encoder) encodeMapEntries(json []byte, v reflect.Value) ([]byte, error) } encodedKeyString = string(encodedKeyBytes) } - encodedKey := []byte(sjsonReplacer.Replace(encodedKeyString)) + encodedKey := []byte(encodedKeyString) pairs = append(pairs, mapPair{key: encodedKey, value: iter.Value()}) } @@ -366,7 +370,7 @@ func (e *encoder) encodeMapEntries(json []byte, v reflect.Value) ([]byte, error) if len(encodedValue) == 0 { continue } - json, err = sjson.SetRawBytes(json, string(p.key), encodedValue) + json, err = sjson.SetRawBytes(json, EscapeSJSONKey(string(p.key)), encodedValue) if err != nil { return nil, err } @@ -386,7 +390,3 @@ func (e *encoder) newMapEncoder(_ reflect.Type) encoderFunc { return json, nil } } - -// If we want to set a literal key value into JSON using sjson, we need to make sure it doesn't have -// special characters that sjson interprets as a path. -var sjsonReplacer *strings.Replacer = strings.NewReplacer(".", "\\.", ":", "\\:", "*", "\\*") diff --git a/internal/apijson/subfield.go b/internal/apijson/subfield.go index 88328ce..091d160 100644 --- a/internal/apijson/subfield.go +++ b/internal/apijson/subfield.go @@ -1,7 +1,7 @@ package apijson import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" "reflect" ) diff --git a/internal/apijson/union.go b/internal/apijson/union.go index 64f6a8c..6d8368c 100644 --- a/internal/apijson/union.go +++ b/internal/apijson/union.go @@ -2,7 +2,7 @@ package apijson import ( "errors" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" "github.com/tidwall/gjson" @@ -39,12 +39,10 @@ func RegisterDiscriminatedUnion[T any](key string, mappings map[string]reflect.T func (d *decoderBuilder) newStructUnionDecoder(t reflect.Type) decoderFunc { type variantDecoder struct { - decoder decoderFunc - field reflect.StructField - discriminatorValue any + decoder decoderFunc + field reflect.StructField } - - variants := []variantDecoder{} + decoders := []variantDecoder{} for i := 0; i < t.NumField(); i++ { field := t.Field(i) @@ -53,18 +51,26 @@ func (d *decoderBuilder) newStructUnionDecoder(t reflect.Type) decoderFunc { } decoder := d.typeDecoder(field.Type) - variants = append(variants, variantDecoder{ + decoders = append(decoders, variantDecoder{ decoder: decoder, field: field, }) } + type discriminatedDecoder struct { + variantDecoder + discriminator any + } + discriminatedDecoders := []discriminatedDecoder{} unionEntry, discriminated := unionRegistry[t] - for _, unionVariant := range unionEntry.variants { - for i := 0; i < len(variants); i++ { - variant := &variants[i] - if variant.field.Type.Elem() == unionVariant.Type { - variant.discriminatorValue = unionVariant.DiscriminatorValue + for _, variant := range unionEntry.variants { + // For each union variant, find a matching decoder and save it + for _, decoder := range decoders { + if decoder.field.Type.Elem() == variant.Type { + discriminatedDecoders = append(discriminatedDecoders, discriminatedDecoder{ + decoder, + variant.DiscriminatorValue, + }) break } } @@ -72,11 +78,11 @@ func (d *decoderBuilder) newStructUnionDecoder(t reflect.Type) decoderFunc { return func(n gjson.Result, v reflect.Value, state *decoderState) error { if discriminated && n.Type == gjson.JSON && len(unionEntry.discriminatorKey) != 0 { - discriminator := n.Get(unionEntry.discriminatorKey).Value() - for _, variant := range variants { - if discriminator == variant.discriminatorValue { - inner := v.FieldByIndex(variant.field.Index) - return variant.decoder(n, inner, state) + discriminator := n.Get(EscapeSJSONKey(unionEntry.discriminatorKey)).Value() + for _, decoder := range discriminatedDecoders { + if discriminator == decoder.discriminator { + inner := v.FieldByIndex(decoder.field.Index) + return decoder.decoder(n, inner, state) } } return errors.New("apijson: was not able to find discriminated union variant") @@ -85,15 +91,15 @@ func (d *decoderBuilder) newStructUnionDecoder(t reflect.Type) decoderFunc { // Set bestExactness to worse than loose bestExactness := loose - 1 bestVariant := -1 - for i, variant := range variants { + for i, decoder := range decoders { // Pointers are used to discern JSON object variants from value variants - if n.Type != gjson.JSON && variant.field.Type.Kind() == reflect.Ptr { + if n.Type != gjson.JSON && decoder.field.Type.Kind() == reflect.Ptr { continue } sub := decoderState{strict: state.strict, exactness: exact} - inner := v.FieldByIndex(variant.field.Index) - err := variant.decoder(n, inner, &sub) + inner := v.FieldByIndex(decoder.field.Index) + err := decoder.decoder(n, inner, &sub) if err != nil { continue } @@ -116,11 +122,11 @@ func (d *decoderBuilder) newStructUnionDecoder(t reflect.Type) decoderFunc { return errors.New("apijson: was not able to coerce type as union strictly") } - for i := 0; i < len(variants); i++ { + for i := 0; i < len(decoders); i++ { if i == bestVariant { continue } - v.FieldByIndex(variants[i].field.Index).SetZero() + v.FieldByIndex(decoders[i].field.Index).SetZero() } return nil @@ -156,7 +162,7 @@ func (d *decoderBuilder) newUnionDecoder(t reflect.Type) decoderFunc { } if len(unionEntry.discriminatorKey) != 0 { - discriminatorValue := n.Get(unionEntry.discriminatorKey).Value() + discriminatorValue := n.Get(EscapeSJSONKey(unionEntry.discriminatorKey)).Value() if discriminatorValue == variant.DiscriminatorValue { inner := reflect.New(variant.Type).Elem() err := decoder(n, inner, state) diff --git a/internal/apiquery/encoder.go b/internal/apiquery/encoder.go index 27408c3..1499419 100644 --- a/internal/apiquery/encoder.go +++ b/internal/apiquery/encoder.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" ) var encoders sync.Map // map[reflect.Type]encoderFunc diff --git a/internal/apiquery/query_test.go b/internal/apiquery/query_test.go index 437c63c..e5df6f3 100644 --- a/internal/apiquery/query_test.go +++ b/internal/apiquery/query_test.go @@ -1,7 +1,7 @@ package apiquery import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "net/url" "testing" "time" diff --git a/internal/apiquery/richparam.go b/internal/apiquery/richparam.go index 888baac..6ffef6a 100644 --- a/internal/apiquery/richparam.go +++ b/internal/apiquery/richparam.go @@ -1,7 +1,7 @@ package apiquery import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" ) diff --git a/internal/encoding/json/decode.go b/internal/encoding/json/decode.go index 199174b..b7b6600 100644 --- a/internal/encoding/json/decode.go +++ b/internal/encoding/json/decode.go @@ -14,7 +14,7 @@ import ( "encoding" "encoding/base64" "fmt" - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/shims" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/shims" "reflect" "strconv" "strings" diff --git a/internal/encoding/json/encode.go b/internal/encoding/json/encode.go index 2065a3a..87185d6 100644 --- a/internal/encoding/json/encode.go +++ b/internal/encoding/json/encode.go @@ -19,8 +19,8 @@ import ( "encoding" "encoding/base64" "fmt" - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/sentinel" - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/shims" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/sentinel" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/shims" "math" "reflect" "slices" diff --git a/internal/encoding/json/sentinel/null.go b/internal/encoding/json/sentinel/null.go index 3877fc6..e84142f 100644 --- a/internal/encoding/json/sentinel/null.go +++ b/internal/encoding/json/sentinel/null.go @@ -1,7 +1,7 @@ package sentinel import ( - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/shims" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/shims" "reflect" "sync" ) diff --git a/internal/encoding/json/sentinel/sentinel_test.go b/internal/encoding/json/sentinel/sentinel_test.go index c4bc972..7fbe91c 100644 --- a/internal/encoding/json/sentinel/sentinel_test.go +++ b/internal/encoding/json/sentinel/sentinel_test.go @@ -1,8 +1,8 @@ package sentinel_test import ( - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/sentinel" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/sentinel" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" "slices" "testing" diff --git a/internal/encoding/json/shims/shims.go b/internal/encoding/json/shims/shims.go index b65a016..fe9a71a 100644 --- a/internal/encoding/json/shims/shims.go +++ b/internal/encoding/json/shims/shims.go @@ -1,5 +1,5 @@ // This package provides shims over Go 1.2{2,3} APIs -// which are missing from Go 1.21, and used by the Go 1.24 encoding/json package. +// which are missing from Go 1.22, and used by the Go 1.24 encoding/json package. // // Inside the vendored package, all shim code has comments that begin look like // // SHIM(...): ... diff --git a/internal/encoding/json/time.go b/internal/encoding/json/time.go index ad7a670..949e646 100644 --- a/internal/encoding/json/time.go +++ b/internal/encoding/json/time.go @@ -2,7 +2,7 @@ package json import ( - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/shims" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/shims" "reflect" "time" ) diff --git a/internal/paramutil/field.go b/internal/paramutil/field.go index d2ba28a..4e201b1 100644 --- a/internal/paramutil/field.go +++ b/internal/paramutil/field.go @@ -1,8 +1,8 @@ package paramutil import ( - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) func AddrIfPresent[T comparable](v param.Opt[T]) *T { diff --git a/internal/paramutil/union.go b/internal/paramutil/union.go index db8e421..51b6553 100644 --- a/internal/paramutil/union.go +++ b/internal/paramutil/union.go @@ -2,7 +2,7 @@ package paramutil import ( "fmt" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "reflect" ) diff --git a/internal/requestconfig/requestconfig.go b/internal/requestconfig/requestconfig.go index 20b8259..8bc0b26 100644 --- a/internal/requestconfig/requestconfig.go +++ b/internal/requestconfig/requestconfig.go @@ -18,10 +18,10 @@ import ( "strings" "time" - "github.com/stainless-sdks/scrapegraphai-go/internal" - "github.com/stainless-sdks/scrapegraphai-go/internal/apierror" - "github.com/stainless-sdks/scrapegraphai-go/internal/apiform" - "github.com/stainless-sdks/scrapegraphai-go/internal/apiquery" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apierror" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apiform" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apiquery" ) func getDefaultHeaders() map[string]string { @@ -461,6 +461,11 @@ func (cfg *RequestConfig) Execute() (err error) { break } + // Close the response body before retrying to prevent connection leaks + if res != nil && res.Body != nil { + res.Body.Close() + } + time.Sleep(retryDelay(res, retryCount)) } diff --git a/internal/version.go b/internal/version.go index 989c168..02eac73 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,4 +2,4 @@ package internal -const PackageVersion = "0.0.1" +const PackageVersion = "0.1.0" // x-release-please-version diff --git a/markdownify.go b/markdownify.go index 7568643..0b6a1d9 100644 --- a/markdownify.go +++ b/markdownify.go @@ -8,12 +8,13 @@ import ( "errors" "fmt" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // MarkdownifyService contains methods and other services that help with @@ -37,7 +38,7 @@ func NewMarkdownifyService(opts ...option.RequestOption) (r MarkdownifyService) // Convert web page content to clean Markdown format func (r *MarkdownifyService) Convert(ctx context.Context, body MarkdownifyConvertParams, opts ...option.RequestOption) (res *CompletedMarkdownify, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "markdownify" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return @@ -45,7 +46,7 @@ func (r *MarkdownifyService) Convert(ctx context.Context, body MarkdownifyConver // Retrieve the status and results of a markdown conversion func (r *MarkdownifyService) GetStatus(ctx context.Context, requestID string, opts ...option.RequestOption) (res *MarkdownifyGetStatusResponseUnion, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) if requestID == "" { err = errors.New("missing required request_id parameter") return diff --git a/markdownify_test.go b/markdownify_test.go index 00af1f5..d55d447 100644 --- a/markdownify_test.go +++ b/markdownify_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestMarkdownifyConvertWithOptionalParams(t *testing.T) { diff --git a/option/requestoption.go b/option/requestoption.go index 4fbe17c..2294f46 100644 --- a/option/requestoption.go +++ b/option/requestoption.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" "github.com/tidwall/sjson" ) @@ -19,7 +19,7 @@ import ( // which can be supplied to clients, services, and methods. You can read more about this functional // options pattern in our [README]. // -// [README]: https://pkg.go.dev/github.com/stainless-sdks/scrapegraphai-go#readme-requestoptions +// [README]: https://pkg.go.dev/github.com/ScrapeGraphAI/scrapegraphai-go#readme-requestoptions type RequestOption = requestconfig.RequestOption // WithBaseURL returns a RequestOption that sets the BaseURL for the client. diff --git a/packages/param/encoder.go b/packages/param/encoder.go index afa9ce5..1465934 100644 --- a/packages/param/encoder.go +++ b/packages/param/encoder.go @@ -4,9 +4,10 @@ import ( "encoding/json" "fmt" "reflect" + "strings" "time" - shimjson "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json" + shimjson "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json" "github.com/tidwall/sjson" ) @@ -14,6 +15,10 @@ import ( // EncodedAsDate is not be stable and shouldn't be relied upon type EncodedAsDate Opt[time.Time] +// If we want to set a literal key value into JSON using sjson, we need to make sure it doesn't have +// special characters that sjson interprets as a path. +var EscapeSJSONKey = strings.NewReplacer("\\", "\\\\", "|", "\\|", "#", "\\#", "@", "\\@", "*", "\\*", ".", "\\.", ":", "\\:", "?", "\\?").Replace + type forceOmit int func (m EncodedAsDate) MarshalJSON() ([]byte, error) { @@ -52,7 +57,7 @@ func MarshalWithExtras[T ParamStruct, R any](f T, underlying any, extras map[str } continue } - bytes, err = sjson.SetBytes(bytes, k, v) + bytes, err = sjson.SetBytes(bytes, EscapeSJSONKey(k), v) if err != nil { return nil, err } diff --git a/packages/param/encoder_test.go b/packages/param/encoder_test.go index f6d6e56..05b6bf1 100644 --- a/packages/param/encoder_test.go +++ b/packages/param/encoder_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" ) type Struct struct { diff --git a/packages/param/null.go b/packages/param/null.go index 66f42ba..e5852f3 100644 --- a/packages/param/null.go +++ b/packages/param/null.go @@ -1,6 +1,6 @@ package param -import "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/sentinel" +import "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/sentinel" // NullMap returns a non-nil map with a length of 0. // When used with [MarshalObject] or [MarshalUnion], it will be marshaled as null. diff --git a/packages/param/null_test.go b/packages/param/null_test.go index 7e41de5..002b05d 100644 --- a/packages/param/null_test.go +++ b/packages/param/null_test.go @@ -2,7 +2,7 @@ package param_test import ( "encoding/json" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" "testing" ) diff --git a/packages/param/option.go b/packages/param/option.go index 6299b22..fb3bf0d 100644 --- a/packages/param/option.go +++ b/packages/param/option.go @@ -3,7 +3,7 @@ package param import ( "encoding/json" "fmt" - shimjson "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json" + shimjson "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json" "time" ) diff --git a/packages/param/param.go b/packages/param/param.go index 84b28bf..951a3c0 100644 --- a/packages/param/param.go +++ b/packages/param/param.go @@ -2,7 +2,7 @@ package param import ( "encoding/json" - "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json/sentinel" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json/sentinel" "reflect" ) diff --git a/packages/respjson/decoder_test.go b/packages/respjson/decoder_test.go index 6376411..5cb3fbc 100644 --- a/packages/respjson/decoder_test.go +++ b/packages/respjson/decoder_test.go @@ -3,8 +3,8 @@ package respjson_test import ( "encoding/json" "fmt" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - rj "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + rj "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" "reflect" "testing" ) diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..a38198e --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,67 @@ +{ + "packages": { + ".": {} + }, + "$schema": "https://raw.githubusercontent.com/stainless-api/release-please/main/schemas/config.json", + "include-v-in-tag": true, + "include-component-in-tag": false, + "versioning": "prerelease", + "prerelease": true, + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": false, + "pull-request-header": "Automated Release PR", + "pull-request-title-pattern": "release: ${version}", + "changelog-sections": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "revert", + "section": "Reverts" + }, + { + "type": "chore", + "section": "Chores" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "style", + "section": "Styles" + }, + { + "type": "refactor", + "section": "Refactors" + }, + { + "type": "test", + "section": "Tests", + "hidden": true + }, + { + "type": "build", + "section": "Build System" + }, + { + "type": "ci", + "section": "Continuous Integration", + "hidden": true + } + ], + "release-type": "go", + "extra-files": [ + "internal/version.go", + "README.md" + ] +} \ No newline at end of file diff --git a/scripts/bootstrap b/scripts/bootstrap index d6ac165..5ab3066 100755 --- a/scripts/bootstrap +++ b/scripts/bootstrap @@ -4,10 +4,18 @@ set -e cd "$(dirname "$0")/.." -if [ -f "Brewfile" ] && [ "$(uname -s)" = "Darwin" ] && [ "$SKIP_BREW" != "1" ]; then +if [ -f "Brewfile" ] && [ "$(uname -s)" = "Darwin" ] && [ "$SKIP_BREW" != "1" ] && [ -t 0 ]; then brew bundle check >/dev/null 2>&1 || { - echo "==> Installing Homebrew dependencies…" - brew bundle + echo -n "==> Install Homebrew dependencies? (y/N): " + read -r response + case "$response" in + [yY][eE][sS]|[yY]) + brew bundle + ;; + *) + ;; + esac + echo } fi diff --git a/searchscraper.go b/searchscraper.go index c2b91cc..4b6bd27 100644 --- a/searchscraper.go +++ b/searchscraper.go @@ -8,12 +8,13 @@ import ( "errors" "fmt" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // SearchscraperService contains methods and other services that help with @@ -39,7 +40,7 @@ func NewSearchscraperService(opts ...option.RequestOption) (r SearchscraperServi // multiple websites. Uses LLM to refine search queries and merge results from // different sources. func (r *SearchscraperService) New(ctx context.Context, body SearchscraperNewParams, opts ...option.RequestOption) (res *CompletedSearchScraper, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "searchscraper" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return @@ -47,7 +48,7 @@ func (r *SearchscraperService) New(ctx context.Context, body SearchscraperNewPar // Retrieve the status and results of a search scraping operation func (r *SearchscraperService) GetStatus(ctx context.Context, requestID string, opts ...option.RequestOption) (res *SearchscraperGetStatusResponseUnion, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) if requestID == "" { err = errors.New("missing required request_id parameter") return diff --git a/searchscraper_test.go b/searchscraper_test.go index ca8cb68..8f89ca3 100644 --- a/searchscraper_test.go +++ b/searchscraper_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestSearchscraperNewWithOptionalParams(t *testing.T) { diff --git a/shared/constant/constants.go b/shared/constant/constants.go index fe618aa..147abb1 100644 --- a/shared/constant/constants.go +++ b/shared/constant/constants.go @@ -3,7 +3,7 @@ package constant import ( - shimjson "github.com/stainless-sdks/scrapegraphai-go/internal/encoding/json" + shimjson "github.com/ScrapeGraphAI/scrapegraphai-go/internal/encoding/json" ) type Constant[T any] interface { diff --git a/smartscraper.go b/smartscraper.go index fe25c80..9bf8187 100644 --- a/smartscraper.go +++ b/smartscraper.go @@ -8,12 +8,13 @@ import ( "errors" "fmt" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/param" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/param" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // SmartscraperService contains methods and other services that help with @@ -38,7 +39,7 @@ func NewSmartscraperService(opts ...option.RequestOption) (r SmartscraperService // Main scraping endpoint with LLM-powered content analysis. Supports various // fetching providers, infinite scrolling, pagination, and custom output schemas. func (r *SmartscraperService) New(ctx context.Context, body SmartscraperNewParams, opts ...option.RequestOption) (res *CompletedSmartscraper, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "smartscraper" err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) return @@ -46,7 +47,7 @@ func (r *SmartscraperService) New(ctx context.Context, body SmartscraperNewParam // Retrieve the status and results of a scraping operation func (r *SmartscraperService) Get(ctx context.Context, requestID string, opts ...option.RequestOption) (res *SmartscraperGetResponseUnion, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) if requestID == "" { err = errors.New("missing required request_id parameter") return @@ -58,7 +59,7 @@ func (r *SmartscraperService) Get(ctx context.Context, requestID string, opts .. // Retrieve the status and results of a scraping operation func (r *SmartscraperService) List(ctx context.Context, opts ...option.RequestOption) (res *SmartscraperListResponseUnion, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "smartscraper" err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, nil, &res, opts...) return diff --git a/smartscraper_test.go b/smartscraper_test.go index 440d6de..40be819 100644 --- a/smartscraper_test.go +++ b/smartscraper_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestSmartscraperNewWithOptionalParams(t *testing.T) { diff --git a/usage_test.go b/usage_test.go index f98206e..aa8cd35 100644 --- a/usage_test.go +++ b/usage_test.go @@ -7,9 +7,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestUsage(t *testing.T) { diff --git a/validate.go b/validate.go index 04d3372..f751880 100644 --- a/validate.go +++ b/validate.go @@ -5,11 +5,12 @@ package scrapegraphai import ( "context" "net/http" + "slices" - "github.com/stainless-sdks/scrapegraphai-go/internal/apijson" - "github.com/stainless-sdks/scrapegraphai-go/internal/requestconfig" - "github.com/stainless-sdks/scrapegraphai-go/option" - "github.com/stainless-sdks/scrapegraphai-go/packages/respjson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/apijson" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/requestconfig" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go/packages/respjson" ) // ValidateService contains methods and other services that help with interacting @@ -33,7 +34,7 @@ func NewValidateService(opts ...option.RequestOption) (r ValidateService) { // Validate the API key and retrieve associated user email func (r *ValidateService) APIKey(ctx context.Context, opts ...option.RequestOption) (res *ValidateAPIKeyResponse, err error) { - opts = append(r.Options[:], opts...) + opts = slices.Concat(r.Options, opts) path := "validate" err = requestconfig.ExecuteNewRequest(ctx, http.MethodGet, path, nil, &res, opts...) return diff --git a/validate_test.go b/validate_test.go index 00a9ef5..11f080c 100644 --- a/validate_test.go +++ b/validate_test.go @@ -8,9 +8,9 @@ import ( "os" "testing" - "github.com/stainless-sdks/scrapegraphai-go" - "github.com/stainless-sdks/scrapegraphai-go/internal/testutil" - "github.com/stainless-sdks/scrapegraphai-go/option" + "github.com/ScrapeGraphAI/scrapegraphai-go" + "github.com/ScrapeGraphAI/scrapegraphai-go/internal/testutil" + "github.com/ScrapeGraphAI/scrapegraphai-go/option" ) func TestValidateAPIKey(t *testing.T) {