fix fetchRetry to work in commonjs

add polyfill for file writing to work in nodejs and bun

update examples extraction and screenshot apis

Author: Granitosaurus

.gitignore

@@ -6,3 +6,5 @@ dev
 cov_profile
 .vscode
 .history
+examples/**/package.json
+examples/**/package-lock.json

README.md

@@ -24,8 +24,10 @@ The SDK is distributed through:
 3. Start scraping: 🚀
 
 ```javascript
-// node or bun:
+// node 
 import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
+// bun
+import { ScrapflyClient, ScrapeConfig} from '@scrapfly/scrapfly-sdk';
 // deno: 
 import { ScrapflyClient, ScrapeConfig } from 'jsr:@scrapfly/scrapfly-sdk';
 

__tests__/client/extraction.test.ts

@@ -2,14 +2,15 @@ import * as errors from '../../src/errors.ts';
 import { ScrapflyClient } from '../../src/client.ts';
 import { ExtractionConfig } from '../../src/extractionconfig.ts';
 import { assertEquals, assertRejects } from "https://deno.land/[email protected]/assert/mod.ts";
-import { stub } from "https://deno.land/std/testing/mock.ts";
+import { stub } from "https://deno.land/std@0.224.0/testing/mock.ts";
 import { responseFactory } from '../utils.ts';
+import type { RequestOptions } from '../../src/utils.ts';
 
 Deno.test('extract: succeeds', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const html = 'very long html file';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const configUrl = new URL(config.url);
         const configBody = await new Response(config.body).text();
         assertEquals(configUrl.origin + configUrl.pathname, client.HOST + '/extraction');
@@ -51,7 +52,7 @@ Deno.test('extract: fails due to invalid API key', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const html = 'very long html file';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = {
             status: 'error',
             http_code: 401,
@@ -81,7 +82,7 @@ Deno.test('extract: fails due to any extraction related error', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const html = 'very long html file';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = {
             code: 'ERR::EXTRACTION::CONTENT_TYPE_NOT_SUPPORTED',
             error_id: 'f0e9a6af-846a-49ab-8321-e21bb12bf494',

__tests__/client/scrape.test.ts

@@ -4,6 +4,7 @@ import { ScrapeConfig } from '../../src/scrapeconfig.ts';
 import { log } from '../../src/logger.ts';
 import { assertEquals, assertRejects } from "https://deno.land/[email protected]/assert/mod.ts";
 import { resultFactory, responseFactory } from '../utils.ts';
+import type { RequestOptions } from '../../src/utils.ts';
 import { stub } from "https://deno.land/std/testing/mock.ts";
 
 log.setLevel('DEBUG');
@@ -12,7 +13,7 @@ Deno.test('scrape: GET success', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const configUrl = new URL(config.url);
         assertEquals(configUrl.origin + configUrl.pathname, client.HOST + '/scrape');
         assertEquals(config.method, 'GET');
@@ -42,7 +43,7 @@ Deno.test('scrape errors: raises ApiHttpServerError on 500 and success', async (
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: 'https://httpbin.dev/json',
             status_code: 500,
@@ -71,7 +72,7 @@ Deno.test('scrape errors: raises BadApiKeyError on 401', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = {
             status: 'error',
             http_code: 401,
@@ -100,7 +101,7 @@ Deno.test('scrape errors: raises TooManyRequests on 429 and success', async () =
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: 'https://httpbin.dev/json',
             status_code: 429,
@@ -128,7 +129,7 @@ Deno.test('scrape errors: raises ScrapflyScrapeError on ::SCRAPE:: resource and
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::SCRAPE::BAD_PROTOCOL',
@@ -155,7 +156,7 @@ Deno.test('scrape errors: raises ScrapflyWebhookError on ::WEBHOOK:: resource an
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::WEBHOOK::DISABLED ',
@@ -182,7 +183,7 @@ Deno.test('scrape errors: raises ScrapflyProxyError on ERR::PROXY::POOL_NOT_FOUN
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::PROXY::POOL_NOT_FOUND ',
@@ -209,7 +210,7 @@ Deno.test('scrape errors: raises ScrapflyScheduleError on ERR::SCHEDULE::DISABLE
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::SCHEDULE::DISABLED',
@@ -236,7 +237,7 @@ Deno.test('scrape errors: raises ScrapflyAspError on ERR::ASP::SHIELD_ERROR reso
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::ASP::SHIELD_ERROR',
@@ -263,7 +264,7 @@ Deno.test('scrape errors: raises ScrapflySessionError on ERR::SESSION::CONCURREN
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::SESSION::CONCURRENT_ACCESS',
@@ -290,7 +291,7 @@ Deno.test('scrape errors: raises ApiHttpClientError on success and unknown statu
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             status: 'ERR::NEW',
@@ -317,7 +318,7 @@ Deno.test('scrape errors: raises UpstreamHttpServerError on failure, ERR::SCRAPE
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             success: false,
@@ -346,7 +347,7 @@ Deno.test('scrape errors: raises UpstreamHttpClientError on failure, ERR::SCRAPE
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             success: false,
@@ -385,7 +386,7 @@ Deno.test('scrape errors: raises resource exceptions on failure', async () => {
     };
 
     for (const [resource, err] of Object.entries(resourceErrMap)) {
-        const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+        const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
             const result = resultFactory({
                 url: config.url,
                 success: false,
@@ -414,7 +415,7 @@ Deno.test('scrape errors: raises ScrapflyError on unhandled failure', async () =
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const result = resultFactory({
             url: config.url,
             success: false,
@@ -443,7 +444,7 @@ Deno.test('scrape errors: account retrieval status unhandled code (e.g. 404)', a
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         return responseFactory(
             {},
             {
@@ -466,7 +467,7 @@ Deno.test('scrape errors: account retrieval bad api key (status 401)', async ()
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         return responseFactory(
             {},
             {
@@ -496,7 +497,7 @@ Deno.test('concurrent scrape: success with explicit concurrency', async () => {
     const results = [];
     const errors = [];
 
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         await new Promise((resolve) => setTimeout(resolve, 100));  // XXX: NEEDS a delay!
         log.error(config.url);
         if (config.url.includes('200')) {

__tests__/client/screenshot.test.ts

@@ -3,13 +3,14 @@ import { ScrapflyClient } from '../../src/client.ts';
 import { ScreenshotConfig } from '../../src/screenshotconfig.ts';
 import { assertEquals, assertRejects } from "https://deno.land/[email protected]/assert/mod.ts";
 import { stub } from "https://deno.land/std/testing/mock.ts";
+import type { RequestOptions } from '../../src/utils.ts';
 import { mockedStream, responseFactory } from '../utils.ts';
 
 Deno.test('screenshot: succeeds', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const url = 'https://web-scraping.dev/';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const configUrl = new URL(config.url);
         // Ensure the URL matches the pattern
         assertEquals(configUrl.origin + configUrl.pathname, client.HOST + '/screenshot');
@@ -42,7 +43,7 @@ Deno.test('screenshot: fails due to failing upstream response', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const url = 'https://domain.com/down-page/';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const body = {
             code: 'ERR::SCREENSHOT::UNABLE_TO_TAKE_SCREENSHOT',
             error_id: '347bc6cb-1cba-467a-bd06-c932a9e7156d',
@@ -72,7 +73,7 @@ Deno.test('screenshot: fails to non html/text web page', async () => {
     const KEY = '__API_KEY__';
     const client = new ScrapflyClient({ key: KEY });
     const url = 'https://web-scraping.dev/assets/pdf/eula.pdf/';
-    const fetchStub = stub(client, 'fetch', async (config: Request): Promise<Response> => {
+    const fetchStub = stub(client, 'fetch', async (config: RequestOptions): Promise<Response> => {
         const body = {
             code: 'ERR::SCREENSHOT::INVALID_CONTENT_TYPE',
             error_id: 'f0e9a6af-846a-49ab-8321-e21bb12bf494',

__tests__/utils.test.ts

@@ -23,8 +23,7 @@ Deno.test('fetchRetry: succeeds on first attempt', async () => {
     return new Response('Success', { status: 200 });
   });
 
-  const request = new Request('https://example.com');
-  const response = await fetchRetry(request);
+  const response = await fetchRetry({"url": "https://scrapfly.io/"});
 
   assertEquals(await response.text(), 'Success');
   assertEquals(response.status, 200);
@@ -43,8 +42,7 @@ Deno.test('fetchRetry: retries on 500 and succeeds', async () => {
     }
   });
 
-  const request = new Request('https://example.com');
-  const response = await fetchRetry(request);
+  const response = await fetchRetry({"url": "https://scrapfly.io/"});
 
   assertEquals(await response.text(), 'Success');
   assertEquals(response.status, 200);
@@ -60,8 +58,7 @@ Deno.test('fetchRetry: does not retry 4xx', async () => {
     return new Response('bad request', { status: 422 });
   });
 
-  const request = new Request('https://example.com');
-  const response = await fetchRetry(request);
+  const response = await fetchRetry({"url": "https://scrapfly.io/"});
 
   assertEquals(await response.text(), 'bad request');
   assertEquals(response.status, 422);
@@ -77,11 +74,9 @@ Deno.test('fetchRetry: fails after max retries', async () => {
     return new Response('Internal Server Error', { status: 500 });
   });
 
-  const request = new Request('https://example.com');
-
   await assertRejects(
     async () => {
-      await fetchRetry(request, 3);
+      await fetchRetry({"url": "https://scrapfly.io/"}, 3);
     },
     Error,
     'Fetch failed with status: 500'
@@ -92,6 +87,18 @@ Deno.test('fetchRetry: fails after max retries', async () => {
   fetchStub.restore();
 });
 
+Deno.test('fetchRetry: fails after max retries on real target', async () => {
+  await assertRejects(
+    async () => {
+      const resp = await fetchRetry({"url": "https://httpbin.dev/status/500"}, 3);
+    },
+    Error,
+    'Fetch failed with status: 500'
+  );
+  
+});
+
+
 // XXX: should we support built-in timeout?
 /* 
 Deno.test('fetchRetry: fails due to timeout', async () => {

deno.json

@@ -4,7 +4,7 @@
   },
   "name": "@scrapfly/scrapfly-sdk",
   "exports": "./src/main.ts",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
   "tasks": {
     "start": "deno run --allow-net --allow-read src/main.ts",
@@ -26,7 +26,5 @@
       "exclude": ["no-explicit-any"]
     }
   },
-  "imports": {
-    "@deno/dnt": "jsr:@deno/dnt@^0.41.2"
-  }
+  "imports": { "@deno/dnt": "jsr:@deno/dnt@^0.41.2", "@scrapfly/scrapfly-sdk": "jsr:@scrapfly/scrapfly-sdk@^0.6.2" }
 }