[pull] main from firecrawl:main

.github/workflows/test-js-sdk.yml

@@ -8,7 +8,7 @@ on:
       - apps/js-sdk/firecrawl/**
 
 env:
-  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+  IDMUX_URL: ${{ secrets.IDMUX_URL }}
 
 jobs:
   test:

apps/api/src/controllers/v2/__tests__/agent-status.test.ts

@@ -0,0 +1,77 @@
+import type { Response } from "express";
+import { agentStatusController } from "../agent-status";
+import type { RequestWithAuth } from "../types";
+import {
+  supabaseGetAgentByIdDirect,
+  supabaseGetAgentRequestByIdDirect,
+} from "../../../lib/supabase-jobs";
+import { getJobFromGCS } from "../../../lib/gcs-jobs";
+
+jest.mock("../../../lib/supabase-jobs", () => ({
+  supabaseGetAgentByIdDirect: jest.fn(),
+  supabaseGetAgentRequestByIdDirect: jest.fn(),
+}));
+
+jest.mock("../../../lib/gcs-jobs", () => ({
+  getJobFromGCS: jest.fn(),
+}));
+
+describe("agentStatusController", () => {
+  const baseReq = {
+    params: { jobId: "job-123" },
+    auth: { team_id: "team-123" },
+  } as RequestWithAuth<{ jobId: string }, any, any>;
+
+  const buildRes = () =>
+    ({
+      status: jest.fn().mockReturnThis(),
+      json: jest.fn(),
+    }) as unknown as Response;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("returns model from agent options", async () => {
+    (supabaseGetAgentRequestByIdDirect as jest.Mock).mockResolvedValue({
+      team_id: "team-123",
+      created_at: "2025-01-01T00:00:00Z",
+    });
+    (supabaseGetAgentByIdDirect as jest.Mock).mockResolvedValue({
+      id: "job-123",
+      is_successful: true,
+      options: { model: "spark-1-mini" },
+      created_at: "2025-01-01T00:00:00Z",
+    });
+    (getJobFromGCS as jest.Mock).mockResolvedValue({ result: "ok" });
+
+    const res = buildRes();
+    await agentStatusController(baseReq, res);
+
+    expect(res.status).toHaveBeenCalledWith(200);
+    expect(res.json).toHaveBeenCalledWith(
+      expect.objectContaining({ model: "spark-1-mini" }),
+    );
+  });
+
+  it("defaults model to spark-1-pro when missing", async () => {
+    (supabaseGetAgentRequestByIdDirect as jest.Mock).mockResolvedValue({
+      team_id: "team-123",
+      created_at: "2025-01-01T00:00:00Z",
+    });
+    (supabaseGetAgentByIdDirect as jest.Mock).mockResolvedValue({
+      id: "job-123",
+      is_successful: false,
+      options: null,
+      created_at: "2025-01-01T00:00:00Z",
+    });
+
+    const res = buildRes();
+    await agentStatusController(baseReq, res);
+
+    expect(res.status).toHaveBeenCalledWith(200);
+    expect(res.json).toHaveBeenCalledWith(
+      expect.objectContaining({ model: "spark-1-pro" }),
+    );
+  });
+});

apps/api/src/controllers/v2/agent-status.ts

@@ -4,8 +4,9 @@ import {
   supabaseGetAgentByIdDirect,
   supabaseGetAgentRequestByIdDirect,
 } from "../../lib/supabase-jobs";
-import { logger as _logger } from "../../lib/logger";
+import { logger as _logger, logger } from "../../lib/logger";
 import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { config } from "../../config";
 
 export async function agentStatusController(
   req: RequestWithAuth<{ jobId: string }, AgentStatusResponse, any>,
@@ -24,6 +25,49 @@ export async function agentStatusController(
 
   const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
 
+  let model: "spark-1-pro" | "spark-1-mini";
+  if (agent) {
+    model = (agent.options?.model ?? "spark-1-pro") as
+      | "spark-1-pro"
+      | "spark-1-mini";
+  } else {
+    try {
+      const optionsRequest = await fetch(
+        config.EXTRACT_V3_BETA_URL +
+          "/v2/extract/" +
+          req.params.jobId +
+          "/options",
+        {
+          headers: {
+            Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,
+          },
+        },
+      );
+
+      if (optionsRequest.status !== 200) {
+        logger.warn("Failed to get agent request details", {
+          status: optionsRequest.status,
+          method: "agentStatusController",
+          module: "api/v2",
+          text: await optionsRequest.text(),
+        });
+        model = "spark-1-pro"; // fall back to this value
+      } else {
+        model = ((await optionsRequest.json()).model ?? "spark-1-pro") as
+          | "spark-1-pro"
+          | "spark-1-mini";
+      }
+    } catch (error) {
+      logger.warn("Failed to get agent request details", {
+        error,
+        method: "agentStatusController",
+        module: "api/v2",
+        extractId: req.params.jobId,
+      });
+      model = "spark-1-pro"; // fall back to this value
+    }
+  }
+
   let data: any = undefined;
   if (agent?.is_successful) {
     data = await getJobFromGCS(agent.id);
@@ -38,6 +82,7 @@ export async function agentStatusController(
         : "failed",
     error: agent?.error || undefined,
     data,
+    model,
     expiresAt: new Date(
       new Date(agent?.created_at ?? agentRequest.created_at).getTime() +
         1000 * 60 * 60 * 24,

apps/api/src/controllers/v2/types.ts

@@ -1133,6 +1133,7 @@ export type AgentStatusResponse =
       status: "processing" | "completed" | "failed";
       error?: string;
       data?: any;
+      model?: "spark-1-pro" | "spark-1-mini";
       expiresAt: string;
       creditsUsed?: number;
     };

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "4.11.3",
+  "version": "4.11.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts

@@ -128,9 +128,9 @@ describe("v2.scrape e2e", () => {
     });
     expect(doc.images).toBeTruthy();
     expect(Array.isArray(doc.images)).toBe(true);
-    expect(doc.images.length).toBeGreaterThan(0);
+    expect(doc.images?.length).toBeGreaterThan(0);
     // Should find firecrawl logo/branding images
-    expect(doc.images.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
+    expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true);
   }, 60_000);
 
   test("images format: works with multiple formats", async () => {
@@ -142,7 +142,7 @@ describe("v2.scrape e2e", () => {
     expect(doc.links).toBeTruthy();
     expect(doc.images).toBeTruthy();
     expect(Array.isArray(doc.images)).toBe(true);
-    expect(doc.images.length).toBeGreaterThan(0);
+    expect(doc.images?.length).toBeGreaterThan(0);
 
     // Images should find things not available in links format
     const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico'];
@@ -151,7 +151,7 @@ describe("v2.scrape e2e", () => {
     ) || [];
 
     // Should discover additional images beyond those with obvious extensions
-    expect(doc.images.length).toBeGreaterThanOrEqual(linkImages.length);
+    expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length);
   }, 60_000);
 
   test("invalid url should throw", async () => {

apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts

@@ -2,7 +2,7 @@
  * E2E tests for v2 search (translated from Python tests)
  */
 import Firecrawl from "../../../index";
-import type { Document, SearchResult } from "../../../index";
+import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index";
 import { config } from "dotenv";
 import { getIdentity, getApiUrl } from "./utils/idmux";
 import { describe, test, expect, beforeAll } from "@jest/globals";
@@ -28,7 +28,7 @@ function collectTexts(entries: any[] | undefined): string[] {
   return texts;
 }
 
-function isDocument(entry: Document | SearchResult | undefined | null): entry is Document {
+function isDocument(entry: Document | SearchResultWeb | SearchResultNews | SearchResultImages | undefined | null): entry is Document {
   if (!entry) return false;
   const d = entry as Document;
   return (
@@ -86,10 +86,10 @@ describe("v2.search e2e", () => {
     expect(results.images == null).toBe(true);
 
     const webTitles = (results.web || [])
-      .filter((r): r is SearchResult => !isDocument(r))
+      .filter((r): r is SearchResultWeb => !isDocument(r))
       .map(r => (r.title || "").toString().toLowerCase());
     const webDescriptions = (results.web || [])
-      .filter((r): r is SearchResult => !isDocument(r))
+      .filter((r): r is SearchResultWeb => !isDocument(r))
       .map(r => (r.description || "").toString().toLowerCase());
     const allWebText = (webTitles.concat(webDescriptions)).join(" ");
     expect(allWebText.includes("firecrawl")).toBe(true);
@@ -183,7 +183,7 @@ describe("v2.search e2e", () => {
           expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
         } else {
           expect(typeof result.url).toBe("string");
-          expect(result.url.startsWith("http")).toBe(true);
+          expect(result.url?.startsWith("http")).toBe(true);
         }
       }
     }
@@ -193,7 +193,7 @@ describe("v2.search e2e", () => {
     for (const result of results.images || []) {
       if (!isDocument(result)) {
         expect(typeof result.url).toBe("string");
-        expect(result.url.startsWith("http")).toBe(true);
+        expect(result.url?.startsWith("http")).toBe(true);
       }
     }
   }, 120_000);

apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts

@@ -23,16 +23,15 @@ describe("v2.usage e2e", () => {
     expect(typeof resp.maxConcurrency).toBe("number");
   }, 60_000);
 
-  // NOTE: Disabled, broken on central team due to overflow
-  // test("get_credit_usage", async () => {
-  //   const resp = await client.getCreditUsage();
-  //   expect(typeof resp.remainingCredits).toBe("number");
-  // }, 60_000);
-
-  // test("get_token_usage", async () => {
-  //   const resp = await client.getTokenUsage();
-  //   expect(typeof resp.remainingTokens).toBe("number");
-  // }, 60_000);
+  test("get_credit_usage", async () => {
+    const resp = await client.getCreditUsage();
+    expect(typeof resp.remainingCredits).toBe("number");
+  }, 60_000);
+
+  test("get_token_usage", async () => {
+    const resp = await client.getTokenUsage();
+    expect(typeof resp.remainingTokens).toBe("number");
+  }, 60_000);
 
   test("get_queue_status", async () => {
     const resp = await client.getQueueStatus();

apps/js-sdk/firecrawl/src/v2/types.ts

@@ -556,6 +556,7 @@ export interface AgentStatusResponse {
   status: 'processing' | 'completed' | 'failed';
   error?: string;
   data?: unknown;
+  model?: 'spark-1-pro' | 'spark-1-mini';
   expiresAt: string;
   creditsUsed?: number;
 }

apps/python-sdk/firecrawl/__init__.py

@@ -17,7 +17,7 @@
     V1ChangeTrackingOptions,
 )
 
-__version__ = "4.13.3"
+__version__ = "4.13.4"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

apps/python-sdk/firecrawl/v2/types.py

@@ -804,6 +804,7 @@ class AgentResponse(BaseModel):
     status: Optional[Literal["processing", "completed", "failed"]] = None
     data: Optional[Any] = None
     error: Optional[str] = None
+    model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None
     expires_at: Optional[datetime] = None
     credits_used: Optional[int] = None