diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.yml index 23cc2de23..ea925b4e6 100644 --- a/.github/workflows/test-js-sdk.yml +++ b/.github/workflows/test-js-sdk.yml @@ -8,7 +8,7 @@ on: - apps/js-sdk/firecrawl/** env: - TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + IDMUX_URL: ${{ secrets.IDMUX_URL }} jobs: test: diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts new file mode 100644 index 000000000..d2bfe77ad --- /dev/null +++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts @@ -0,0 +1,77 @@ +import type { Response } from "express"; +import { agentStatusController } from "../agent-status"; +import type { RequestWithAuth } from "../types"; +import { + supabaseGetAgentByIdDirect, + supabaseGetAgentRequestByIdDirect, +} from "../../../lib/supabase-jobs"; +import { getJobFromGCS } from "../../../lib/gcs-jobs"; + +jest.mock("../../../lib/supabase-jobs", () => ({ + supabaseGetAgentByIdDirect: jest.fn(), + supabaseGetAgentRequestByIdDirect: jest.fn(), +})); + +jest.mock("../../../lib/gcs-jobs", () => ({ + getJobFromGCS: jest.fn(), +})); + +describe("agentStatusController", () => { + const baseReq = { + params: { jobId: "job-123" }, + auth: { team_id: "team-123" }, + } as RequestWithAuth<{ jobId: string }, any, any>; + + const buildRes = () => + ({ + status: jest.fn().mockReturnThis(), + json: jest.fn(), + }) as unknown as Response; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("returns model from agent options", async () => { + (supabaseGetAgentRequestByIdDirect as jest.Mock).mockResolvedValue({ + team_id: "team-123", + created_at: "2025-01-01T00:00:00Z", + }); + (supabaseGetAgentByIdDirect as jest.Mock).mockResolvedValue({ + id: "job-123", + is_successful: true, + options: { model: "spark-1-mini" }, + created_at: "2025-01-01T00:00:00Z", + }); + (getJobFromGCS as jest.Mock).mockResolvedValue({ result: "ok" }); + + const res = buildRes(); + await agentStatusController(baseReq, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ model: "spark-1-mini" }), + ); + }); + + it("defaults model to spark-1-pro when missing", async () => { + (supabaseGetAgentRequestByIdDirect as jest.Mock).mockResolvedValue({ + team_id: "team-123", + created_at: "2025-01-01T00:00:00Z", + }); + (supabaseGetAgentByIdDirect as jest.Mock).mockResolvedValue({ + id: "job-123", + is_successful: false, + options: null, + created_at: "2025-01-01T00:00:00Z", + }); + + const res = buildRes(); + await agentStatusController(baseReq, res); + + expect(res.status).toHaveBeenCalledWith(200); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ model: "spark-1-pro" }), + ); + }); +}); diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.ts index a2c99ce5f..482445f07 100644 --- a/apps/api/src/controllers/v2/agent-status.ts +++ b/apps/api/src/controllers/v2/agent-status.ts @@ -4,8 +4,9 @@ import { supabaseGetAgentByIdDirect, supabaseGetAgentRequestByIdDirect, } from "../../lib/supabase-jobs"; -import { logger as _logger } from "../../lib/logger"; +import { logger as _logger, logger } from "../../lib/logger"; import { getJobFromGCS } from "../../lib/gcs-jobs"; +import { config } from "../../config"; export async function agentStatusController( req: RequestWithAuth<{ jobId: string }, AgentStatusResponse, any>, @@ -24,6 +25,49 @@ export async function agentStatusController( const agent = await supabaseGetAgentByIdDirect(req.params.jobId); + let model: "spark-1-pro" | "spark-1-mini"; + if (agent) { + model = (agent.options?.model ?? "spark-1-pro") as + | "spark-1-pro" + | "spark-1-mini"; + } else { + try { + const optionsRequest = await fetch( + config.EXTRACT_V3_BETA_URL + + "/v2/extract/" + + req.params.jobId + + "/options", + { + headers: { + Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`, + }, + }, + ); + + if (optionsRequest.status !== 200) { + logger.warn("Failed to get agent request details", { + status: optionsRequest.status, + method: "agentStatusController", + module: "api/v2", + text: await optionsRequest.text(), + }); + model = "spark-1-pro"; // fall back to this value + } else { + model = ((await optionsRequest.json()).model ?? "spark-1-pro") as + | "spark-1-pro" + | "spark-1-mini"; + } + } catch (error) { + logger.warn("Failed to get agent request details", { + error, + method: "agentStatusController", + module: "api/v2", + extractId: req.params.jobId, + }); + model = "spark-1-pro"; // fall back to this value + } + } + let data: any = undefined; if (agent?.is_successful) { data = await getJobFromGCS(agent.id); @@ -38,6 +82,7 @@ export async function agentStatusController( : "failed", error: agent?.error || undefined, data, + model, expiresAt: new Date( new Date(agent?.created_at ?? agentRequest.created_at).getTime() + 1000 * 60 * 60 * 24, diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index 4cb2b3beb..1273b92c0 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -1133,6 +1133,7 @@ export type AgentStatusResponse = status: "processing" | "completed" | "failed"; error?: string; data?: any; + model?: "spark-1-pro" | "spark-1-mini"; expiresAt: string; creditsUsed?: number; }; diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index 9357ae19f..fb0479336 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.11.3", + "version": "4.11.4", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts index 53c4f12ae..e44663768 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts @@ -128,9 +128,9 @@ describe("v2.scrape e2e", () => { }); expect(doc.images).toBeTruthy(); expect(Array.isArray(doc.images)).toBe(true); - expect(doc.images.length).toBeGreaterThan(0); + expect(doc.images?.length).toBeGreaterThan(0); // Should find firecrawl logo/branding images - expect(doc.images.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true); + expect(doc.images?.some(img => img.includes("firecrawl") || img.includes("logo"))).toBe(true); }, 60_000); test("images format: works with multiple formats", async () => { @@ -142,7 +142,7 @@ describe("v2.scrape e2e", () => { expect(doc.links).toBeTruthy(); expect(doc.images).toBeTruthy(); expect(Array.isArray(doc.images)).toBe(true); - expect(doc.images.length).toBeGreaterThan(0); + expect(doc.images?.length).toBeGreaterThan(0); // Images should find things not available in links format const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico']; @@ -151,7 +151,7 @@ describe("v2.scrape e2e", () => { ) || []; // Should discover additional images beyond those with obvious extensions - expect(doc.images.length).toBeGreaterThanOrEqual(linkImages.length); + expect(doc.images?.length).toBeGreaterThanOrEqual(linkImages.length); }, 60_000); test("invalid url should throw", async () => { diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts index 3d8ecd4d1..b1b31ffd2 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts @@ -2,7 +2,7 @@ * E2E tests for v2 search (translated from Python tests) */ import Firecrawl from "../../../index"; -import type { Document, SearchResult } from "../../../index"; +import type { Document, SearchResultWeb, SearchResultNews, SearchResultImages } from "../../../index"; import { config } from "dotenv"; import { getIdentity, getApiUrl } from "./utils/idmux"; import { describe, test, expect, beforeAll } from "@jest/globals"; @@ -28,7 +28,7 @@ function collectTexts(entries: any[] | undefined): string[] { return texts; } -function isDocument(entry: Document | SearchResult | undefined | null): entry is Document { +function isDocument(entry: Document | SearchResultWeb | SearchResultNews | SearchResultImages | undefined | null): entry is Document { if (!entry) return false; const d = entry as Document; return ( @@ -86,10 +86,10 @@ describe("v2.search e2e", () => { expect(results.images == null).toBe(true); const webTitles = (results.web || []) - .filter((r): r is SearchResult => !isDocument(r)) + .filter((r): r is SearchResultWeb => !isDocument(r)) .map(r => (r.title || "").toString().toLowerCase()); const webDescriptions = (results.web || []) - .filter((r): r is SearchResult => !isDocument(r)) + .filter((r): r is SearchResultWeb => !isDocument(r)) .map(r => (r.description || "").toString().toLowerCase()); const allWebText = (webTitles.concat(webDescriptions)).join(" "); expect(allWebText.includes("firecrawl")).toBe(true); @@ -183,7 +183,7 @@ describe("v2.search e2e", () => { expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true); } else { expect(typeof result.url).toBe("string"); - expect(result.url.startsWith("http")).toBe(true); + expect(result.url?.startsWith("http")).toBe(true); } } } @@ -193,7 +193,7 @@ describe("v2.search e2e", () => { for (const result of results.images || []) { if (!isDocument(result)) { expect(typeof result.url).toBe("string"); - expect(result.url.startsWith("http")).toBe(true); + expect(result.url?.startsWith("http")).toBe(true); } } }, 120_000); diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts index 8cbb61752..e11d7527e 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts @@ -23,16 +23,15 @@ describe("v2.usage e2e", () => { expect(typeof resp.maxConcurrency).toBe("number"); }, 60_000); - // NOTE: Disabled, broken on central team due to overflow - // test("get_credit_usage", async () => { - // const resp = await client.getCreditUsage(); - // expect(typeof resp.remainingCredits).toBe("number"); - // }, 60_000); - - // test("get_token_usage", async () => { - // const resp = await client.getTokenUsage(); - // expect(typeof resp.remainingTokens).toBe("number"); - // }, 60_000); + test("get_credit_usage", async () => { + const resp = await client.getCreditUsage(); + expect(typeof resp.remainingCredits).toBe("number"); + }, 60_000); + + test("get_token_usage", async () => { + const resp = await client.getTokenUsage(); + expect(typeof resp.remainingTokens).toBe("number"); + }, 60_000); test("get_queue_status", async () => { const resp = await client.getQueueStatus(); diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index 02be36336..b3ec7523e 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -556,6 +556,7 @@ export interface AgentStatusResponse { status: 'processing' | 'completed' | 'failed'; error?: string; data?: unknown; + model?: 'spark-1-pro' | 'spark-1-mini'; expiresAt: string; creditsUsed?: number; } diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 76deaa9be..fbea47891 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ V1ChangeTrackingOptions, ) -__version__ = "4.13.3" +__version__ = "4.13.4" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py index 297332a04..6439c3142 100644 --- a/apps/python-sdk/firecrawl/v2/types.py +++ b/apps/python-sdk/firecrawl/v2/types.py @@ -804,6 +804,7 @@ class AgentResponse(BaseModel): status: Optional[Literal["processing", "completed", "failed"]] = None data: Optional[Any] = None error: Optional[str] = None + model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None expires_at: Optional[datetime] = None credits_used: Optional[int] = None