diff --git a/README.md b/README.md index f6d33589..b7580654 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ Install using [npm](https://docs.npmjs.com/about-npm/): ```bash -npm install @apidevtools/json-schema-ref-parser -yarn add @apidevtools/json-schema-ref-parser -bun add @apidevtools/json-schema-ref-parser +npm install @hey-api/json-schema-ref-parser +yarn add @hey-api/json-schema-ref-parser +bun add @hey-api/json-schema-ref-parser ``` ## The Problem: @@ -69,21 +69,61 @@ JavaScript objects. ## Example ```javascript -import $RefParser from "@apidevtools/json-schema-ref-parser"; +import { $RefParser } from "@hey-api/json-schema-ref-parser"; try { - await $RefParser.dereference(mySchema); - // note - by default, mySchema is modified in place, and the returned value is a reference to the same object - console.log(mySchema.definitions.person.properties.firstName); - - // if you want to avoid modifying the original schema, you can disable the `mutateInputSchema` option - let clonedSchema = await $RefParser.dereference(mySchema, { mutateInputSchema: false }); - console.log(clonedSchema.definitions.person.properties.firstName); + const parser = new $RefParser(); + await parser.dereference({ pathOrUrlOrSchema: mySchema }); + console.log(parser.schema.definitions.person.properties.firstName); } catch (err) { console.error(err); } ``` +### New in this fork (@hey-api) + +- **Multiple inputs with `bundleMany`**: Merge and bundle several OpenAPI/JSON Schema inputs (files, URLs, or raw objects) into a single schema. Components are prefixed to avoid name collisions, paths are namespaced on conflict, and `$ref`s are rewritten accordingly. + +```javascript +import { $RefParser } from "@hey-api/json-schema-ref-parser"; + +const parser = new $RefParser(); +const merged = await parser.bundleMany({ + pathOrUrlOrSchemas: [ + "./specs/a.yaml", + "https://example.com/b.yaml", + { openapi: "3.1.0", info: { title: "Inline" }, paths: {} }, + ], +}); + +// merged.components.* will contain prefixed names like a_, b_, etc. +``` + +- **Dereference hooks**: Fine-tune dereferencing with `excludedPathMatcher(path) => boolean` to skip subpaths and `onDereference(path, value, parent, parentPropName)` to observe replacements. + +```javascript +const parser = new $RefParser(); +parser.options.dereference.excludedPathMatcher = (p) => p.includes("/example/"); +parser.options.dereference.onDereference = (p, v) => { + // inspect p / v as needed +}; +await parser.dereference({ pathOrUrlOrSchema: "./openapi.yaml" }); +``` + +- **Smart input resolution**: You can pass a file path, URL, or raw schema object. If a raw schema includes `$id`, it is used as the base URL for resolving relative `$ref`s. + +```javascript +await new $RefParser().bundle({ + pathOrUrlOrSchema: { + $id: "https://api.example.com/openapi.json", + openapi: "3.1.0", + paths: { + "/ping": { get: { responses: { 200: { description: "ok" } } } }, + }, + }, +}); +``` + ## Polyfills If you are using Node.js < 18, you'll need a polyfill for `fetch`, diff --git a/lib/bundle.ts b/lib/bundle.ts index 6652b80f..4bc45afd 100644 --- a/lib/bundle.ts +++ b/lib/bundle.ts @@ -607,7 +607,21 @@ function remap(parser: $RefParser, inventory: InventoryEntry[]) { let defName = namesForPrefix.get(targetKey); if (!defName) { - const proposed = `${baseName(entry.file)}_${lastToken(entry.hash)}`; + // If the external file is one of the original input sources, prefer its assigned prefix + let proposedBase = baseName(entry.file); + try { + const parserAny: any = parser as any; + if (parserAny && parserAny.sourcePathToPrefix && typeof parserAny.sourcePathToPrefix.get === "function") { + const withoutHash = (entry.file || "").split("#")[0]; + const mapped = parserAny.sourcePathToPrefix.get(withoutHash); + if (mapped && typeof mapped === "string") { + proposedBase = mapped; + } + } + } catch { + // Ignore errors + } + const proposed = `${proposedBase}_${lastToken(entry.hash)}`; defName = uniqueName(container, proposed); namesForPrefix.set(targetKey, defName); // Store the resolved value under the container diff --git a/lib/index.ts b/lib/index.ts index cf4db3f0..91c4025c 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -62,6 +62,15 @@ export const getResolvedInput = ({ return resolvedInput; }; +const _ensureResolvedInputPath = (input: ResolvedInput, fallbackPath: string): ResolvedInput => { + if (input.type === "json" && (!input.path || input.path.length === 0)) { + return { ...input, path: fallbackPath }; + } + return input; +}; + +// NOTE: previously used helper removed as unused + /** * This class parses a JSON schema, builds a map of its JSON references and their resolved values, * and provides methods for traversing, manipulating, and dereferencing those references. @@ -82,6 +91,9 @@ export class $RefParser { * @readonly */ public schema: JSONSchema | null = null; + public schemaMany: JSONSchema[] = []; + public schemaManySources: string[] = []; + public sourcePathToPrefix: Map = new Map(); /** * Bundles all referenced files/URLs into a single schema that only has internal `$ref` pointers. This lets you split-up your schema however you want while you're building it, but easily combine all those files together when it's time to package or distribute the schema to other people. The resulting schema size will be small, since it will still contain internal JSON references rather than being fully-dereferenced. @@ -109,6 +121,7 @@ export class $RefParser { pathOrUrlOrSchema, resolvedInput, }); + await resolveExternal(this, this.options); const errors = JSONParserErrorGroup.getParserErrors(this); if (errors.length > 0) { @@ -122,6 +135,39 @@ export class $RefParser { return this.schema!; } + /** + * Bundles multiple roots (files/URLs/objects) into a single schema by creating a synthetic root + * that references each input, resolving all externals, and then hoisting via the existing bundler. + */ + public async bundleMany({ + arrayBuffer, + fetch, + pathOrUrlOrSchemas, + resolvedInputs, + }: { + arrayBuffer?: ArrayBuffer[]; + fetch?: RequestInit; + pathOrUrlOrSchemas: Array; + resolvedInputs?: ResolvedInput[]; + }): Promise { + await this.parseMany({ arrayBuffer, fetch, pathOrUrlOrSchemas, resolvedInputs }); + this.mergeMany(); + + await resolveExternal(this, this.options); + const errors = JSONParserErrorGroup.getParserErrors(this); + if (errors.length > 0) { + throw new JSONParserErrorGroup(this); + } + _bundle(this, this.options); + // Merged root is ready for bundling + + const errors2 = JSONParserErrorGroup.getParserErrors(this); + if (errors2.length > 0) { + throw new JSONParserErrorGroup(this); + } + return this.schema!; + } + /** * Dereferences all `$ref` pointers in the JSON Schema, replacing each reference with its resolved value. This results in a schema object that does not contain any `$ref` pointers. Instead, it's a normal JavaScript object tree that can easily be crawled and used just like any other JavaScript object. This is great for programmatic usage, especially when using tools that don't understand JSON references. * @@ -223,6 +269,316 @@ export class $RefParser { schema, }; } + + private async parseMany({ + arrayBuffer, + fetch, + pathOrUrlOrSchemas, + resolvedInputs: _resolvedInputs, + }: { + arrayBuffer?: ArrayBuffer[]; + fetch?: RequestInit; + pathOrUrlOrSchemas: Array; + resolvedInputs?: ResolvedInput[]; + }): Promise<{ schemaMany: JSONSchema[] }> { + const resolvedInputs = [...(_resolvedInputs || [])]; + resolvedInputs.push(...(pathOrUrlOrSchemas.map((schema) => getResolvedInput({ pathOrUrlOrSchema: schema })) || [])); + + this.schemaMany = []; + this.schemaManySources = []; + this.sourcePathToPrefix = new Map(); + + for (let i = 0; i < resolvedInputs.length; i++) { + const resolvedInput = resolvedInputs[i]; + const { path, type } = resolvedInput; + let { schema } = resolvedInput; + + if (schema) { + // keep schema as-is + } else if (type !== "json") { + const file = newFile(path); + + // Add a new $Ref for this file, even though we don't have the value yet. + // This ensures that we don't simultaneously read & parse the same file multiple times + const $refAdded = this.$refs._add(file.url); + $refAdded.pathType = type; + try { + const resolver = type === "file" ? fileResolver : urlResolver; + await resolver.handler({ + arrayBuffer: arrayBuffer?.[i], + fetch, + file, + }); + const parseResult = await parseFile(file, this.options); + $refAdded.value = parseResult.result; + schema = parseResult.result; + } catch (err) { + if (isHandledError(err)) { + $refAdded.value = err; + } + + throw err; + } + } + + if (schema === null || typeof schema !== "object" || Buffer.isBuffer(schema)) { + throw ono.syntax(`"${this.$refs._root$Ref.path || schema}" is not a valid JSON Schema`); + } + + this.schemaMany.push(schema); + this.schemaManySources.push(path && path.length ? path : url.cwd()); + } + + return { + schemaMany: this.schemaMany, + }; + } + + public mergeMany(): JSONSchema { + const schemas = this.schemaMany || []; + if (schemas.length === 0) { + throw ono("mergeMany called with no schemas. Did you run parseMany?"); + } + + const merged: any = {}; + + // Determine spec version: prefer first occurrence of openapi, else swagger + let chosenOpenapi: string | undefined; + let chosenSwagger: string | undefined; + for (const s of schemas) { + if (!chosenOpenapi && s && typeof (s as any).openapi === "string") { + chosenOpenapi = (s as any).openapi; + } + if (!chosenSwagger && s && typeof (s as any).swagger === "string") { + chosenSwagger = (s as any).swagger; + } + if (chosenOpenapi && chosenSwagger) { + break; + } + } + if (typeof chosenOpenapi === "string") { + merged.openapi = chosenOpenapi; + } else if (typeof chosenSwagger === "string") { + merged.swagger = chosenSwagger; + } + + // Merge info: take first non-empty per-field across inputs + const infoAccumulator: any = {}; + for (const s of schemas) { + const info = (s as any)?.info; + if (info && typeof info === "object") { + for (const [k, v] of Object.entries(info)) { + if (infoAccumulator[k] === undefined && v !== undefined) { + infoAccumulator[k] = JSON.parse(JSON.stringify(v)); + } + } + } + } + if (Object.keys(infoAccumulator).length > 0) { + merged.info = infoAccumulator; + } + + // Merge servers: union by url+description + const servers: any[] = []; + const seenServers = new Set(); + for (const s of schemas) { + const arr = (s as any)?.servers; + if (Array.isArray(arr)) { + for (const srv of arr) { + if (srv && typeof srv === "object") { + const key = `${srv.url || ""}|${srv.description || ""}`; + if (!seenServers.has(key)) { + seenServers.add(key); + servers.push(JSON.parse(JSON.stringify(srv))); + } + } + } + } + } + if (servers.length > 0) { + merged.servers = servers; + } + + merged.paths = {}; + merged.components = {}; + + const componentSections = [ + "schemas", + "parameters", + "requestBodies", + "responses", + "headers", + "securitySchemes", + "examples", + "links", + "callbacks", + ]; + for (const sec of componentSections) { + merged.components[sec] = {}; + } + + const tagNameSet = new Set(); + const tags: any[] = []; + const usedOpIds = new Set(); + + const baseName = (p: string) => { + try { + const withoutHash = p.split("#")[0]; + const parts = withoutHash.split("/"); + const filename = parts[parts.length - 1] || "schema"; + const dot = filename.lastIndexOf("."); + const raw = dot > 0 ? filename.substring(0, dot) : filename; + return raw.replace(/[^A-Za-z0-9_-]/g, "_"); + } catch { + return "schema"; + } + }; + const unique = (set: Set, proposed: string) => { + let name = proposed; + let i = 2; + while (set.has(name)) { + name = `${proposed}_${i++}`; + } + set.add(name); + return name; + }; + + const rewriteRef = (ref: string, refMap: Map): string => { + // OAS3: #/components/{section}/{name}... + let m = ref.match(/^#\/components\/([^/]+)\/([^/]+)(.*)$/); + if (m) { + const base = `#/components/${m[1]}/${m[2]}`; + const mapped = refMap.get(base); + if (mapped) { + return mapped + (m[3] || ""); + } + } + // OAS2: #/definitions/{name}... + m = ref.match(/^#\/definitions\/([^/]+)(.*)$/); + if (m) { + const base = `#/components/schemas/${m[1]}`; + const mapped = refMap.get(base); + if (mapped) { + // map definitions -> components/schemas + return mapped + (m[2] || ""); + } + } + return ref; + }; + + const cloneAndRewrite = ( + obj: any, + refMap: Map, + tagMap: Map, + opIdPrefix: string, + basePath: string, + ): any => { + if (obj === null || obj === undefined) { + return obj; + } + if (Array.isArray(obj)) { + return obj.map((v) => cloneAndRewrite(v, refMap, tagMap, opIdPrefix, basePath)); + } + if (typeof obj !== "object") { + return obj; + } + + const out: any = {}; + for (const [k, v] of Object.entries(obj)) { + if (k === "$ref" && typeof v === "string") { + const s = v as string; + if (s.startsWith("#")) { + out[k] = rewriteRef(s, refMap); + } else { + const proto = url.getProtocol(s); + if (proto === undefined) { + // relative external ref -> absolutize against source base path + out[k] = url.resolve(basePath + "#", s); + } else { + out[k] = s; + } + } + } else if (k === "tags" && Array.isArray(v) && v.every((x) => typeof x === "string")) { + out[k] = v.map((t) => tagMap.get(t) || t); + } else if (k === "operationId" && typeof v === "string") { + out[k] = unique(usedOpIds, `${opIdPrefix}_${v}`); + } else { + out[k] = cloneAndRewrite(v as any, refMap, tagMap, opIdPrefix, basePath); + } + } + return out; + }; + + for (let i = 0; i < schemas.length; i++) { + const schema: any = schemas[i] || {}; + const sourcePath = this.schemaManySources[i] || `multi://input/${i + 1}`; + const prefix = baseName(sourcePath); + + // Track prefix for this source path (strip hash). Only map real file/http paths + const withoutHash = url.stripHash(sourcePath); + const protocol = url.getProtocol(withoutHash); + if (protocol === undefined || protocol === "file" || protocol === "http" || protocol === "https") { + this.sourcePathToPrefix.set(withoutHash, prefix); + } + + const refMap = new Map(); + const tagMap = new Map(); + + const srcComponents = (schema.components || {}) as any; + for (const sec of componentSections) { + const group = srcComponents[sec] || {}; + for (const [name] of Object.entries(group)) { + const newName = `${prefix}_${name}`; + refMap.set(`#/components/${sec}/${name}`, `#/components/${sec}/${newName}`); + } + } + + const srcTags: any[] = Array.isArray(schema.tags) ? schema.tags : []; + for (const t of srcTags) { + if (!t || typeof t !== "object" || typeof t.name !== "string") { + continue; + } + const desired = t.name; + const finalName = tagNameSet.has(desired) ? `${prefix}_${desired}` : desired; + tagNameSet.add(finalName); + tagMap.set(desired, finalName); + if (!tags.find((x) => x && x.name === finalName)) { + tags.push({ ...t, name: finalName }); + } + } + + for (const sec of componentSections) { + const group = (schema.components && schema.components[sec]) || {}; + for (const [name, val] of Object.entries(group)) { + const newName = `${prefix}_${name}`; + merged.components[sec][newName] = cloneAndRewrite(val, refMap, tagMap, prefix, url.stripHash(sourcePath)); + } + } + + const srcPaths = (schema.paths || {}) as Record; + for (const [p, item] of Object.entries(srcPaths)) { + let targetPath = p; + if (merged.paths[p]) { + const trimmed = p.startsWith("/") ? p.substring(1) : p; + targetPath = `/${prefix}/${trimmed}`; + } + merged.paths[targetPath] = cloneAndRewrite(item, refMap, tagMap, prefix, url.stripHash(sourcePath)); + } + } + + if (tags.length > 0) { + merged.tags = tags; + } + + // Rebuild $refs root using the first input's path to preserve external resolution semantics + const rootPath = this.schemaManySources[0] || url.cwd(); + this.$refs = new $Refs(); + const rootRef = this.$refs._add(rootPath); + rootRef.pathType = url.isFileSystemPath(rootPath) ? "file" : "http"; + rootRef.value = merged; + this.schema = merged; + return merged as JSONSchema; + } } export { sendRequest } from "./resolvers/url.js"; diff --git a/lib/resolve-external.ts b/lib/resolve-external.ts index 6adc4481..e3bfb703 100644 --- a/lib/resolve-external.ts +++ b/lib/resolve-external.ts @@ -21,10 +21,7 @@ import { urlResolver } from "./resolvers/url.js"; * The promise resolves once all JSON references in the schema have been resolved, * including nested references that are contained in externally-referenced files. */ -export function resolveExternal( - parser: $RefParser, - options: $RefParserOptions, -) { +export function resolveExternal(parser: $RefParser, options: $RefParserOptions) { try { // console.log('Resolving $ref pointers in %s', parser.$refs._root$Ref.path); const promises = crawl(parser.schema, parser.$refs._root$Ref.path + "#", parser.$refs, options); @@ -101,29 +98,33 @@ async function resolve$Ref( // $ref.$ref = url.relative($refs._root$Ref.path, resolvedPath); + // If this ref points back to an input source we've already merged, avoid re-importing + // by checking if the path (without hash) matches a known source in parser and we can serve it internally later. + // We keep normal flow but ensure cache hit if already added. // Do we already have this $ref? const ref = $refs._$refs[withoutHash]; if (ref) { - // We've already parsed this $ref, so use the existing value - return Promise.resolve(ref.value); + // We've already parsed this $ref, so crawl it to resolve its own externals + const promises = crawl(ref.value as S, `${withoutHash}#`, $refs, options, new Set(), true); + return Promise.all(promises); } // Parse the $referenced file/url - const file = newFile(resolvedPath) + const file = newFile(resolvedPath); // Add a new $Ref for this file, even though we don't have the value yet. // This ensures that we don't simultaneously read & parse the same file multiple times const $refAdded = $refs._add(file.url); try { - const resolvedInput = getResolvedInput({ pathOrUrlOrSchema: resolvedPath }) + const resolvedInput = getResolvedInput({ pathOrUrlOrSchema: resolvedPath }); $refAdded.pathType = resolvedInput.type; let promises: any = []; - if (resolvedInput.type !== 'json') { - const resolver = resolvedInput.type === 'file' ? fileResolver : urlResolver; + if (resolvedInput.type !== "json") { + const resolver = resolvedInput.type === "file" ? fileResolver : urlResolver; await resolver.handler({ file }); const parseResult = await parseFile(file, options); $refAdded.value = parseResult.result; diff --git a/package.json b/package.json index 863d8879..30436121 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hey-api/json-schema-ref-parser", - "version": "1.0.3", + "version": "1.2.0", "description": "Parse, Resolve, and Dereference JSON Schema $ref pointers", "homepage": "https://heyapi.dev/", "repository": {