Fix bugs with remote $refs, add cwd option for JSON schema parsing

Author: drwpow

.changeset/silly-maps-rush.md

@@ -0,0 +1,5 @@
+---
+"openapi-typescript": patch
+---
+
+Fix bugs with remote $refs, add `cwd` option for JSON schema parsing

docs/src/content/docs/node.md

@@ -43,12 +43,13 @@ const output = await openapiTS("https://myurl.com/v1/openapi.yaml");
 
 The Node API supports all the [CLI flags](/cli#options) in `camelCase` format, plus the following additional options:
 
-| Name            |    Type    | Default | Description                                                                      |
-| :-------------- | :--------: | :------ | :------------------------------------------------------------------------------- |
-| `commentHeader` |  `string`  |         | Override the default “This file was auto-generated …” file heading               |
-| `inject`        |  `string`  |         | Inject arbitrary TypeScript types into the start of the file                     |
-| `transform`     | `Function` |         | Override the default Schema Object ➝ TypeScript transformer in certain scenarios |
-| `postTransform` | `Function` |         | Same as `transform` but runs _after_ the TypeScript transformation               |
+| Name            |      Type       | Default | Description                                                                                                          |
+| :-------------- | :-------------: | :------ | :------------------------------------------------------------------------------------------------------------------- |
+| `commentHeader` |    `string`     |         | Override the default “This file was auto-generated …” file heading                                                   |
+| `inject`        |    `string`     |         | Inject arbitrary TypeScript types into the start of the file                                                         |
+| `transform`     |   `Function`    |         | Override the default Schema Object ➝ TypeScript transformer in certain scenarios                                     |
+| `postTransform` |   `Function`    |         | Same as `transform` but runs _after_ the TypeScript transformation                                                   |
+| `cwd`           | `string \| URL` |         | (optional) Provide the current working directory to resolve remote `$ref`s (only needed for in-memory JSON objects). |
 
 ### transform / postTransform
 

packages/openapi-typescript/src/index.ts

@@ -1,4 +1,4 @@
-import type { GlobalContext, OpenAPI3, OpenAPITSOptions, SchemaObject, Subschema } from "./types.js";
+import type { GlobalContext, OpenAPI3, OpenAPITSOptions, ParameterObject, SchemaObject, Subschema } from "./types.js";
 import type { Readable } from "node:stream";
 import { URL } from "node:url";
 import load, { resolveSchema, VIRTUAL_JSON_URL } from "./load.js";
@@ -40,6 +40,7 @@ async function openapiTS(schema: string | URL | OpenAPI3 | Readable, options: Op
   const ctx: GlobalContext = {
     additionalProperties: options.additionalProperties ?? false,
     alphabetize: options.alphabetize ?? false,
+    cwd: options.cwd ?? new URL(`file://${process.cwd()}/`),
     defaultNonNullable: options.defaultNonNullable ?? false,
     discriminators: {},
     transform: typeof options.transform === "function" ? options.transform : undefined,
@@ -55,17 +56,31 @@ async function openapiTS(schema: string | URL | OpenAPI3 | Readable, options: Op
     excludeDeprecated: options.excludeDeprecated ?? false,
   };
 
-  // note: we may be loading many large schemas into memory at once; take care to reuse references without cloning
-  const isInlineSchema = typeof schema !== "string" && schema instanceof URL === false; // eslint-disable-line @typescript-eslint/no-unnecessary-boolean-literal-compare
-
   // 1. load schema (and subschemas)
   const allSchemas: { [id: string]: Subschema } = {};
   const schemaURL: URL = typeof schema === "string" ? resolveSchema(schema) : (schema as URL);
+  let rootURL: URL = schemaURL;
+
+  // 1a. if passed as in-memory JSON, handle `cwd` option
+  const isInlineSchema = typeof schema !== "string" && schema instanceof URL === false; // eslint-disable-line @typescript-eslint/no-unnecessary-boolean-literal-compare
+  if (isInlineSchema) {
+    if (ctx.cwd) {
+      if (ctx.cwd instanceof URL) {
+        rootURL = ctx.cwd;
+      } else if (typeof ctx.cwd === "string") {
+        rootURL = new URL(ctx.cwd, `file://${process.cwd()}/`);
+      }
+      rootURL = new URL("root.yaml", rootURL); // give the root schema an arbitrary filename ("root.yaml")
+    } else {
+      rootURL = new URL(VIRTUAL_JSON_URL); // otherwise, set virtual filename (which prevents resolutions)
+    }
+  }
+
   await load(schemaURL, {
     ...ctx,
     auth: options.auth,
     schemas: allSchemas,
-    rootURL: isInlineSchema ? new URL(VIRTUAL_JSON_URL) : schemaURL, // if an inline schema is passed, use virtual URL
+    rootURL,
     urlCache: new Set(),
     httpHeaders: options.httpHeaders,
     httpMethod: options.httpMethod,
@@ -185,7 +200,7 @@ async function openapiTS(schema: string | URL | OpenAPI3 | Readable, options: Op
             const c = getSchemaObjectComment(schemaObject as SchemaObject, indentLv);
             if (c) subschemaOutput += indent(c, indentLv);
 
-            // This might be a Path Item Object; only way to test is if top-level contains a method (not allowed on Schema Object)
+            // Test for Path Item Object
             if (!("type" in schemaObject) && !("$ref" in schemaObject)) {
               for (const method of ["get", "put", "post", "delete", "options", "head", "patch", "trace"] as Method[]) {
                 if (method in schemaObject) {
@@ -194,6 +209,11 @@ async function openapiTS(schema: string | URL | OpenAPI3 | Readable, options: Op
                 }
               }
             }
+            // Test for Parameter
+            if ("in" in schemaObject) {
+              subschemaOutput += indent(`${escObjKey(name)}: ${transformParameterObject(schemaObject as ParameterObject, { path: `${path}${name}`, ctx: { ...ctx, indentLv } })};\n`, indentLv);
+              continue;
+            }
 
             // Otherwise, this is a Schema Object
             subschemaOutput += indent(`${escObjKey(name)}: ${transformSchemaObject(schemaObject, { path: `${path}${name}`, ctx: { ...ctx, indentLv } })};\n`, indentLv);
@@ -204,7 +224,7 @@ async function openapiTS(schema: string | URL | OpenAPI3 | Readable, options: Op
           break;
         }
         case "SchemaObject": {
-          subschemaOutput = transformSchemaObject(subschema.schema, { path, ctx: { ...ctx, indentLv } });
+          subschemaOutput = `${transformSchemaObject(subschema.schema, { path, ctx: { ...ctx, indentLv } })};`;
           break;
         }
         default: {

packages/openapi-typescript/src/load.ts

@@ -11,7 +11,7 @@ interface SchemaMap {
   [id: string]: Subschema;
 }
 
-const EXT_RE = /\.(yaml|yml|json)$/i;
+const EXT_RE = /\.(yaml|yml|json)#?\/?/i;
 export const VIRTUAL_JSON_URL = `file:///_json`; // fake URL reserved for dynamic JSON
 
 function parseYAML(schema: string) {
@@ -69,7 +69,7 @@ function parseHttpHeaders(httpHeaders: Record<string, unknown>): Record<string,
         const stringVal = JSON.stringify(v);
         finalHeaders[k] = stringVal;
       } catch (err) {
-        error(`Cannot parse key: ${k} into JSON format. Continuing with the next HTTP header that is specified`);
+        error(`Can’t parse key: ${k} into JSON format. Continuing with the next HTTP header that is specified`);
       }
     }
   }
@@ -99,9 +99,13 @@ export default async function load(schema: URL | Subschema | Readable, options:
     const hint = options.hint ?? "OpenAPI3";
 
     // normalize ID
-    if (schema.href !== options.rootURL.href) schemaID = relativePath(options.rootURL, schema);
+    if (schema.href !== options.rootURL.href) {
+      schemaID = relativePath(options.rootURL, schema);
+    }
 
-    if (options.urlCache.has(schemaID)) return options.schemas; // exit early if already indexed
+    if (options.urlCache.has(schemaID)) {
+      return options.schemas; // exit early if already indexed
+    }
     options.urlCache.add(schemaID);
 
     const ext = path.extname(schema.pathname).toLowerCase();
@@ -138,16 +142,17 @@ export default async function load(schema: URL | Subschema | Readable, options:
     // local file
     else {
       const contents = fs.readFileSync(schema, "utf8");
-      if (ext === ".yaml" || ext === ".yml")
+      if (ext === ".yaml" || ext === ".yml") {
         options.schemas[schemaID] = {
           hint,
           schema: parseYAML(contents) as any, // eslint-disable-line @typescript-eslint/no-explicit-any
         };
-      else if (ext === ".json")
+      } else if (ext === ".json") {
         options.schemas[schemaID] = {
           hint,
           schema: parseJSON(contents),
         };
+      }
     }
   }
   // 1b. Readable stream
@@ -225,30 +230,23 @@ export default async function load(schema: URL | Subschema | Readable, options:
     hintPath.push(...ref.path);
     const hint = isRemoteFullSchema ? "OpenAPI3" : getHint({ path: hintPath, external: !!ref.filename, startFrom: options.hint });
 
-    // if root schema is remote and this is a relative reference, treat as remote
-    if (schema instanceof URL) {
-      const nextURL = new URL(ref.filename, schema);
-      const nextID = relativePath(schema, nextURL);
-      if (options.urlCache.has(nextID)) return;
-      refPromises.push(load(nextURL, { ...options, hint }));
-      node.$ref = node.$ref.replace(ref.filename, nextID);
-      return;
-    }
-    // otherwise, if $ref is remote use that
     if (isRemoteURL(ref.filename) || isFilepath(ref.filename)) {
       const nextURL = new URL(ref.filename.startsWith("//") ? `https://${ref.filename}` : ref.filename);
-      if (options.urlCache.has(nextURL.href)) return;
       refPromises.push(load(nextURL, { ...options, hint }));
       node.$ref = node.$ref.replace(ref.filename, nextURL.href);
       return;
     }
-    // if this is dynamic JSON, we have no idea how to resolve external URLs, so throw here
+
+    // if this is dynamic JSON (with no cwd), we have no idea how to resolve external URLs, so throw here
     if (options.rootURL.href === VIRTUAL_JSON_URL) {
-      error(`Can’t resolve "${ref.filename}" from dynamic JSON. Load this schema from a URL instead.`);
+      error(`Can’t resolve "${ref.filename}" from dynamic JSON. Either load this schema from a filepath/URL, or set the \`cwd\` option: \`openapiTS(schema, { cwd: '/path/to/cwd' })\`.`);
       process.exit(1);
     }
-    error(`Can’t resolve "${ref.filename}"`);
-    process.exit(1);
+
+    const nextURL = new URL(ref.filename, schema instanceof URL ? schema : options.rootURL);
+    const nextID = relativePath(schema instanceof URL ? schema : options.rootURL, nextURL);
+    refPromises.push(load(nextURL, { ...options, hint }));
+    node.$ref = node.$ref.replace(ref.filename, nextID);
   });
   await Promise.all(refPromises);
 
@@ -322,7 +320,12 @@ export interface GetHintOptions {
   startFrom?: Subschema["hint"];
 }
 
-/** given a path array (an array of indices), what type of object is this? */
+/**
+ * Hinting
+ * A remote `$ref` may point to anything—A full OpenAPI schema, partial OpenAPI schema, Schema Object, Parameter Object, etc.
+ * The only way to parse its contents correctly is to trace the path from the root schema and infer the type it should be.
+ * “Hinting” is the process of tracing its lineage back to the root schema to invoke the correct transformations on it.
+ */
 export function getHint({ path, external, startFrom }: GetHintOptions): Subschema["hint"] | undefined {
   if (startFrom && startFrom !== "OpenAPI3") {
     switch (startFrom) {
@@ -332,6 +335,8 @@ export function getHint({ path, external, startFrom }: GetHintOptions): Subschem
         return getHintFromRequestBodyObject(path, external);
       case "ResponseObject":
         return getHintFromResponseObject(path, external);
+      case "SchemaMap":
+        return "SchemaObject";
       default:
         return startFrom;
     }

packages/openapi-typescript/src/types.ts

@@ -1,4 +1,4 @@
-import type { URL } from "node:url";
+import type { PathLike } from "node:fs";
 import type { RequestInfo, RequestInit, Response } from "undici";
 import type { TransformSchemaObjectOptions } from "./transform/schema-object.js";
 
@@ -614,7 +614,7 @@ export interface OpenAPITSOptions {
   /** Allow schema objects with no specified properties to have additional properties if not expressly forbidden? (default: false) */
   emptyObjectsUnknown?: boolean;
   /** Specify current working directory (cwd) to resolve remote schemas on disk (not needed for remote URL schemas) */
-  cwd?: URL;
+  cwd?: PathLike;
   /** Should schema objects with a default value not be considered optional? */
   defaultNonNullable?: boolean;
   /** Manually transform certain Schema Objects with a custom TypeScript type */
@@ -685,6 +685,7 @@ export type Subschema =
 export interface GlobalContext {
   additionalProperties: boolean;
   alphabetize: boolean;
+  cwd?: PathLike;
   emptyObjectsUnknown: boolean;
   defaultNonNullable: boolean;
   discriminators: { [$ref: string]: DiscriminatorObject };

packages/openapi-typescript/test/fixtures/remote-ref-test-2.yaml renamed to packages/openapi-typescript/test/fixtures/_remote-ref-full.yaml

@@ -0,0 +1,15 @@
+/ref-path:
+  get:
+    responses:
+      200:
+        description: OK
+        content:
+          application/json:
+            schema:
+              $ref: "./nested-ref-2/_nested-ref-2.yaml"
+StringParam:
+  type: parameter
+  required: true
+  in: path
+  schema:
+    type: string

packages/openapi-typescript/test/fixtures/_schema-test-partial.yaml renamed to packages/openapi-typescript/test/fixtures/_remote-ref-partial.yaml

@@ -0,0 +1,4 @@
+type: object
+properties:
+  string:
+    type: string

packages/openapi-typescript/test/fixtures/nested-ref/_nested-ref-partial.yaml

@@ -0,0 +1,2 @@
+
+$ref: '../_nested-ref-2.yaml'

packages/openapi-typescript/test/fixtures/nested-ref/nested-ref-2/_nested-ref-2.yaml

@@ -12,9 +12,16 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/RemoteType"
+  /ref-path:
+    $ref: "./nested-ref/_nested-ref-partial.yaml#~1ref-path"
 components:
+  parameters:
+    NestedParam:
+      $ref: "./nested-ref/_nested-ref-partial.yaml#StringParam"
   schemas:
+    NestedType:
+      $ref: "nested-ref/nested-ref-2/nested-ref-3/_nested-ref-3.yaml"
     RemoteType:
-      $ref: "./remote-ref-test-2.yaml#/components/schemas/SchemaType"
+      $ref: "_remote-ref-full.yaml#/components/schemas/SchemaType"
     RemotePartialType:
-      $ref: '_schema-test-partial.yaml#/PartialType'
+      $ref: "_remote-ref-partial.yaml#/PartialType"