fix(http,openapi3): support nested unions in operation return types (#8961)

Nested unions (e.g., `op read(): A | (B | C)` or using a named union
like `union Conflict { ... }`) were not being expanded recursively when
used as operation return types, causing responses to be missing from
the generated OpenAPI output.

This fix:
- Adds recursive union expansion in processResponseType() to handle
  nested unions at any level
- Updates getResponses() to group responses by status code before
  processing to properly handle cases where multiple union variants
  map to the same status code
- Ensures response descriptions are taken from the first variant when
  multiple variants share a status code

The most common case is named unions, but this also fixes any scenario
where unions are nested within other unions in the return type.

----

The reasons for why I want this fixed is because I want to be able to
control the description on responses when there is multiple responses on
the same status code. See
#8962 for details.

Author: tellnes

.chronus/changes/named-unions-response-type-2025-10-10-16-34-2.md

@@ -0,0 +1,8 @@
+---
+changeKind: fix
+packages:
+  - "@typespec/http"
+  - "@typespec/openapi3"
+---
+
+Support nested unions in operation return types

packages/http/src/responses.ts

@@ -82,6 +82,22 @@ function processResponseType(
   responses: ResponseIndex,
   responseType: Type,
 ) {
+  const tk = $(program);
+
+  // If the response type is itself a union (and not discriminated), expand it recursively.
+  // This handles cases where a named union is used as a return type (e.g., `op read(): MyUnion`)
+  // or when unions are nested (e.g., a union variant is itself a union).
+  // Each variant will be processed separately to extract its status codes and responses.
+  if (tk.union.is(responseType) && !tk.union.getDiscriminatedUnion(responseType)) {
+    for (const option of responseType.variants.values()) {
+      if (isNullType(option.type)) {
+        continue;
+      }
+      processResponseType(program, diagnostics, operation, responses, option.type);
+    }
+    return;
+  }
+
   // Get body
   let { body: resolvedBody, metadata } = diagnostics.pipe(
     resolveHttpPayload(program, responseType, Visibility.Read, HttpPayloadDisposition.Response),

packages/openapi3/src/openapi.ts

@@ -964,14 +964,35 @@ function createOAPIEmitter(
     responses: HttpOperationResponse[],
     examples: OperationExamples,
   ): Record<string, Refable<OpenAPI3Response>> {
-    const result: Record<string, Refable<OpenAPI3Response>> = {};
+    const responseMap = new Map<string, HttpOperationResponse[]>();
+
+    // Group responses by status code first. When named unions are expanded into individual
+    // response variants, multiple variants may map to the same status code. We need to collect
+    // all variants for each status code before processing to properly merge content types and
+    // select the appropriate description.
     for (const response of responses) {
       for (const statusCode of diagnostics.pipe(
         getOpenAPI3StatusCodes(program, response.statusCodes, response.type),
       )) {
-        result[statusCode] = getResponseForStatusCode(operation, statusCode, [response], examples);
+        if (responseMap.has(statusCode)) {
+          responseMap.get(statusCode)!.push(response);
+        } else {
+          responseMap.set(statusCode, [response]);
+        }
       }
     }
+
+    // Generate OpenAPI response for each status code
+    const result: Record<string, Refable<OpenAPI3Response>> = {};
+    for (const [statusCode, statusCodeResponses] of responseMap) {
+      result[statusCode] = getResponseForStatusCode(
+        operation,
+        statusCode,
+        statusCodeResponses,
+        examples,
+      );
+    }
+
     return result;
   }
 

packages/openapi3/test/response-descriptions.test.ts

@@ -61,4 +61,46 @@ worksFor(supportedVersions, ({ openApiFor }) => {
     strictEqual(res.paths["/"].get.responses["404"].description, "Not found model");
     strictEqual(res.paths["/"].get.responses["default"].description, "Generic error");
   });
+
+  it("uses first model's description when multiple models have same status code", async () => {
+    const res = await openApiFor(
+      `
+      @doc("Foo") model Foo { @statusCode _: 409 }
+      @doc("Bar") model Bar { @statusCode _: 409 }
+      op read(): { @statusCode _: 200, content: string } | Foo | Bar;
+      `,
+    );
+    strictEqual(res.paths["/"].get.responses["200"].description, "The request has succeeded.");
+    strictEqual(res.paths["/"].get.responses["409"].description, "Foo");
+  });
+
+  it("expands named union in return type and uses first variant's description", async () => {
+    const res = await openApiFor(
+      `
+      @doc("Foo") model Foo { @statusCode _: 409 }
+      @doc("Bar") model Bar { @statusCode _: 409 }
+      union Conflict { Foo: Foo; Bar: Bar };
+      op read(): { @statusCode _: 200, content: string } | Conflict;
+      `,
+    );
+    strictEqual(res.paths["/"].get.responses["200"].description, "The request has succeeded.");
+    strictEqual(res.paths["/"].get.responses["409"].description, "Foo");
+  });
+
+  it("recursively expands deeply nested unions", async () => {
+    const res = await openApiFor(
+      `
+      @doc("Model A") model A { @statusCode _: 400 }
+      @doc("Model B") model B { @statusCode _: 401 }
+      @doc("Model C") model C { @statusCode _: 403 }
+      union Inner { A: A; B: B };
+      union Outer { inner: Inner; C: C };
+      op read(): { @statusCode _: 200, content: string } | Outer;
+      `,
+    );
+    strictEqual(res.paths["/"].get.responses["200"].description, "The request has succeeded.");
+    strictEqual(res.paths["/"].get.responses["400"].description, "Model A");
+    strictEqual(res.paths["/"].get.responses["401"].description, "Model B");
+    strictEqual(res.paths["/"].get.responses["403"].description, "Model C");
+  });
 });

packages/openapi3/test/status-codes.test.ts

@@ -58,4 +58,53 @@ worksFor(supportedVersions, ({ diagnoseOpenApiFor, openApiFor }) => {
       },
     ]);
   });
+
+  it("extracts status codes from models in a named union", async () => {
+    await expectStatusCodes(
+      `
+      model A {
+        @statusCode _: 418;
+      }
+
+      model B {
+        @statusCode _: 200;
+      }
+
+      union R {
+        A: A;
+        B: B;
+      };
+
+      op read(): R;
+      `,
+      ["200", "418"],
+    );
+  });
+
+  it("deduplicates status codes when multiple models in union have same code", async () => {
+    await expectStatusCodes(
+      `
+      model A {
+        @statusCode _: 418;
+      }
+
+      model B {
+        @statusCode _: 418;
+      }
+
+      model C {
+        @statusCode _: 200;
+      }
+
+      union R {
+        A: A;
+        B: B;
+        C: C;
+      };
+
+      op read(): R;
+      `,
+      ["200", "418"],
+    );
+  });
 });