inline project env files in worker (#181)

* extract env vars from file system

* combine variables from a global with the request-scoped env

* inline build-time env vars in the worker script

* for some reason the tests failed in the pipeline but not locally

* switch between modes at runtime and apply on process.env

* add test for referencing variables

* use a .env.mjs file for the vars

* Update packages/cloudflare/src/cli/build/patches/investigated/copy-package-cli-files.ts

* move the merging to extractProjectEnvVars

* rename secrets to nextEnvVars

* add missing mode when retrieving value

* add link to nextjs var load order

* rename to compile

* change function to read a single file

* move the readEnvFile call inside the flatMap

* remove process.env.node_env usage

* add e2e test for env vars

* move locations

examples/api/.dev.vars

@@ -0,0 +1 @@
+NEXTJS_ENV=development

examples/api/.env.development

@@ -0,0 +1 @@
+TEST_ENV_VAR=TEST_VALUE

examples/api/app/api/env/route.ts

@@ -0,0 +1,3 @@
+export async function GET() {
+  return new Response(JSON.stringify(process.env));
+}

examples/api/e2e/base.spec.ts

@@ -30,3 +30,8 @@ test("the hello-world api POST route works as intended", async ({ page }) => {
   expect(res.headers()["content-type"]).toContain("text/plain");
   await expect(res.text()).resolves.toEqual("Hello post-World! body=some body");
 });
+
+test("sets environment variables from the Next.js env file", async ({ page }) => {
+  const res = await page.request.get("/api/env");
+  await expect(res.json()).resolves.toEqual(expect.objectContaining({ TEST_ENV_VAR: "TEST_VALUE" }));
+});

packages/cloudflare/env.d.ts

@@ -7,6 +7,8 @@ declare global {
       NEXT_PRIVATE_DEBUG_CACHE?: string;
       __OPENNEXT_KV_BINDING_NAME: string;
       OPEN_NEXT_ORIGIN: string;
+      NODE_ENV?: string;
+      __OPENNEXT_PROCESSED_ENV?: string;
     }
   }
 }

packages/cloudflare/package.json

@@ -59,11 +59,14 @@
     "tsup": "catalog:",
     "typescript": "catalog:",
     "typescript-eslint": "catalog:",
-    "vitest": "catalog:"
+    "vitest": "catalog:",
+    "mock-fs": "catalog:",
+    "@types/mock-fs": "catalog:"
   },
   "dependencies": {
     "@opennextjs/aws": "https://pkg.pr.new/@opennextjs/aws@678",
-    "ts-morph": "catalog:"
+    "ts-morph": "catalog:",
+    "@dotenvx/dotenvx": "catalog:"
   },
   "peerDependencies": {
     "wrangler": "catalog:"

packages/cloudflare/src/cli/build/index.ts

@@ -15,6 +15,7 @@ import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
 import type { ProjectOptions } from "../config";
 import { containsDotNextDir, getConfig } from "../config";
 import { bundleServer } from "./bundle-server";
+import { compileEnvFiles } from "./open-next/compile-env-files";
 import { createServerBundle } from "./open-next/createServerBundle";
 
 /**
@@ -71,6 +72,9 @@ export async function build(projectOpts: ProjectOptions): Promise<void> {
   // Compile cache.ts
   compileCache(options);
 
+  // Compile .env files
+  compileEnvFiles(options);
+
   // Compile middleware
   await createMiddleware(options, { forceOnlyBuildOnce: true });
 

packages/cloudflare/src/cli/build/open-next/compile-env-files.ts

@@ -0,0 +1,18 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { BuildOptions } from "@opennextjs/aws/build/helper.js";
+
+import { extractProjectEnvVars } from "../utils";
+
+/**
+ * Compiles the values extracted from the project's env files to the output directory for use in the worker.
+ */
+export function compileEnvFiles(options: BuildOptions) {
+  ["production", "development", "test"].forEach((mode) =>
+    fs.appendFileSync(
+      path.join(options.outputDir, `.env.mjs`),
+      `export const ${mode} = ${JSON.stringify(extractProjectEnvVars(mode, options))};\n`
+    )
+  );
+}

packages/cloudflare/src/cli/build/utils/extract-project-env-vars.spec.ts

@@ -0,0 +1,70 @@
+import { appendFileSync, writeFileSync } from "node:fs";
+
+import { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import mockFs from "mock-fs";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { extractProjectEnvVars } from "./extract-project-env-vars";
+
+const options = { monorepoRoot: "", appPath: "" } as BuildOptions;
+
+describe("extractProjectEnvVars", () => {
+  beforeEach(() => {
+    mockFs({
+      ".env": "ENV_VAR=value",
+      ".env.local": "ENV_LOCAL_VAR=value",
+      ".env.development": "ENV_DEV_VAR=value",
+      ".env.development.local": "ENV_DEV_LOCAL_VAR=value",
+      ".env.production": "ENV_PROD_VAR=value",
+      ".env.production.local": "ENV_PROD_LOCAL_VAR=value",
+    });
+  });
+
+  afterEach(() => mockFs.restore());
+
+  it("should extract production env vars", () => {
+    const result = extractProjectEnvVars("production", options);
+    expect(result).toEqual({
+      ENV_LOCAL_VAR: "value",
+      ENV_PROD_LOCAL_VAR: "value",
+      ENV_PROD_VAR: "value",
+      ENV_VAR: "value",
+    });
+  });
+
+  it("should extract development env vars", () => {
+    writeFileSync(".dev.vars", 'NEXTJS_ENV = "development"');
+
+    const result = extractProjectEnvVars("development", options);
+    expect(result).toEqual({
+      ENV_LOCAL_VAR: "value",
+      ENV_DEV_LOCAL_VAR: "value",
+      ENV_DEV_VAR: "value",
+      ENV_VAR: "value",
+    });
+  });
+
+  it("should override env vars with those in a local file", () => {
+    writeFileSync(".env.production.local", "ENV_PROD_VAR=overridden");
+
+    const result = extractProjectEnvVars("production", options);
+    expect(result).toEqual({
+      ENV_LOCAL_VAR: "value",
+      ENV_PROD_VAR: "overridden",
+      ENV_VAR: "value",
+    });
+  });
+
+  it("should support referencing variables", () => {
+    appendFileSync(".env.production.local", "\nENV_PROD_LOCAL_VAR_REF=$ENV_PROD_LOCAL_VAR");
+
+    const result = extractProjectEnvVars("production", options);
+    expect(result).toEqual({
+      ENV_LOCAL_VAR: "value",
+      ENV_PROD_LOCAL_VAR: "value",
+      ENV_PROD_LOCAL_VAR_REF: "value",
+      ENV_PROD_VAR: "value",
+      ENV_VAR: "value",
+    });
+  });
+});

packages/cloudflare/src/cli/build/utils/extract-project-env-vars.ts

@@ -0,0 +1,36 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+import { parse } from "@dotenvx/dotenvx";
+import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+
+function readEnvFile(filePath: string) {
+  if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) {
+    return parse(fs.readFileSync(filePath).toString());
+  }
+}
+
+/**
+ * Extracts the environment variables defined in various .env files for a project.
+ *
+ * The `NEXTJS_ENV` environment variable in `.dev.vars` determines the mode.
+ *
+ * Merged variables respect the following priority order.
+ * 1. `.env.{mode}.local`
+ * 2. `.env.local`
+ * 3. `.env.{mode}`
+ * 4. `.env`
+ *
+ * https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables#environment-variable-load-order
+ *
+ * In a monorepo, the env files in an app's directory will take precedence over
+ * the env files at the root of the monorepo.
+ */
+export function extractProjectEnvVars(mode: string, { monorepoRoot, appPath }: BuildOptions) {
+  return [".env", `.env.${mode}`, ".env.local", `.env.${mode}.local`]
+    .flatMap((fileName) => [
+      ...(monorepoRoot !== appPath ? [readEnvFile(path.join(monorepoRoot, fileName))] : []),
+      readEnvFile(path.join(appPath, fileName)),
+    ])
+    .reduce<Record<string, string>>((acc, overrides) => ({ ...acc, ...overrides }), {});
+}

packages/cloudflare/src/cli/build/utils/index.ts

@@ -1,3 +1,4 @@
 export * from "./copy-prerendered-routes";
+export * from "./extract-project-env-vars";
 export * from "./normalize-path";
 export * from "./ts-parse-file";

packages/cloudflare/src/cli/templates/worker.ts

@@ -21,6 +21,21 @@ const cloudflareContextALS = new AsyncLocalStorage<CloudflareContext>();
   }
 );
 
+async function applyProjectEnvVars(mode: string) {
+  if (process.env.__OPENNEXT_PROCESSED_ENV === "1") return;
+
+  // @ts-expect-error: resolved by wrangler build
+  const nextEnvVars = await import("./.env.mjs");
+
+  if (nextEnvVars[mode]) {
+    for (const key in nextEnvVars[mode]) {
+      process.env[key] = nextEnvVars[mode][key];
+    }
+  }
+
+  process.env.__OPENNEXT_PROCESSED_ENV = "1";
+}
+
 export default {
   async fetch(request, env, ctx) {
     return cloudflareContextALS.run({ env, ctx, cf: request.cf }, async () => {
@@ -34,6 +49,8 @@ export default {
         },
       });
 
+      await applyProjectEnvVars(env.NEXTJS_ENV ?? "production");
+
       // The Middleware handler can return either a `Response` or a `Request`:
       // - `Response`s should be returned early
       // - `Request`s are handled by the Next server
@@ -46,4 +63,4 @@ export default {
       return serverHandler(reqOrResp, env, ctx);
     });
   },
-} as ExportedHandler<{ ASSETS: Fetcher }>;
+} as ExportedHandler<{ ASSETS: Fetcher; NEXTJS_ENV?: string }>;