fix(server-core): rebuild the cached driver when its configuration changes

docs-mintlify/admin/connect-to-data/oauth-authentication.mdx

@@ -26,6 +26,32 @@ Because every user connects with different credentials, you also need
 per-user query orchestrator state. Without this, one user's cached
 connection could leak to another.
 
+<Warning>
+
+Cube caches one database connection per
+[`context_to_orchestrator_id`][ref-context-to-orchestrator-id]. **The
+orchestrator ID must therefore distinguish every user your
+`driver_factory` can return a different connection for** — otherwise two
+users share one pool and one user's credential is reused for another's
+queries.
+
+Do not add the token itself to the ID. When the token rotates, Cube
+notices that `driver_factory` now resolves a different configuration and
+rebuilds the connection in place, so a username is enough. Keying on the
+token instead creates a new orchestrator — with its own pool, queues and
+pre-aggregation cache — on every rotation.
+
+</Warning>
+
+<Note>
+
+Return the resolved credential from `driver_factory`, not a function that
+fetches one. Cube compares the configuration values the factory returns,
+and a function compares as unchanged however the credential behind it
+rotates — the connection would never be rebuilt.
+
+</Note>
+
 ## Prerequisites
 
 - A [Cube Cloud][ref-cube-cloud] deployment connected to an
@@ -152,50 +178,72 @@ value with the correct `type` and driver-specific options. See the
 
 ```python cube.py
 from cube import config
+from datetime import datetime, timezone
 import os
+import time
+
+# A token is handed to the driver once, but the pool keeps opening new sessions
+# with it afterwards. Reject one that is too close to expiry to survive that
+# gap, rather than one that is merely still valid at this instant.
+EXPIRY_SKEW_SECONDS = 120
+
+
+def _parse_expiry(value):
+    """Seconds since the epoch, or None if the value is absent or unparseable."""
+    if not value:
+        return None
+    if isinstance(value, (int, float)):
+        # Epoch milliseconds if the value is far too large to be seconds.
+        return value / 1000 if value > 1e11 else float(value)
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.timestamp()
+
+
+def _access_token(ctx: dict):
+    """The user's OAuth token, or None to fall back to the service account."""
+    # For other data sources, swap "databricks" for "snowflake", etc.
+    cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+    creds = (cube_cloud.get("userCredentials") or {}).get("databricks") or {}
+
+    access_token = creds.get("accessToken")
+    expires_at = _parse_expiry(creds.get("accessTokenExpiresAt"))
+
+    # Gate on the expiry rather than on `status`: a failed background refresh
+    # can flag the record while the token already in hand is still valid, and
+    # treating that as fatal drops the user onto the service account for no
+    # reason.
+    if access_token and expires_at and expires_at > time.time() + EXPIRY_SKEW_SECONDS:
+        return access_token
+
+    return None
 
 
 @config("driver_factory")
 def driver_factory(ctx: dict) -> dict:
-    # Extract the Cube Cloud security context, which contains
-    # per-user OAuth credentials when available.
-    # For other data sources, swap "databricks" for "snowflake", etc.
-    databricks_creds = (
-        ctx
-        .get("securityContext", {})
-        .get("cubeCloud", {})
-        .get("userCredentials", {})
-        .get("databricks", {})
-    )
-
-    # Only use the OAuth token when the credential status is "active".
-    # An expired or revoked token falls back to the service account.
-    oauth_token = (
-        databricks_creds.get("accessToken")
-        if databricks_creds.get("status") == "active"
-        else None
-    )
-
+    # Cube rebuilds this connection whenever the returned configuration
+    # changes, so returning a rotated token here is enough to replace it.
     return {
         "type": "databricks-jdbc",
         "url": os.environ["CUBEJS_DB_DATABRICKS_URL"],
-        # Prefer the user's OAuth token; fall back to the service account token
-        "token": oauth_token or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
+        "token": _access_token(ctx) or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
         "acceptPolicy": True,
         "catalog": os.environ.get("CUBEJS_DB_DATABRICKS_CATALOG"),
     }
 
 
 @config("context_to_orchestrator_id")
 def context_to_orchestrator_id(ctx: dict) -> str:
-    # Give each user a separate orchestrator instance (DB connections,
-    # execution queues, pre-aggregation caches)
-    username = (
-        ctx
-        .get("securityContext", {})
-        .get("cubeCloud", {})
-        .get("username", "default")
-    )
+    # One orchestrator per user: separate DB connections, execution queues and
+    # pre-aggregation caches. Deliberately not keyed on the token — see the
+    # warning above.
+    cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+    username = cube_cloud.get("username") or "default"
+
     return f"CUBE_APP_{username}"
 ```
 
@@ -204,37 +252,51 @@ def context_to_orchestrator_id(ctx: dict) -> str:
 <Tab title="JavaScript">
 
 ```javascript cube.js
-module.exports = {
-  driverFactory: ({ securityContext }) => {
-    // Extract the Cube Cloud security context, which contains
-    // per-user OAuth credentials when available.
-    // For other data sources, swap `databricks` for `snowflake`, etc.
-    const databricksCreds =
-      securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
-
-    // Only use the OAuth token when the credential status is "active".
-    // An expired or revoked token falls back to the service account.
-    const oauthToken =
-      databricksCreds.status === "active"
-        ? databricksCreds.accessToken
-        : null;
+// A token is handed to the driver once, but the pool keeps opening new sessions
+// with it afterwards. Reject one that is too close to expiry to survive that
+// gap, rather than one that is merely still valid at this instant.
+const EXPIRY_SKEW_MS = 120 * 1000;
+
+/** The user's OAuth token, or undefined to fall back to the service account. */
+function accessToken(securityContext) {
+  // For other data sources, swap `databricks` for `snowflake`, etc.
+  const creds = securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
+  const raw = creds.accessTokenExpiresAt;
+  // Epoch milliseconds if the value is far too large to be seconds. Reading
+  // seconds as milliseconds would land in 1970 and reject every token.
+  const expiresAt =
+    typeof raw === "number"
+      ? (raw > 1e11 ? raw : raw * 1000)
+      : Date.parse(raw ?? "");
+
+  // Gate on the expiry rather than on `status`: a failed background refresh can
+  // flag the record while the token already in hand is still valid, and
+  // treating that as fatal drops the user onto the service account for no
+  // reason. NaN fails this comparison, so an unparseable expiry falls back too.
+  if (creds.accessToken && expiresAt > Date.now() + EXPIRY_SKEW_MS) {
+    return creds.accessToken;
+  }
+
+  return undefined;
+}
 
-    return {
-      type: "databricks-jdbc",
-      url: process.env.CUBEJS_DB_DATABRICKS_URL,
-      // Prefer the user's OAuth token; fall back to the service account token
-      token: oauthToken || process.env.CUBEJS_DB_DATABRICKS_TOKEN,
-      acceptPolicy: true,
-      catalog: process.env.CUBEJS_DB_DATABRICKS_CATALOG,
-    };
-  },
-
-  // Give each user a separate orchestrator instance (DB connections,
-  // execution queues, pre-aggregation caches)
-  contextToOrchestratorId: ({ securityContext }) => {
-    const username = securityContext?.cubeCloud?.username ?? "default";
-    return `CUBE_APP_${username}`;
-  },
+module.exports = {
+  // Cube rebuilds this connection whenever the returned configuration changes,
+  // so returning a rotated token here is enough to replace it.
+  driverFactory: ({ securityContext }) => ({
+    type: "databricks-jdbc",
+    url: process.env.CUBEJS_DB_DATABRICKS_URL,
+    token:
+      accessToken(securityContext) ?? process.env.CUBEJS_DB_DATABRICKS_TOKEN,
+    acceptPolicy: true,
+    catalog: process.env.CUBEJS_DB_DATABRICKS_CATALOG,
+  }),
+
+  // One orchestrator per user: separate DB connections, execution queues and
+  // pre-aggregation caches. Deliberately not keyed on the token — see the
+  // warning above.
+  contextToOrchestratorId: ({ securityContext }) =>
+    `CUBE_APP_${securityContext?.cubeCloud?.username ?? "default"}`,
 };
 ```
 
@@ -248,21 +310,48 @@ module.exports = {
    credentials to `securityContext.cubeCloud.userCredentials.<data_source>`
    (for example, `.databricks` or `.snowflake`).
 
-2. **`driver_factory` resolves the credential** — If the credential status
-   is `active`, the user's OAuth token is used. Otherwise, Cube falls back
-   to the service account credential stored in environment variables.
+2. **`driver_factory` resolves the credential** — If the user has a token
+   that has not expired, it is used. Otherwise, Cube falls back to the
+   service account credential stored in environment variables.
 
 3. **Per-user orchestrator** —
-   [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns
-   a unique key per username, so each user gets their own database
+   [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns a
+   key derived from the username, so each user gets their own database
    connection pool, execution queues, and pre-aggregation table cache.
-   Without this, Cube would share a single cached connection across all
-   users, causing one user's credentials to be reused for another user's
-   queries.
+   Without it, every user shares one cached connection and the first
+   user's credential is reused for everyone else's queries.
+
+4. **Rotation replaces the connection** — on the next request after a
+   rotation, Cube compares what `driver_factory` now resolves against what
+   the cached connection was built from. When they differ it builds a
+   replacement and drains the old pool, so in-flight queries finish on the
+   connection they started on.
+
+## Operational notes
+
+- **One orchestrator per user, not per token.** The orchestrator survives
+  rotations, so pre-aggregation caches and queues stay warm and the
+  orchestrator count tracks your concurrent user count rather than growing
+  with every rotation.
+- **Don't make [`context_to_app_id`][ref-context-to-appid] per-user.** The
+  data model is identical for every user — only the connection differs —
+  so a per-user app ID forces a full data-model recompile per user on
+  every replica for no benefit. Leave it unset, or return a constant if
+  your deployment already sets one.
+- **Give the service account the minimum it needs to pass a connection
+  check.** If it has no access at all, liveness checks and any query that
+  falls back to it fail with an opaque authorization error from the driver
+  rather than something diagnosable.
+- **Falling back is silent.** A missing or near-expired token sends the
+  query to the service account instead of failing, so results reflect the
+  service account's permissions rather than the user's. If that is not
+  acceptable for your deployment, raise an error in `driver_factory`
+  instead of returning the fallback credential.
 
 [ref-config]: /reference/configuration/config
 [ref-driver-factory]: /reference/configuration/config#driver_factory
 [ref-context-to-orchestrator-id]: /reference/configuration/config#context_to_orchestrator_id
+[ref-context-to-appid]: /reference/configuration/config#context_to_app_id
 [ref-databricks-jdbc]: /admin/connect-to-data/data-sources/databricks-jdbc
 [ref-snowflake]: /admin/connect-to-data/data-sources/snowflake
 [ref-data-sources]: /admin/connect-to-data/data-sources

packages/cubejs-server-core/src/core/driver-config-fingerprint.ts

@@ -0,0 +1,105 @@
+/**
+ * @copyright Cube Dev, Inc.
+ * @license Apache-2.0
+ * @fileoverview Fingerprinting for driver configurations and security contexts.
+ */
+
+import crypto from 'crypto';
+
+/**
+ * Deterministic JSON used for fingerprinting. Object keys are emitted in sorted
+ * order so two structures that differ only in property order hash the same, and
+ * values JSON cannot represent are reduced to stable placeholders rather than
+ * silently disappearing. Throws on a circular structure, which callers treat as
+ * "not fingerprintable".
+ */
+function stableStringify(value: unknown, seen: Set<unknown>): string {
+  if (value === undefined || value === null) {
+    return 'null';
+  }
+
+  const type = typeof value;
+
+  if (type === 'string' || type === 'number' || type === 'boolean') {
+    return JSON.stringify(value);
+  }
+
+  if (type === 'bigint') {
+    return JSON.stringify((value as bigint).toString());
+  }
+
+  // A closure's identity cannot be compared meaningfully across calls, so it
+  // contributes a constant. Two configs differing only in a function body are
+  // therefore treated as equal — deliberately conservative: it can only lead to
+  // reusing a connection, never to swapping one out unnecessarily.
+  //
+  // The practical consequence is that a config carrying its credential as a
+  // provider callback rather than a resolved value fingerprints identically
+  // however the credential rotates, so such a driver is never rebuilt. A
+  // `driverFactory` that needs rotation to be noticed has to return the
+  // resolved value.
+  if (type === 'function' || type === 'symbol') {
+    return JSON.stringify(`[${type}]`);
+  }
+
+  if (value instanceof Date) {
+    return JSON.stringify(value.toISOString());
+  }
+
+  if (seen.has(value)) {
+    throw new Error('Circular structure cannot be fingerprinted');
+  }
+
+  seen.add(value);
+
+  try {
+    if (Array.isArray(value)) {
+      return `[${value.map((item) => stableStringify(item, seen)).join(',')}]`;
+    }
+
+    // Own enumerable keys only, so a class instance holding its values behind
+    // prototype accessors fingerprints as `{}` — constant, and therefore another
+    // shape whose rotation goes unnoticed. Plain configs are unaffected.
+    const entries = Object.keys(value as Record<string, unknown>)
+      .sort()
+      .reduce<string[]>((acc, key) => {
+        const entry = (value as Record<string, unknown>)[key];
+
+        // Match JSON.stringify: undefined-valued properties are absent, so
+        // `{ a: undefined }` and `{}` fingerprint the same.
+        if (entry !== undefined) {
+          acc.push(`${JSON.stringify(key)}:${stableStringify(entry, seen)}`);
+        }
+
+        return acc;
+      }, []);
+
+    return `{${entries.join(',')}}`;
+  } finally {
+    seen.delete(value);
+  }
+}
+
+/**
+ * A short, stable digest of `value`, or `null` when it cannot be fingerprinted.
+ *
+ * Hashed rather than kept verbatim because the values being compared include
+ * database passwords and OAuth access tokens: a raw copy would live for the
+ * lifetime of the process and surface in any heap dump. `null` means "cannot
+ * tell whether this changed", and every caller must treat that as "assume it
+ * did not" so behaviour falls back to the previous resolve-once semantics.
+ */
+export function fingerprint(value: unknown): string | null {
+  try {
+    return crypto
+      .createHash('sha256')
+      .update(stableStringify(value, new Set()))
+      // 32 hex chars = 128 bits, which is far more than an equality check over
+      // the handful of configurations one process resolves needs, and keeps the
+      // digest short enough to sit in a log line.
+      .digest('hex')
+      .slice(0, 32);
+  } catch (e) {
+    return null;
+  }
+}