diff --git a/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx b/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
index c18f0e2ef4a23..70e4313f5a65b 100644
--- a/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
+++ b/docs-mintlify/admin/connect-to-data/oauth-authentication.mdx
@@ -26,6 +26,32 @@ Because every user connects with different credentials, you also need
per-user query orchestrator state. Without this, one user's cached
connection could leak to another.
+
+
+Cube caches one database connection per
+[`context_to_orchestrator_id`][ref-context-to-orchestrator-id]. **The
+orchestrator ID must therefore distinguish every user your
+`driver_factory` can return a different connection for** — otherwise two
+users share one pool and one user's credential is reused for another's
+queries.
+
+Do not add the token itself to the ID. When the token rotates, Cube
+notices that `driver_factory` now resolves a different configuration and
+rebuilds the connection in place, so a username is enough. Keying on the
+token instead creates a new orchestrator — with its own pool, queues and
+pre-aggregation cache — on every rotation.
+
+
+
+
+
+Return the resolved credential from `driver_factory`, not a function that
+fetches one. Cube compares the configuration values the factory returns,
+and a function compares as unchanged however the credential behind it
+rotates — the connection would never be rebuilt.
+
+
+
## Prerequisites
- A [Cube Cloud][ref-cube-cloud] deployment connected to an
@@ -152,35 +178,59 @@ value with the correct `type` and driver-specific options. See the
```python cube.py
from cube import config
+from datetime import datetime, timezone
import os
+import time
+
+# A token is handed to the driver once, but the pool keeps opening new sessions
+# with it afterwards. Reject one that is too close to expiry to survive that
+# gap, rather than one that is merely still valid at this instant.
+EXPIRY_SKEW_SECONDS = 120
+
+
+def _parse_expiry(value):
+ """Seconds since the epoch, or None if the value is absent or unparseable."""
+ if not value:
+ return None
+ if isinstance(value, (int, float)):
+ # Epoch milliseconds if the value is far too large to be seconds.
+ return value / 1000 if value > 1e11 else float(value)
+ try:
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ return parsed.timestamp()
+
+
+def _access_token(ctx: dict):
+ """The user's OAuth token, or None to fall back to the service account."""
+ # For other data sources, swap "databricks" for "snowflake", etc.
+ cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+ creds = (cube_cloud.get("userCredentials") or {}).get("databricks") or {}
+
+ access_token = creds.get("accessToken")
+ expires_at = _parse_expiry(creds.get("accessTokenExpiresAt"))
+
+ # Gate on the expiry rather than on `status`: a failed background refresh
+ # can flag the record while the token already in hand is still valid, and
+ # treating that as fatal drops the user onto the service account for no
+ # reason.
+ if access_token and expires_at and expires_at > time.time() + EXPIRY_SKEW_SECONDS:
+ return access_token
+
+ return None
@config("driver_factory")
def driver_factory(ctx: dict) -> dict:
- # Extract the Cube Cloud security context, which contains
- # per-user OAuth credentials when available.
- # For other data sources, swap "databricks" for "snowflake", etc.
- databricks_creds = (
- ctx
- .get("securityContext", {})
- .get("cubeCloud", {})
- .get("userCredentials", {})
- .get("databricks", {})
- )
-
- # Only use the OAuth token when the credential status is "active".
- # An expired or revoked token falls back to the service account.
- oauth_token = (
- databricks_creds.get("accessToken")
- if databricks_creds.get("status") == "active"
- else None
- )
-
+ # Cube rebuilds this connection whenever the returned configuration
+ # changes, so returning a rotated token here is enough to replace it.
return {
"type": "databricks-jdbc",
"url": os.environ["CUBEJS_DB_DATABRICKS_URL"],
- # Prefer the user's OAuth token; fall back to the service account token
- "token": oauth_token or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
+ "token": _access_token(ctx) or os.environ["CUBEJS_DB_DATABRICKS_TOKEN"],
"acceptPolicy": True,
"catalog": os.environ.get("CUBEJS_DB_DATABRICKS_CATALOG"),
}
@@ -188,14 +238,12 @@ def driver_factory(ctx: dict) -> dict:
@config("context_to_orchestrator_id")
def context_to_orchestrator_id(ctx: dict) -> str:
- # Give each user a separate orchestrator instance (DB connections,
- # execution queues, pre-aggregation caches)
- username = (
- ctx
- .get("securityContext", {})
- .get("cubeCloud", {})
- .get("username", "default")
- )
+ # One orchestrator per user: separate DB connections, execution queues and
+ # pre-aggregation caches. Deliberately not keyed on the token — see the
+ # warning above.
+ cube_cloud = (ctx.get("securityContext") or {}).get("cubeCloud") or {}
+ username = cube_cloud.get("username") or "default"
+
return f"CUBE_APP_{username}"
```
@@ -204,37 +252,51 @@ def context_to_orchestrator_id(ctx: dict) -> str:
```javascript cube.js
-module.exports = {
- driverFactory: ({ securityContext }) => {
- // Extract the Cube Cloud security context, which contains
- // per-user OAuth credentials when available.
- // For other data sources, swap `databricks` for `snowflake`, etc.
- const databricksCreds =
- securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
-
- // Only use the OAuth token when the credential status is "active".
- // An expired or revoked token falls back to the service account.
- const oauthToken =
- databricksCreds.status === "active"
- ? databricksCreds.accessToken
- : null;
+// A token is handed to the driver once, but the pool keeps opening new sessions
+// with it afterwards. Reject one that is too close to expiry to survive that
+// gap, rather than one that is merely still valid at this instant.
+const EXPIRY_SKEW_MS = 120 * 1000;
+
+/** The user's OAuth token, or undefined to fall back to the service account. */
+function accessToken(securityContext) {
+ // For other data sources, swap `databricks` for `snowflake`, etc.
+ const creds = securityContext?.cubeCloud?.userCredentials?.databricks ?? {};
+ const raw = creds.accessTokenExpiresAt;
+ // Epoch milliseconds if the value is far too large to be seconds. Reading
+ // seconds as milliseconds would land in 1970 and reject every token.
+ const expiresAt =
+ typeof raw === "number"
+ ? (raw > 1e11 ? raw : raw * 1000)
+ : Date.parse(raw ?? "");
+
+ // Gate on the expiry rather than on `status`: a failed background refresh can
+ // flag the record while the token already in hand is still valid, and
+ // treating that as fatal drops the user onto the service account for no
+ // reason. NaN fails this comparison, so an unparseable expiry falls back too.
+ if (creds.accessToken && expiresAt > Date.now() + EXPIRY_SKEW_MS) {
+ return creds.accessToken;
+ }
+
+ return undefined;
+}
- return {
- type: "databricks-jdbc",
- url: process.env.CUBEJS_DB_DATABRICKS_URL,
- // Prefer the user's OAuth token; fall back to the service account token
- token: oauthToken || process.env.CUBEJS_DB_DATABRICKS_TOKEN,
- acceptPolicy: true,
- catalog: process.env.CUBEJS_DB_DATABRICKS_CATALOG,
- };
- },
-
- // Give each user a separate orchestrator instance (DB connections,
- // execution queues, pre-aggregation caches)
- contextToOrchestratorId: ({ securityContext }) => {
- const username = securityContext?.cubeCloud?.username ?? "default";
- return `CUBE_APP_${username}`;
- },
+module.exports = {
+ // Cube rebuilds this connection whenever the returned configuration changes,
+ // so returning a rotated token here is enough to replace it.
+ driverFactory: ({ securityContext }) => ({
+ type: "databricks-jdbc",
+ url: process.env.CUBEJS_DB_DATABRICKS_URL,
+ token:
+ accessToken(securityContext) ?? process.env.CUBEJS_DB_DATABRICKS_TOKEN,
+ acceptPolicy: true,
+ catalog: process.env.CUBEJS_DB_DATABRICKS_CATALOG,
+ }),
+
+ // One orchestrator per user: separate DB connections, execution queues and
+ // pre-aggregation caches. Deliberately not keyed on the token — see the
+ // warning above.
+ contextToOrchestratorId: ({ securityContext }) =>
+ `CUBE_APP_${securityContext?.cubeCloud?.username ?? "default"}`,
};
```
@@ -248,21 +310,48 @@ module.exports = {
credentials to `securityContext.cubeCloud.userCredentials.`
(for example, `.databricks` or `.snowflake`).
-2. **`driver_factory` resolves the credential** — If the credential status
- is `active`, the user's OAuth token is used. Otherwise, Cube falls back
- to the service account credential stored in environment variables.
+2. **`driver_factory` resolves the credential** — If the user has a token
+ that has not expired, it is used. Otherwise, Cube falls back to the
+ service account credential stored in environment variables.
3. **Per-user orchestrator** —
- [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns
- a unique key per username, so each user gets their own database
+ [`context_to_orchestrator_id`][ref-context-to-orchestrator-id] returns a
+ key derived from the username, so each user gets their own database
connection pool, execution queues, and pre-aggregation table cache.
- Without this, Cube would share a single cached connection across all
- users, causing one user's credentials to be reused for another user's
- queries.
+ Without it, every user shares one cached connection and the first
+ user's credential is reused for everyone else's queries.
+
+4. **Rotation replaces the connection** — on the next request after a
+ rotation, Cube compares what `driver_factory` now resolves against what
+ the cached connection was built from. When they differ it builds a
+ replacement and drains the old pool, so in-flight queries finish on the
+ connection they started on.
+
+## Operational notes
+
+- **One orchestrator per user, not per token.** The orchestrator survives
+ rotations, so pre-aggregation caches and queues stay warm and the
+ orchestrator count tracks your concurrent user count rather than growing
+ with every rotation.
+- **Don't make [`context_to_app_id`][ref-context-to-appid] per-user.** The
+ data model is identical for every user — only the connection differs —
+ so a per-user app ID forces a full data-model recompile per user on
+ every replica for no benefit. Leave it unset, or return a constant if
+ your deployment already sets one.
+- **Give the service account the minimum it needs to pass a connection
+ check.** If it has no access at all, liveness checks and any query that
+ falls back to it fail with an opaque authorization error from the driver
+ rather than something diagnosable.
+- **Falling back is silent.** A missing or near-expired token sends the
+ query to the service account instead of failing, so results reflect the
+ service account's permissions rather than the user's. If that is not
+ acceptable for your deployment, raise an error in `driver_factory`
+ instead of returning the fallback credential.
[ref-config]: /reference/configuration/config
[ref-driver-factory]: /reference/configuration/config#driver_factory
[ref-context-to-orchestrator-id]: /reference/configuration/config#context_to_orchestrator_id
+[ref-context-to-appid]: /reference/configuration/config#context_to_app_id
[ref-databricks-jdbc]: /admin/connect-to-data/data-sources/databricks-jdbc
[ref-snowflake]: /admin/connect-to-data/data-sources/snowflake
[ref-data-sources]: /admin/connect-to-data/data-sources
diff --git a/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts b/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts
new file mode 100644
index 0000000000000..f1c4e963a0149
--- /dev/null
+++ b/packages/cubejs-server-core/src/core/driver-config-fingerprint.ts
@@ -0,0 +1,105 @@
+/**
+ * @copyright Cube Dev, Inc.
+ * @license Apache-2.0
+ * @fileoverview Fingerprinting for driver configurations and security contexts.
+ */
+
+import crypto from 'crypto';
+
+/**
+ * Deterministic JSON used for fingerprinting. Object keys are emitted in sorted
+ * order so two structures that differ only in property order hash the same, and
+ * values JSON cannot represent are reduced to stable placeholders rather than
+ * silently disappearing. Throws on a circular structure, which callers treat as
+ * "not fingerprintable".
+ */
+function stableStringify(value: unknown, seen: Set): string {
+ if (value === undefined || value === null) {
+ return 'null';
+ }
+
+ const type = typeof value;
+
+ if (type === 'string' || type === 'number' || type === 'boolean') {
+ return JSON.stringify(value);
+ }
+
+ if (type === 'bigint') {
+ return JSON.stringify((value as bigint).toString());
+ }
+
+ // A closure's identity cannot be compared meaningfully across calls, so it
+ // contributes a constant. Two configs differing only in a function body are
+ // therefore treated as equal — deliberately conservative: it can only lead to
+ // reusing a connection, never to swapping one out unnecessarily.
+ //
+ // The practical consequence is that a config carrying its credential as a
+ // provider callback rather than a resolved value fingerprints identically
+ // however the credential rotates, so such a driver is never rebuilt. A
+ // `driverFactory` that needs rotation to be noticed has to return the
+ // resolved value.
+ if (type === 'function' || type === 'symbol') {
+ return JSON.stringify(`[${type}]`);
+ }
+
+ if (value instanceof Date) {
+ return JSON.stringify(value.toISOString());
+ }
+
+ if (seen.has(value)) {
+ throw new Error('Circular structure cannot be fingerprinted');
+ }
+
+ seen.add(value);
+
+ try {
+ if (Array.isArray(value)) {
+ return `[${value.map((item) => stableStringify(item, seen)).join(',')}]`;
+ }
+
+ // Own enumerable keys only, so a class instance holding its values behind
+ // prototype accessors fingerprints as `{}` — constant, and therefore another
+ // shape whose rotation goes unnoticed. Plain configs are unaffected.
+ const entries = Object.keys(value as Record)
+ .sort()
+ .reduce((acc, key) => {
+ const entry = (value as Record)[key];
+
+ // Match JSON.stringify: undefined-valued properties are absent, so
+ // `{ a: undefined }` and `{}` fingerprint the same.
+ if (entry !== undefined) {
+ acc.push(`${JSON.stringify(key)}:${stableStringify(entry, seen)}`);
+ }
+
+ return acc;
+ }, []);
+
+ return `{${entries.join(',')}}`;
+ } finally {
+ seen.delete(value);
+ }
+}
+
+/**
+ * A short, stable digest of `value`, or `null` when it cannot be fingerprinted.
+ *
+ * Hashed rather than kept verbatim because the values being compared include
+ * database passwords and OAuth access tokens: a raw copy would live for the
+ * lifetime of the process and surface in any heap dump. `null` means "cannot
+ * tell whether this changed", and every caller must treat that as "assume it
+ * did not" so behaviour falls back to the previous resolve-once semantics.
+ */
+export function fingerprint(value: unknown): string | null {
+ try {
+ return crypto
+ .createHash('sha256')
+ .update(stableStringify(value, new Set()))
+ // 32 hex chars = 128 bits, which is far more than an equality check over
+ // the handful of configurations one process resolves needs, and keeps the
+ // digest short enough to sit in a log line.
+ .digest('hex')
+ .slice(0, 32);
+ } catch (e) {
+ return null;
+ }
+}
diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts
index 48b146c8abb70..0cb3e07dd7f60 100644
--- a/packages/cubejs-server-core/src/core/server.ts
+++ b/packages/cubejs-server-core/src/core/server.ts
@@ -37,6 +37,7 @@ import { agentCollect } from './agentCollect';
import { OrchestratorStorage } from './OrchestratorStorage';
import { createLogger } from './logger';
import { OptsHandler } from './OptsHandler';
+import { fingerprint } from './driver-config-fingerprint';
import {
driverDependencies,
lookupDriverClass,
@@ -74,6 +75,35 @@ import {
const { version } = require('../../../package.json');
+/**
+ * Rebuilds of one data source's driver before the log escalates to naming a
+ * likely misconfiguration. A rotating credential rebuilds a few times a day, so
+ * reaching this within a process means contexts are displacing each other.
+ */
+const DRIVER_REBUILD_WARN_THRESHOLD = 50;
+
+/**
+ * How many times one request will retry after losing the race to rebuild a
+ * driver before settling for whatever is cached. Bounds the work a single
+ * request can be made to do when contexts keep displacing each other's driver.
+ */
+const MAX_DRIVER_REBUILD_ATTEMPTS = 3;
+
+/**
+ * What a cached driver was built from. `null` on either field means "cannot
+ * tell whether it changed", which is always read as "assume it did not".
+ */
+type DriverOrigin = {
+ securityContextFingerprint: string | null;
+ configFingerprint: string | null;
+};
+
+/** A `driverFactory` result together with the context that produced it. */
+type DriverFactoryResult = {
+ value: DriverConfig | BaseDriver;
+ securityContextFingerprint: string | null;
+};
+
function wrapToFnIfNeeded(possibleFn: T | ((a: R) => T)): (a: R) => T {
if (typeof possibleFn === 'function') {
return possibleFn;
@@ -130,6 +160,19 @@ export class CubejsServerCore {
protected readonly orchestratorStorage: OrchestratorStorage = new OrchestratorStorage();
+ /**
+ * The request context each cached orchestrator most recently served.
+ *
+ * An orchestrator's driver factory closes over the context of the request
+ * that created it, and the driver it resolves is then cached for the life of
+ * the process. When that driver's configuration is derived from the context —
+ * a per-user OAuth token, say — it goes stale the moment the credential
+ * rotates. Tracking the latest context lets the factory notice. Keyed by the
+ * api instance so an entry disappears with the orchestrator it belongs to.
+ */
+ protected readonly orchestratorRequestContexts =
+ new WeakMap();
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
protected repositoryFactory: ((context: RequestContext) => SchemaFileRepository) | (() => FileRepository);
@@ -573,9 +616,22 @@ export class CubejsServerCore {
const orchestratorId = await this.contextToOrchestratorId(context);
if (this.orchestratorStorage.has(orchestratorId)) {
- return this.orchestratorStorage.get(orchestratorId);
+ const cachedOrchestratorApi = this.orchestratorStorage.get(orchestratorId);
+ const cachedContextRef = this.orchestratorRequestContexts.get(cachedOrchestratorApi);
+
+ // Keep the driver factory's view of the request context current. Without
+ // this it stays pinned to whichever request happened to create the
+ // orchestrator, and a driver built from context-derived credentials can
+ // never be rebuilt when they rotate.
+ if (cachedContextRef) {
+ cachedContextRef.current = context;
+ }
+
+ return cachedOrchestratorApi;
}
+ const requestContextRef: { current: RequestContext } = { current: context };
+
/**
* Hash table to store promises which will be resolved with the
* datasource drivers. DriverFactoryByDataSource function is closure
@@ -583,6 +639,21 @@ export class CubejsServerCore {
*/
const driverPromise: Record> = {};
+ /**
+ * What each cached driver in `driverPromise` was built from, so a changed
+ * configuration can be detected. Keyed identically to `driverPromise`.
+ */
+ const driverOrigin: Record = {};
+
+ /**
+ * How many times each key has been rebuilt. Reported with the rebuild so a
+ * deployment whose `contextToOrchestratorId` does not partition by whatever
+ * `driverFactory` reads — every user sharing one orchestrator, say — is
+ * diagnosable: it rebuilds on request after request rather than once per
+ * credential rotation.
+ */
+ const driverRebuilds: Record = {};
+
let externalPreAggregationsDriverPromise: Promise | null = null;
const contextToDbType: DbTypeInternalFn = this.contextToDbType.bind(this);
@@ -596,74 +667,226 @@ export class CubejsServerCore {
(await this.orchestratorOptions(context)) || {},
);
- const orchestratorApi = this.createOrchestratorApi(
+ /**
+ * Driver factory function `DriverFactoryByDataSource`. Named so the rebuild
+ * path can re-enter it when another caller wins the race to replace a key.
+ */
+ const resolveDataSourceDriver = async (
+ dataSource = 'default',
+ preAggregations = false,
+ attempt = 0,
+ ): Promise => {
+ const factoryKey = preAggregations ? `${dataSource}@pre_agg` : dataSource;
+
+ const hasSeparatePreAggEnv = hasPreAggregationsEnvVars(dataSource);
+ const usePreAgg = preAggregations && hasSeparatePreAggEnv && !this.optsHandler.isCustomDriverFactory();
+
+ const driverContext = (): DriverContext => ({
+ ...requestContextRef.current,
+ dataSource,
+ preAggregations: usePreAgg || false,
+ });
+
/**
- * Driver factory function `DriverFactoryByDataSource`.
+ * Every key that resolves to the one driver built here. Without separate
+ * pre-aggregation credentials `usePreAgg` is false whichever key was
+ * asked for, so both describe an identically configured driver and share
+ * a single instance — they must therefore be written, and invalidated,
+ * together. Doing it per requested key instead lets the two diverge into
+ * two pools where the deployment expects one.
*/
- async (dataSource = 'default', preAggregations = false) => {
- const factoryKey = preAggregations ? `${dataSource}@pre_agg` : dataSource;
- if (driverPromise[factoryKey]) {
- return driverPromise[factoryKey];
- }
+ const aliasedKeys = hasSeparatePreAggEnv
+ ? [factoryKey]
+ : [dataSource, `${dataSource}@pre_agg`];
+
+ const invalidate = () => aliasedKeys.forEach((key) => {
+ driverPromise[key] = null;
+ delete driverOrigin[key];
+ });
+
+ // Already resolved by the staleness check below, so the factory is not
+ // asked twice for the same rebuild.
+ let resolvedFactoryResult: DriverFactoryResult | undefined;
- const hasSeparatePreAggEnv = hasPreAggregationsEnvVars(dataSource);
- const usePreAgg = preAggregations && hasSeparatePreAggEnv && !this.optsHandler.isCustomDriverFactory();
+ const cached = driverPromise[factoryKey];
- if (preAggregations && hasSeparatePreAggEnv && this.optsHandler.isCustomDriverFactory()) {
- this.logger('Pre-aggregation driver conflict', {
- error: 'Both driverFactory and PRE_AGGREGATIONS env vars are defined. driverFactory will take precedence.',
+ if (cached) {
+ const staleness = await this.resolveDriverStaleness(
+ driverOrigin[factoryKey],
+ driverContext(),
+ );
+
+ // `resolveDriverStaleness` awaits the user's factory, so another caller
+ // may have replaced or invalidated this key in the meantime. Its work
+ // supersedes ours, and `cached` is no longer ours to reuse or release:
+ // it has either been handed to that caller's requests or already
+ // released by it.
+ const superseding = driverPromise[factoryKey];
+
+ if (superseding !== cached) {
+ // Retry, so this request ends up on a driver matching its own
+ // context — but bounded. Where contexts keep displacing each other
+ // this request could otherwise lose every round and pay for a
+ // user-supplied factory call each time. Past the bound, take what is
+ // cached: degrading to a reused driver is this design's fallback
+ // everywhere else, and it is strictly better than starving.
+ if (attempt < MAX_DRIVER_REBUILD_ATTEMPTS) {
+ return resolveDataSourceDriver(dataSource, preAggregations, attempt + 1);
+ }
+
+ if (superseding) {
+ return superseding;
+ }
+
+ // Invalidated rather than replaced — the winning caller's own build
+ // failed, so it released `cached` and left nothing to reuse. Build
+ // below, which cannot recurse again, carrying the probe's result when
+ // it already resolved one so the factory is not asked twice.
+ resolvedFactoryResult = staleness.stale ? staleness.factoryResult : undefined;
+ } else if (!staleness.stale) {
+ return cached;
+ } else {
+ // Counted per alias set, not per key: a rotation seen first through
+ // `default@pre_agg` and then through `default` is one rebuild of one
+ // shared driver, and must not read as two counters at 1.
+ const rebuildKey = aliasedKeys[0];
+ driverRebuilds[rebuildKey] = (driverRebuilds[rebuildKey] || 0) + 1;
+ const rebuildCount = driverRebuilds[rebuildKey];
+
+ // Carries `warning` so it survives the default log level: a
+ // plain-params message matches no allowlist in
+ // `prodLogger`/`devLogger` and is dropped below `trace`. Tearing down
+ // a connection pool is an event an operator needs to be able to
+ // correlate against, and the threshold message below arrives too late
+ // to reconstruct the first rebuilds.
+ this.logger('Rebuilding driver on configuration change', {
dataSource,
+ preAggregations,
+ rebuildCount,
+ warning: 'Driver configuration changed; replacing the connection.',
});
+
+ // A credential rotation rebuilds a handful of times a day. Rebuilding
+ // this often means the orchestrator id does not partition by whatever
+ // the factory reads, so contexts that need different connections keep
+ // displacing each other's driver.
+ if (rebuildCount === DRIVER_REBUILD_WARN_THRESHOLD) {
+ this.logger('Driver rebuilt repeatedly', {
+ dataSource,
+ rebuildCount,
+ warning: 'Driver configuration keeps changing for one orchestrator. '
+ + 'contextToOrchestratorId likely does not distinguish the contexts '
+ + 'driverFactory returns different connections for.',
+ });
+ }
+
+ // Clear every key pointing at the replaced driver, not just the one
+ // asked for: a surviving alias would keep handing out a driver whose
+ // pool is being drained, and would release it a second time when it
+ // was itself found stale.
+ Object.keys(driverPromise)
+ .filter((key) => driverPromise[key] === cached)
+ .forEach((key) => {
+ driverPromise[key] = null;
+ delete driverOrigin[key];
+ });
+
+ // Graceful: `release` drains the pool, so queries already running on
+ // the replaced driver finish before its connections are closed. It is
+ // deliberately not awaited — this request should not wait on the
+ // previous driver's in-flight work — and its failure must not fail
+ // this request.
+ cached
+ .then((driver) => driver.release())
+ .catch((error) => this.logger('Driver release error', {
+ dataSource,
+ error: (error as Error).stack || (error as Error).toString(),
+ }));
+
+ resolvedFactoryResult = staleness.factoryResult;
}
+ }
- driverPromise[factoryKey] = (async () => {
- let driver: BaseDriver | null = null;
-
- try {
- driver = await this.resolveDriver(
- {
- ...context,
- dataSource,
- preAggregations: usePreAgg || false,
- },
- orchestratorOptions,
- );
-
- if (typeof driver === 'object' && driver != null) {
- if (driver.setLogger) {
- driver.setLogger(this.logger);
- }
+ if (preAggregations && hasSeparatePreAggEnv && this.optsHandler.isCustomDriverFactory()) {
+ this.logger('Pre-aggregation driver conflict', {
+ error: 'Both driverFactory and PRE_AGGREGATIONS env vars are defined. driverFactory will take precedence.',
+ dataSource,
+ });
+ }
- await driver.testConnection();
+ // Shared by reference across `aliasedKeys`, so every key describes the
+ // one driver they all resolve to. Starts empty: until the factory has
+ // been called there is nothing to compare against, and
+ // `resolveDriverStaleness` reads that as "reuse".
+ const origin: DriverOrigin = {
+ securityContextFingerprint: null,
+ configFingerprint: null,
+ };
- return driver;
- }
+ aliasedKeys.forEach((key) => {
+ driverOrigin[key] = origin;
+ });
- throw new Error(
- `Unexpected return type, driverFactory must return driver (dataSource: "${dataSource}"), actual: ${getRealType(driver)}`
- );
- } catch (e) {
- driverPromise[factoryKey] = null;
+ const pending = (async () => {
+ let driver: BaseDriver | null = null;
- if (!preAggregations && !hasSeparatePreAggEnv) {
- driverPromise[`${dataSource}@pre_agg`] = null;
- }
+ try {
+ const currentDriverContext = driverContext();
+ const factoryResult = resolvedFactoryResult ?? {
+ value: await this.options.driverFactory(currentDriverContext),
+ securityContextFingerprint: fingerprint(currentDriverContext.securityContext),
+ };
+
+ origin.securityContextFingerprint = factoryResult.securityContextFingerprint;
+ origin.configFingerprint = isDriver(factoryResult.value)
+ ? null
+ : fingerprint(factoryResult.value);
- if (driver) {
- await driver.release();
+ driver = await this.createDriverFromFactoryResult(
+ factoryResult.value,
+ currentDriverContext,
+ orchestratorOptions,
+ );
+
+ if (typeof driver === 'object' && driver != null) {
+ if (driver.setLogger) {
+ driver.setLogger(this.logger);
}
- throw e;
+ await driver.testConnection();
+
+ return driver;
}
- })();
- // No separate pre-agg driver needed — share the same promise for both keys
- if (!preAggregations && !hasSeparatePreAggEnv) {
- driverPromise[`${dataSource}@pre_agg`] = driverPromise[factoryKey];
+ throw new Error(
+ `Unexpected return type, driverFactory must return driver (dataSource: "${dataSource}"), actual: ${getRealType(driver)}`
+ );
+ } catch (e) {
+ // Only if this build still owns the keys. A concurrent rebuild
+ // installs its own `origin`, and its driver must not be evicted
+ // because ours failed.
+ if (driverOrigin[factoryKey] === origin) {
+ invalidate();
+ }
+
+ if (driver) {
+ await driver.release();
+ }
+
+ throw e;
}
+ })();
+
+ // No separate pre-agg driver needed — share the same promise across keys
+ aliasedKeys.forEach((key) => {
+ driverPromise[key] = pending;
+ });
+
+ return pending;
+ };
- return driverPromise[factoryKey];
- },
+ const orchestratorApi = this.createOrchestratorApi(
+ resolveDataSourceDriver,
{
externalDriverFactory: this.options.externalDriverFactory && (async () => {
if (externalPreAggregationsDriverPromise) {
@@ -713,6 +936,7 @@ export class CubejsServerCore {
}
);
+ this.orchestratorRequestContexts.set(orchestratorApi, requestContextRef);
this.orchestratorStorage.set(orchestratorId, orchestratorApi);
return orchestratorApi;
@@ -877,7 +1101,24 @@ export class CubejsServerCore {
context: DriverContext,
options?: OrchestratorInitedOptions,
): Promise {
- const val = await this.options.driverFactory(context);
+ return this.createDriverFromFactoryResult(
+ await this.options.driverFactory(context),
+ context,
+ options,
+ );
+ }
+
+ /**
+ * Build a driver from whatever `driverFactory` returned. Split out of
+ * `resolveDriver` so a caller that has already invoked the factory — to
+ * compare its result against the cached driver's — can build from that same
+ * result instead of invoking a user-supplied function a second time.
+ */
+ protected async createDriverFromFactoryResult(
+ val: DriverConfig | BaseDriver,
+ context: DriverContext,
+ options?: OrchestratorInitedOptions,
+ ): Promise {
if (isDriver(val)) {
return val;
} else {
@@ -895,6 +1136,97 @@ export class CubejsServerCore {
}
}
+ /**
+ * Decide whether a cached driver still reflects what `driverFactory` would
+ * resolve for the current request context.
+ *
+ * The check is deliberately layered so that deployments which cannot be
+ * affected never leave the fast path, and no user-supplied function is called
+ * more often than it has to be:
+ *
+ * 1. No custom `driverFactory`, or one that hands back a constructed driver
+ * rather than a config — nothing context-derived to compare. Reuse.
+ * 2. The security context is byte-for-byte what the cached driver was built
+ * from. Reuse, without calling the factory at all. This is the common
+ * case: `requestId` changes per request, credentials do not.
+ * 3. The security context changed, so ask the factory. Most factories ignore
+ * it and return an identical config — reuse, and remember the new context
+ * so step 2 short-circuits next time.
+ * 4. The config genuinely changed. Rebuild.
+ *
+ * Step 4 is what fixes a rotated per-user credential: previously the driver
+ * built from the first request's token was reused for the life of the
+ * process, so every new connection it opened failed to authenticate.
+ *
+ * Note this follows the documented contract of `contextToOrchestratorId` —
+ * that it is the cache key for database connections. Two contexts that
+ * resolve to different connections but share an orchestrator id are a
+ * misconfiguration; they were already sharing one user's connection before
+ * this change.
+ */
+ protected async resolveDriverStaleness(
+ origin: DriverOrigin | undefined,
+ context: DriverContext,
+ ): Promise<{ stale: false } | { stale: true, factoryResult: DriverFactoryResult }> {
+ if (
+ !origin ||
+ origin.configFingerprint === null ||
+ !this.optsHandler.isCustomDriverFactory()
+ ) {
+ return { stale: false };
+ }
+
+ const securityContextFingerprint = fingerprint(context.securityContext);
+
+ if (
+ securityContextFingerprint === null ||
+ securityContextFingerprint === origin.securityContextFingerprint
+ ) {
+ return { stale: false };
+ }
+
+ let value: DriverConfig | BaseDriver;
+
+ try {
+ value = await this.options.driverFactory(context);
+ } catch (error) {
+ // This call is a probe, not the request's own resolution: a cache hit
+ // never used to invoke the factory at all, so letting a transient failure
+ // here propagate would fail a query the cached driver could have served.
+ // Degrade to reuse, as with anything else that cannot be compared.
+ this.logger('Driver staleness check error', {
+ dataSource: context.dataSource,
+ error: (error as Error).stack || (error as Error).toString(),
+ });
+
+ return { stale: false };
+ }
+
+ const configFingerprint = isDriver(value) ? null : fingerprint(value);
+
+ if (configFingerprint === null || configFingerprint === origin.configFingerprint) {
+ // A driver the factory constructed for this probe is about to be dropped,
+ // so hand back whatever it opened rather than leaking it. Only reachable
+ // for a factory that returns a config sometimes and a driver other times.
+ if (isDriver(value)) {
+ try {
+ await (value).release();
+ } catch (error) {
+ this.logger('Driver release error', {
+ dataSource: context.dataSource,
+ error: (error as Error).stack || (error as Error).toString(),
+ });
+ }
+ }
+
+ origin.securityContextFingerprint = securityContextFingerprint;
+
+ return { stale: false };
+ }
+
+ return { stale: true, factoryResult: { value, securityContextFingerprint } };
+ }
+
public async testConnections() {
return this.orchestratorStorage.testConnections();
}
diff --git a/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts b/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts
new file mode 100644
index 0000000000000..bb18e293fc7fc
--- /dev/null
+++ b/packages/cubejs-server-core/test/unit/driver-cache-invalidation.test.ts
@@ -0,0 +1,440 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+import { BaseDriver } from '@cubejs-backend/query-orchestrator';
+import type { DriverFactoryByDataSource } from '@cubejs-backend/query-orchestrator';
+
+import { CreateOptions, CubejsServerCore } from '../../src';
+
+type FakeDriver = BaseDriver & {
+ builtFrom: any;
+ release: jest.Mock;
+ testConnection: jest.Mock;
+};
+
+/**
+ * Stands in for real driver construction so these tests exercise the caching
+ * decisions without needing a database. Everything else — the driver factory
+ * closure, the fingerprinting, the rebuild — is the production code path.
+ */
+class TestServerCore extends CubejsServerCore {
+ public builtDrivers: FakeDriver[] = [];
+
+ /** Set to fail the next driver construction, as a bad credential would. */
+ public failNextBuild = false;
+
+ /**
+ * Runs before each staleness probe, standing in for another caller that wins
+ * the race while this one is awaiting the factory. Re-entrant probes skip it,
+ * so the hook can drive the driver factory itself.
+ */
+ public onStalenessProbe: (() => Promise) | undefined;
+
+ private inStalenessHook = false;
+
+ protected async resolveDriverStaleness(origin: any, context: any): Promise {
+ if (this.onStalenessProbe && !this.inStalenessHook) {
+ this.inStalenessHook = true;
+
+ try {
+ await this.onStalenessProbe();
+ } finally {
+ this.inStalenessHook = false;
+ }
+ }
+
+ return super.resolveDriverStaleness(origin, context);
+ }
+
+ protected async createDriverFromFactoryResult(
+ val: any,
+ context: any,
+ options?: any,
+ ): Promise {
+ // A factory that hands back a constructed driver takes the real path — that
+ // branch is exactly what one of these tests is about.
+ if (val instanceof BaseDriver) {
+ return super.createDriverFromFactoryResult(val, context, options);
+ }
+
+ if (this.failNextBuild) {
+ this.failNextBuild = false;
+
+ throw new Error('driver construction failed');
+ }
+
+ const driver = {
+ builtFrom: val,
+ release: jest.fn(async () => {}),
+ testConnection: jest.fn(async () => {}),
+ setLogger: () => {},
+ } as unknown as FakeDriver;
+
+ this.builtDrivers.push(driver);
+
+ return driver;
+ }
+}
+
+/**
+ * Boot a core, resolve its orchestrator once, and hand back the driver factory
+ * the orchestrator was created with — the same closure the query orchestrator
+ * calls for every query.
+ */
+async function createCore(options: CreateOptions, securityContext: unknown) {
+ const logger = jest.fn();
+ const core = new TestServerCore({
+ contextToOrchestratorId: () => 'ORCHESTRATOR',
+ logger,
+ ...options,
+ });
+ const spy = jest.spyOn(core, 'createOrchestratorApi');
+
+ await core.getOrchestratorApi({ requestId: 'req-1', securityContext });
+
+ const driverFactory = spy.mock.calls[0][0];
+
+ return {
+ core,
+ driverFactory,
+ logger,
+ /** Log messages, with the params each was reported with. */
+ logged: (message: string) => logger.mock.calls.filter(([msg]) => msg === message).map(([, params]) => params),
+ /** Serve another request through the cached orchestrator. */
+ request: (nextSecurityContext: unknown, requestId = 'req-n') => core.getOrchestratorApi({ requestId, securityContext: nextSecurityContext }),
+ };
+}
+
+describe('driver cache invalidation', () => {
+ beforeAll(() => {
+ process.env.CUBEJS_API_SECRET = 'api-secret';
+ });
+
+ // The CUB-3599 regression: the orchestrator closed over the context of the
+ // request that created it, so a driver built from a per-user credential was
+ // reused for the life of the process. Every connection it opened after the
+ // token rotated failed to authenticate.
+ test('rebuilds the driver when a context-derived credential changes', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+ expect(first.builtFrom).toMatchObject({ password: 'token-a' });
+
+ await request({ token: 'token-b' });
+ const second = await driverFactory('default');
+
+ expect(second).not.toBe(first);
+ expect(second.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+
+ test('releases the driver it replaced, so its pool is drained', async () => {
+ const { driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+ await driverFactory('default');
+
+ // Released off the request path, so give the detached promise a tick.
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(first.release).toHaveBeenCalledTimes(1);
+ });
+
+ test('reuses the driver when the security context is unchanged', async () => {
+ const factory = jest.fn((ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }));
+ const { driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // A different request, same user, same credential.
+ await request({ token: 'token-a' }, 'req-2');
+
+ expect(await driverFactory('default')).toBe(first);
+ // Not re-invoked: an unchanged security context short-circuits before the
+ // user's factory is called at all.
+ expect(factory).toHaveBeenCalledTimes(1);
+ });
+
+ test('reuses the driver when the factory ignores the security context', async () => {
+ const factory = jest.fn(() => ({ type: 'postgres', password: 'from-env' }));
+ const { core, driverFactory, request } = await createCore({ driverFactory: factory }, { user: 'a' });
+
+ const first = await driverFactory('default');
+
+ await request({ user: 'b' });
+ const second = await driverFactory('default');
+
+ expect(second).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ // Asked once more because the context changed, but the answer matched, so
+ // nothing was rebuilt.
+ expect(factory).toHaveBeenCalledTimes(2);
+ // ...and that answer is remembered, so a third request with the same
+ // context does not ask again.
+ await request({ user: 'b' }, 'req-3');
+ await driverFactory('default');
+ expect(factory).toHaveBeenCalledTimes(2);
+ });
+
+ test('never rebuilds when the factory returns a constructed driver', async () => {
+ class ConstructedDriver extends BaseDriver {
+ public release = jest.fn(async () => {});
+
+ public testConnection = jest.fn(async () => {});
+
+ public async query(): Promise {
+ return [];
+ }
+ }
+
+ const driver = new ConstructedDriver();
+ const factory = jest.fn(() => driver);
+ const { driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ // A constructed driver carries no configuration to compare, so the previous
+ // resolve-once behaviour is preserved rather than guessed at.
+ expect(await driverFactory('default')).toBe(first);
+ expect(driver.release).not.toHaveBeenCalled();
+ expect(factory).toHaveBeenCalledTimes(1);
+ });
+
+ test('keeps the pre-aggregation alias pointing at the rebuilt driver', async () => {
+ const { driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+ await request({ token: 'token-b' });
+
+ const rebuilt = await driverFactory('default');
+ const preAgg = await driverFactory('default', true);
+
+ expect(preAgg).toBe(rebuilt);
+ expect(preAgg.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // The `default` and `default@pre_agg` keys share one driver when the data
+ // source has no separate pre-aggregation credentials. A pre-aggregation build
+ // can be the first caller to observe a rotation, so invalidation has to clear
+ // both keys whichever one asked: clearing only the requested key left the
+ // other serving the drained driver, released it twice, and then built a
+ // second pool for what should be a single shared driver.
+ test('rebuilds once when a pre-aggregation build observes the rotation first', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ const preAgg = await driverFactory('default', true);
+ const regular = await driverFactory('default');
+
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(preAgg).toBe(regular);
+ expect(regular.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ // Exactly once — a second release would run against an already-drained pool.
+ expect(first.release).toHaveBeenCalledTimes(1);
+ });
+
+ test('a failed rebuild does not leave a poisoned cache entry', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ await driverFactory('default');
+
+ // The rotation is detected, but building the replacement fails.
+ core.failNextBuild = true;
+ await request({ token: 'token-b' });
+ await expect(driverFactory('default')).rejects.toThrow('driver construction failed');
+
+ // The next attempt resolves from scratch rather than serving the failure.
+ const recovered = await driverFactory('default');
+ expect(recovered.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // The staleness check calls the factory speculatively, on a path that used to
+ // be a pure cache hit. A factory that reads a secret store can fail
+ // transiently, and that must not fail a query the cached driver can serve.
+ test('reuses the cached driver when the staleness probe throws', async () => {
+ let shouldFail = false;
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => {
+ if (shouldFail) {
+ throw new Error('secret store unreachable');
+ }
+
+ return { type: 'postgres', password: ctx.securityContext.token };
+ },
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ shouldFail = true;
+ await request({ token: 'token-b' });
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+
+ // Once the factory recovers, the rotation is picked up as usual.
+ shouldFail = false;
+ const rebuilt = await driverFactory('default');
+ expect(rebuilt).not.toBe(first);
+ expect(rebuilt.builtFrom).toMatchObject({ password: 'token-b' });
+ });
+
+ // Two queries in flight when a rotation lands both see the cached driver as
+ // stale. Only one may rebuild: the loser must not release the driver the
+ // winner has already handed to its caller, nor stand up a second pool.
+ test('concurrent callers rebuild once and release once', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ await request({ token: 'token-b' });
+
+ const [a, b] = await Promise.all([
+ driverFactory('default'),
+ driverFactory('default'),
+ ]);
+
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(a).toBe(b);
+ expect(a.builtFrom).toMatchObject({ password: 'token-b' });
+ expect(core.builtDrivers).toHaveLength(2);
+ expect(first.release).toHaveBeenCalledTimes(1);
+ // The driver handed back is usable — not one whose pool is being drained.
+ expect(a.release).not.toHaveBeenCalled();
+ });
+
+ // Both rebuild logs have to survive the default log level, which drops any
+ // message carrying neither `error` nor `warning`. Without that param the
+ // rebuild — a connection pool being torn down — is invisible in production.
+ test('reports every rebuild, and escalates once it looks like a misconfiguration', async () => {
+ const { driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-0' });
+
+ await driverFactory('default');
+
+ for (let i = 1; i <= 50; i++) {
+ // eslint-disable-next-line no-await-in-loop
+ await request({ token: `token-${i}` }, `req-${i}`);
+ // eslint-disable-next-line no-await-in-loop
+ await driverFactory('default');
+ }
+
+ const rebuilds = logged('Rebuilding driver on configuration change');
+
+ expect(rebuilds).toHaveLength(50);
+ expect(rebuilds.every((params) => params.warning)).toBe(true);
+ expect(rebuilds[0]).toMatchObject({ dataSource: 'default', rebuildCount: 1 });
+ // Counted per alias set, so the 50th rotation reads as 50, not as a pair of
+ // separate counters for `default` and `default@pre_agg`.
+ expect(rebuilds[49]).toMatchObject({ rebuildCount: 50 });
+
+ const escalations = logged('Driver rebuilt repeatedly');
+
+ expect(escalations).toHaveLength(1);
+ expect(escalations[0]).toMatchObject({ rebuildCount: 50 });
+ expect(escalations[0].warning).toContain('contextToOrchestratorId');
+ });
+
+ // Losing the race enough times to exhaust the retry bound, where the winner's
+ // own build then failed, leaves the key invalidated rather than replaced. The
+ // driver this caller started from has already been released by that winner, so
+ // it can be neither handed back nor released again — the only safe move is to
+ // build. Reaching it needs four lost races and a failed build, hence the hook.
+ test('builds instead of reusing a released driver when the retry bound is exhausted', async () => {
+ const { core, driverFactory, request, logged } = await createCore({
+ driverFactory: (ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }),
+ }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ let round = 0;
+
+ core.onStalenessProbe = async () => {
+ round += 1;
+
+ // A different context takes the orchestrator over, then rebuilds — which
+ // replaces the key for the first three rounds. On the fourth that rebuild
+ // fails, so it invalidates the key and releases what it replaced.
+ await request({ token: `concurrent-${round}` }, `req-${round}`);
+
+ if (round === 4) {
+ core.failNextBuild = true;
+ }
+
+ await Promise.resolve(driverFactory('default')).catch(() => {});
+ };
+
+ const resolved = await driverFactory('default');
+
+ core.onStalenessProbe = undefined;
+ await new Promise((resolve) => setImmediate(resolve));
+
+ // Four rounds, so the bound was genuinely exhausted rather than short-circuited.
+ expect(round).toBe(4);
+ expect(resolved).not.toBe(first);
+ // The returned driver is usable: not one some other caller already drained.
+ expect(resolved.release).not.toHaveBeenCalled();
+ // And nothing was released twice on the way there.
+ expect(core.builtDrivers.every((driver) => driver.release.mock.calls.length <= 1)).toBe(true);
+ expect(logged('Driver release error')).toHaveLength(0);
+ });
+
+ test('reuses the driver when the security context cannot be fingerprinted', async () => {
+ const circular: any = { token: 'token-a' };
+ circular.self = circular;
+
+ const factory = jest.fn((ctx: any) => ({ type: 'postgres', password: ctx.securityContext.token }));
+ const { core, driverFactory, request } = await createCore({ driverFactory: factory }, { token: 'token-a' });
+
+ const first = await driverFactory('default');
+
+ // A circular security context fingerprints as null, which every caller must
+ // read as "assume unchanged" rather than rebuilding blindly.
+ await request(circular);
+
+ expect(await driverFactory('default')).toBe(first);
+ expect(core.builtDrivers).toHaveLength(1);
+ });
+
+ // The refresh scheduler's default context carries no security context at all,
+ // so it shares an orchestrator with API traffic on a deployment that does not
+ // partition by user.
+ test('treats an absent security context as a change in both directions', async () => {
+ const { core, driverFactory, request } = await createCore({
+ driverFactory: (ctx: any) => ({
+ type: 'postgres',
+ password: ctx.securityContext?.token ?? 'service-account',
+ }),
+ }, { token: 'token-a' });
+
+ const user = await driverFactory('default');
+ expect(user.builtFrom).toMatchObject({ password: 'token-a' });
+
+ await request(undefined);
+ const scheduler = await driverFactory('default');
+
+ expect(scheduler).not.toBe(user);
+ expect(scheduler.builtFrom).toMatchObject({ password: 'service-account' });
+ expect(core.builtDrivers).toHaveLength(2);
+ });
+});
diff --git a/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts b/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts
new file mode 100644
index 0000000000000..b0572385cfd2d
--- /dev/null
+++ b/packages/cubejs-server-core/test/unit/driver-config-fingerprint.test.ts
@@ -0,0 +1,65 @@
+import { fingerprint } from '../../src/core/driver-config-fingerprint';
+
+describe('fingerprint', () => {
+ test('is stable across property order', () => {
+ expect(fingerprint({ type: 'databricks-jdbc', token: 'a', url: 'u' }))
+ .toEqual(fingerprint({ url: 'u', token: 'a', type: 'databricks-jdbc' }));
+ });
+
+ test('changes when a nested value changes', () => {
+ expect(fingerprint({ type: 'postgres', options: { password: 'one' } }))
+ .not.toEqual(fingerprint({ type: 'postgres', options: { password: 'two' } }));
+ });
+
+ // The case this exists for: a rotated per-user OAuth token has to be visible
+ // as a different configuration, or the cached driver is never rebuilt.
+ test('changes when only the credential changes', () => {
+ const base = { type: 'databricks-jdbc', url: 'jdbc:databricks://host', acceptPolicy: true };
+
+ expect(fingerprint({ ...base, token: 'token-issued-at-09:00' }))
+ .not.toEqual(fingerprint({ ...base, token: 'token-issued-at-10:00' }));
+ });
+
+ test('treats an absent property and an undefined one as equal', () => {
+ expect(fingerprint({ type: 'postgres', catalog: undefined }))
+ .toEqual(fingerprint({ type: 'postgres' }));
+ });
+
+ test('distinguishes arrays by order', () => {
+ expect(fingerprint({ scopes: ['a', 'b'] })).not.toEqual(fingerprint({ scopes: ['b', 'a'] }));
+ });
+
+ test('handles dates, bigints and nested structures', () => {
+ const value = {
+ when: new Date('2026-07-31T12:00:00.000Z'),
+ big: BigInt(42),
+ nested: [{ a: 1 }, { b: [true, null] }],
+ };
+
+ expect(fingerprint(value)).toEqual(fingerprint({
+ nested: [{ a: 1 }, { b: [true, null] }],
+ big: BigInt(42),
+ when: new Date('2026-07-31T12:00:00.000Z'),
+ }));
+ expect(fingerprint(value)).not.toEqual(fingerprint({ ...value, big: BigInt(43) }));
+ });
+
+ test('does not expose the value it hashes', () => {
+ const digest = fingerprint({ type: 'postgres', password: 'super-secret' });
+
+ expect(digest).not.toContain('super-secret');
+ expect(digest).toMatch(/^[0-9a-f]{32}$/);
+ });
+
+ test('returns null for a circular structure rather than throwing', () => {
+ const circular: Record = { type: 'postgres' };
+ circular.self = circular;
+
+ expect(fingerprint(circular)).toBeNull();
+ });
+
+ test('returns a digest for null and undefined', () => {
+ expect(fingerprint(null)).toEqual(fingerprint(undefined));
+ expect(fingerprint(null)).not.toBeNull();
+ });
+});