Skip to content

Commit af167f4

Browse files
fixup! fix(volume): poll staged statements before native completion
Signed-off-by: Aakash Saravanan <aakash.saravanan@databricks.com>
1 parent 0947cc8 commit af167f4

11 files changed

Lines changed: 44 additions & 35 deletions

File tree

‎CONNECTION_PARAMETERS.md‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,17 @@ Per-statement `useCloudFetch` overrides remain unsupported on the kernel backend
7676

7777
## HTTP client, proxy, and retries
7878

79-
| Option | Type | Thrift | Kernel | Default Value | Note |
80-
| --------------------------- | ------------------------ | :----: | :----: | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
81-
| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel uses a total SQL request deadline and a storage idle timeout (15 minutes by default). Positive values set both; `0` keeps the SQL default and disables storage inactivity. Storage idle time bounds reads, upload progress and PUT response waits. |
82-
| `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. |
83-
| `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. |
84-
| `customHeaders` | `Record<string, string>` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. |
85-
| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. |
86-
| `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. |
87-
| `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. |
88-
| `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. |
89-
| `maxConnections` (internal) | `number` | ❌ | ✅ | — / Kernel default | Kernel connection-pool size. Must be a positive integer within the napi `u32` range. |
79+
| Option | Type | Thrift | Kernel | Default Value | Note |
80+
| --------------------------- | ------------------------ | :----: | :----: | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
81+
| `socketTimeout` | `number` (ms) | ✅ | ⚠️ | `900000` / `120000` | Kernel forwards only positive values. `0` means no timeout on Thrift but is omitted on Kernel, which then keeps its 120-second default. |
82+
| `proxy` | `ProxyOptions` | ✅ | ⚠️ | — | `{protocol, host, port, auth}`. Kernel supports HTTP/HTTPS proxies only; Thrift also supports the declared SOCKS variants. |
83+
| `noProxy` (internal) | `string` | ❌ | ⚠️ | — | Runtime-only Kernel bypass list, forwarded as `bypassHosts` only when an explicit `proxy` is also supplied. |
84+
| `customHeaders` | `Record<string, string>` | ✅ | ✅ | — | Sent on workspace, telemetry and feature-flag requests, not OAuth. Workspace calls ignore caller `Authorization`, `User-Agent`, `Content-Type`, `Content-Length`, `Transfer-Encoding`, `Connection`; Kernel also ignores `x-databricks-org-id`. |
85+
| `retryMaxAttempts` | `number` | ✅ | ✅ | `5` | Total attempts, including the initial request. Kernel converts this to its retries-after-first representation internally. |
86+
| `retriesTimeout` | `number` (ms) | ✅ | ✅ | `900000` | Kernel converts milliseconds to whole seconds. |
87+
| `retryDelayMin` | `number` (ms) | ✅ | ✅ | `1000` | Kernel converts milliseconds to whole seconds. |
88+
| `retryDelayMax` | `number` (ms) | ✅ | ✅ | `60000` | Kernel converts milliseconds to whole seconds. |
89+
| `maxConnections` (internal) | `number` | ❌ | ✅ | — / Kernel default | Kernel connection-pool size. Must be a positive integer within the napi `u32` range. |
9090

9191
## TLS / SSL
9292

‎KERNEL_REV‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
ac32e64ee24b6fac30ba230a1ed32bbf3260e4b9
1+
254245b3b4fcc770f543b02e299b114666ac26a2

‎lib/contracts/IDBSQLSession.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ export type ExecuteStatementOptions = {
3232
* `false`/unset (default) runs the blocking direct-results path (faster,
3333
* cancellable mid-compute); `true` submits and polls (returns a pending
3434
* handle before completion). Default is sync, matching the python
35-
* connector's `cursor.execute()`. With `stagingAllowedLocalPath` configured,
36-
* execution waits for staging completion regardless of this option.
35+
* connector's `cursor.execute()`. With stagingAllowedLocalPath set,
36+
* executeStatement resolves only after the statement finishes, regardless of this option.
3737
*/
3838
runAsync?: boolean;
3939
maxRows?: number | bigint | Int64 | null;

‎lib/kernel/KernelAuth.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -523,9 +523,14 @@ export function buildKernelHttpOptions(options: ConnectionOptions): KernelHttpOp
523523
headers.push({ name: 'User-Agent', value: buildUserAgentString(userAgentEntry) });
524524

525525
const http: KernelHttpOptions = { customHeaders: headers };
526-
// Forward 0 as well: it turns off the Volume transfer idle timeout,
527-
// matching Thrift where 0 means no timeout.
528-
if (typeof socketTimeout === 'number' && socketTimeout >= 0) {
526+
// Per-connection socket read timeout (ms). The public `socketTimeout`
527+
// ConnectionOption maps onto the kernel napi `socketTimeoutMs`
528+
// (kernel `HttpConfig::request_timeout` / reqwest `Client::timeout`).
529+
// Only forward a POSITIVE value: `socketTimeout: 0` means "disabled / wait
530+
// indefinitely" on the Thrift path, but forwarding `0` would make reqwest
531+
// time out immediately, so we omit it and let the kernel keep its (large)
532+
// default — preserving the "effectively no idle timeout" semantics.
533+
if (typeof socketTimeout === 'number' && socketTimeout > 0) {
529534
http.socketTimeoutMs = socketTimeout;
530535
}
531536
return http;

‎lib/kernel/KernelErrorMapping.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ const kindCodes: Record<string, string> = {
1212
AlreadyExists: 'EEXIST',
1313
NotADirectory: 'ENOTDIR',
1414
IsADirectory: 'EISDIR',
15-
DirectoryNotEmpty: 'ENOTEMPTY',
1615
};
1716

1817
/**

‎lib/kernel/KernelOperationBackend.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ export default class KernelOperationBackend implements IOperationBackend {
407407
// `arrowSchema` doesn't hit apache-arrow@13's `Unrecognized type
408408
// "Duration" (18)`. Matches what the per-batch fetch path already does.
409409
arrowSchema: patchIpcBytes(arrowSchemaIpc.ipcBytes),
410-
isStagingOperation: Boolean(handle.isVolumeOperation),
410+
isStagingOperation: handle.isVolumeOperation,
411411
};
412412
this.metadata = meta;
413413
return meta;

‎native/kernel/index.d.ts‎

Lines changed: 9 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/e2e/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ The `e2e-test` job in `.github/workflows/main.yml` runs against the minimum patc
2121
Per-test isolation is achieved by:
2222

2323
- **Tables**: all DDL in tests is templated against `${E2E_TABLE_SUFFIX}`, which in CI is `${{ github.sha }}_node${{ matrix.node-suffix }}`. The matrix suffix uses underscores because dots and hyphens aren't valid within these SQL unquoted identifiers.
24-
- **Volume files**: `tests/e2e/staging_ingestion.test.ts` generates per-file `uuid.v4()` names. Multiple matrix entries can read/write the volume concurrently without collisions.
24+
- **Volume files**: `tests/e2e/utils/stagingTests.ts` generates per-file `uuid.v4()` names. Multiple matrix entries can read/write the volume concurrently without collisions.
2525

2626
No test creates or drops the shared catalog/schema/volume. If you add a test that does, you'll need to suffix-unique the resource name too — verify before merging.
2727

‎tests/e2e/kernel/staging-close-e2e.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
import { expect } from 'chai';
22
import { createServer } from 'node:http';
33
import { AddressInfo } from 'node:net';
4-
import { openSession } from '../../../native/kernel';
4+
import { tryGetKernelNative } from '../../../lib/kernel/KernelNativeLoader';
55
import DBSQLClient from '../../../lib/DBSQLClient';
66
import DBSQLSession from '../../../lib/DBSQLSession';
77
import KernelSessionBackend from '../../../lib/kernel/KernelSessionBackend';
88

99
describe('native staging close', () => {
10+
const native = tryGetKernelNative();
11+
if (native === undefined) {
12+
it.skip('kernel native binding not available on this platform');
13+
return;
14+
}
15+
1016
it('closes a session during a pending staged statement with default options', async function () {
1117
this.timeout(10000);
1218
let polled!: () => void;
@@ -28,7 +34,7 @@ describe('native staging close', () => {
2834
server.listen(0, '127.0.0.1', resolve);
2935
});
3036
const context = new DBSQLClient({ logger: { log() {} } });
31-
const connection = await openSession({
37+
const connection = await native.openSession({
3238
hostName: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
3339
httpPath: '/sql/1.0/warehouses/wh',
3440
token: 'test-token',
Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import stagingTests from '../utils/stagingTests';
2-
import config from '../utils/config';
32

43
const kernelDescribe =
54
process.env.DATABRICKS_PECOTESTING_SERVER_HOSTNAME &&
@@ -10,9 +9,9 @@ const kernelDescribe =
109

1110
kernelDescribe('Kernel Staging Test', () => {
1211
stagingTests(() => ({
13-
host: config.host,
14-
path: config.path,
15-
token: config.token,
12+
host: process.env.DATABRICKS_PECOTESTING_SERVER_HOSTNAME as string,
13+
path: process.env.DATABRICKS_PECOTESTING_HTTP_PATH as string,
14+
token: process.env.DATABRICKS_PECOTESTING_TOKEN_PERSONAL as string,
1615
useKernel: true,
1716
}));
1817
});

0 commit comments

Comments
 (0)