From de2cd22be11406b275e6e64377b0f1b2b347a5af Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 13 Nov 2026 23:01:41 +0000 Subject: [PATCH 01/38] Complete Volume transfers through the native backend Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 + connection.go | 36 +++++----- connection_test.go | 35 ++++++++++ doc.go | 10 ++- internal/backend/kernel/backend.go | 11 --- internal/backend/kernel/cgo.go | 5 ++ internal/backend/kernel/errors_classify.go | 3 + .../kernel/include/databricks_kernel.h | 20 +++++- internal/backend/kernel/kernel_test.go | 27 -------- internal/backend/kernel/operation.go | 69 +++++++++++++++++-- internal/backend/kernel/staging.go | 67 ------------------ internal/backend/kernel/staging_test.go | 41 ----------- 12 files changed, 153 insertions(+), 173 deletions(-) delete mode 100644 internal/backend/kernel/staging.go delete mode 100644 internal/backend/kernel/staging_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f93aa25..6db51418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). - **Behavior change:** Support Arrow TIME results on Thrift and SEA as UTC `time.Time` values on `0001-01-01`, preserving precision and using clock-only nested JSON. TIME metadata now reports `TIME`/`time.Time` with no length, including empty and NULL-only results; previously Thrift reported `STRING`/`string` with unbounded length, and SEA reported an empty name with `*interface{}` (databricks/databricks-sql-go#505). - **Behavior change:** Explicit `WithMaxDownloadThreads` / `maxDownloadThreads` settings tuned for Thrift also control kernel buffering and can change memory use and throughput. The kernel window defaults to 16 when unset; explicit chunk-setting overrides take precedence. +- Support Volume PUT/GET/REMOVE through kernel ExecContext with staging roots, including resolved-symlink authorization and context cancellation. QueryContext retains raw instructions. + - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/connection.go b/connection.go index b6269686..48cf6c31 100644 --- a/connection.go +++ b/connection.go @@ -820,25 +820,29 @@ func (c *conn) execStagingOperation( return nil } - if len(driverctx.StagingPathsFromContext(ctx)) != 0 { - // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). - telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { - if c.telemetry != nil { - c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) - c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) - } + if len(driverctx.StagingPathsFromContext(ctx)) == 0 { + return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) + } + if native, ok := op.(interface{ CompleteStaging(context.Context) error }); ok { + if err := native.CompleteStaging(ctx); err != nil { + return dbsqlerrint.NewDriverError(ctx, "error performing staging operation", err) } - row, err = op.Results(ctx, &rows.TelemetryCallbacks{ - OnChunkFetched: telemetryUpdate, - }) - if err != nil { - return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) + return nil + } + // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). + telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { + if c.telemetry != nil { + c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) + c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) } - defer row.Close() //nolint:errcheck - - } else { - return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) } + row, err = op.Results(ctx, &rows.TelemetryCallbacks{ + OnChunkFetched: telemetryUpdate, + }) + if err != nil { + return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) + } + defer row.Close() //nolint:errcheck var sqlRow []driver.Value colNames := row.Columns() diff --git a/connection_test.go b/connection_test.go index 9057e50f..6a69f3bf 100644 --- a/connection_test.go +++ b/connection_test.go @@ -17,7 +17,9 @@ import ( "github.com/apache/thrift/lib/go/thrift" "github.com/pkg/errors" + "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" + "github.com/databricks/databricks-sql-go/internal/backend" thriftbackend "github.com/databricks/databricks-sql-go/internal/backend/thrift" "github.com/databricks/databricks-sql-go/internal/cli_service" "github.com/databricks/databricks-sql-go/internal/client" @@ -1074,3 +1076,36 @@ func TestConn_handleStagingRetry(t *testing.T) { assert.Less(t, elapsed, 2*time.Second, "context cancel should abort PUT retry backoff promptly") }) } + +type nativeStagingStub struct { + backend.Operation + marked bool + completed bool + err error +} + +func (o *nativeStagingStub) StatementID() string { return "volume-test" } +func (o *nativeStagingStub) IsStaging(context.Context) (bool, error) { return o.marked, nil } +func (o *nativeStagingStub) CompleteStaging(context.Context) error { o.completed = true; return o.err } + +func TestConnNativeStagingDispatch(t *testing.T) { + for _, tc := range []struct { + name string + marked, roots, wantCall, wantError bool + }{ + {"ordinary query", false, true, false, false}, + {"missing roots", true, false, false, true}, + {"native completion", true, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + if tc.roots { + ctx = driverctx.NewContextWithStagingInfo(ctx, []string{t.TempDir()}) + } + op := &nativeStagingStub{marked: tc.marked} + err := (&conn{}).execStagingOperation(op, ctx) + assert.Equal(t, tc.wantError, err != nil) + assert.Equal(t, tc.wantCall, op.completed) + }) + } +} diff --git a/doc.go b/doc.go index d2ae07be..dfc557e6 100644 --- a/doc.go +++ b/doc.go @@ -227,8 +227,7 @@ RetryWaitMin / RetryWaitMax / RetryMax, including the disable form, forwarded to kernel's HTTP retry config); and the TLS, proxy, and session-conf (query tags, statement timeout, time zone) options. Federated token providers use one token snapshot during setup. Nothing is silently ignored: WithTimeout and token-provider / external / -static authenticators are rejected at connect; staging (PUT/GET/REMOVE on a -Unity Catalog volume) is rejected at execute. WithMaxRows is accepted but inert (the +static authenticators are rejected at connect. WithMaxRows is accepted but inert (the kernel manages fetching below the C ABI). Kernel HTTP requests inherit the driver's ClientTimeout (900s by default). If @@ -371,6 +370,13 @@ After doing so, you can execute staging operations using this context using the _, err1 := db.ExecContext(ctx, `PUT 'staging/file.csv' INTO '/Volumes/main/staging_test/e2etests/file.csv' OVERWRITE`) +With the kernel backend, ExecContext completes the transfer and reports zero +rows affected. QueryContext returns raw instruction rows without file effects. +Both the requested local path and its resolved symlink target must remain +within the configured staging roots. Symlinks within those roots are permitted; +escapes are rejected, which is stricter than Thrift's requested-path check. +The staging context is required for REMOVE as well as PUT and GET. + # Errors There are three error types exposed via dbsql/errors diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index d8e20d08..bdb8c54f 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -72,7 +72,6 @@ import ( "sync/atomic" "time" - dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/databricks/databricks-sql-go/internal/backend" ) @@ -908,15 +907,5 @@ func (k *KernelBackend) SessionID() string { return k.sessionID } // parameters (req.Params) are bound onto the statement in execute (see // bindParams). func (k *KernelBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { - // Staging (Unity Catalog volume PUT/GET/REMOVE) needs a local file transfer the - // kernel path can't perform, and the kernel C ABI surfaces no IsStagingOperation - // signal to drive conn.execStagingOperation. Reject it here rather than let - // IsStaging return false and report success with no file moved (silent data loss). - // (Bound parameters, by contrast, ARE now supported — bound in execute via the - // kernel's raw-param C ABI.) - if isStagingStatement(req.Query) { - return &kernelOp{}, fmt.Errorf("databricks: staging operations (PUT/GET/REMOVE on a volume) are %w; "+ - "use the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel) - } return k.execute(ctx, req) } diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index a8a79f6d..9e31b65b 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -38,7 +38,9 @@ import "C" import ( "context" "fmt" + "os" "runtime" + "syscall" "unsafe" "github.com/databricks/databricks-sql-go/driverctx" @@ -172,6 +174,9 @@ func lastError(code C.KernelStatusCode) *KernelError { HTTPStatus: uint16(e.http_status), Retryable: bool(e.retryable), } + if code := C.kernel_get_last_error_local_os_error(); code != 0 { + ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} + } if e.sql_state != nil { ke.SQLState = C.GoString(e.sql_state) } diff --git a/internal/backend/kernel/errors_classify.go b/internal/backend/kernel/errors_classify.go index 73b7c39c..7d4fdd7b 100644 --- a/internal/backend/kernel/errors_classify.go +++ b/internal/backend/kernel/errors_classify.go @@ -49,8 +49,11 @@ type KernelError struct { HTTPStatus uint16 Retryable bool QueryID string + cause error } +func (e *KernelError) Unwrap() error { return e.cause } + func (e *KernelError) Error() string { // Append the server query id when present — it is the one correlation handle // to server-side query history, and on the execute-error path (no exec handle) diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 965c09b6..fe88926b 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -190,6 +190,11 @@ typedef struct KernelError { * last kernel call cleared the buffer. */ bool kernel_get_last_error(KernelError* out); +/* These accessors preserve the last error. Strings are borrowed until the next + * error-clearing kernel call; absent strings are NULL, absent OS code is zero. */ +int32_t kernel_get_last_error_local_os_error(void); +const char* kernel_get_last_error_local_kind(void); +const char* kernel_get_last_error_local_path(void); /* ─── Opaque handles ────────────────────────────────────────────────── * @@ -595,12 +600,12 @@ KernelStatusCode kernel_session_config_set_u2m_token_cache_config(KernelSessionC * * Recognised keys (case-insensitive): * - "cloudfetch_enabled" = "true"|"false" — false serves every result - * inline (SEA disposition = INLINE), mirroring JDBC EnableQueryResultDownload=0. + * through paged inline delivery without cloud-storage downloads. * - "cloudfetch_link_prefetch_window" = positive integer. * - "cloudfetch_max_chunks_in_memory" = positive integer (bounds peak RSS; * values above the kernel ceiling of 256 are clamped with a warning). * - "inline_max_chunks_in_memory" = positive integer (inline-Arrow prefetch - * window, used when cloudfetch_enabled = false). + * window for paged inline results). * - "complex_types_as_json" = "true"|"false" — render ARRAY / MAP / STRUCT / * VARIANT / GEOMETRY / GEOGRAPHY as JSON text (PySQL * _use_arrow_native_complex_types=False, JDBC complexDatatypeSupport=false). @@ -966,6 +971,17 @@ KernelStatusCode kernel_executed_async_statement_close( */ KernelStatusCode kernel_result_stream_get_schema(kernel_result_stream_t* stream, struct ArrowSchema* out); +/* Volume completion preserves raw rows. Roots are a JSON array of strings; + * out_status is 0=ordinary result, 1=input bytes required, 2=completed. + * The detached result-stream canceller can interrupt completion. */ +KernelStatusCode kernel_executed_statement_is_volume_operation( + const kernel_executed_statement_t* exec, bool* out); +KernelStatusCode kernel_result_stream_is_volume_operation( + const kernel_result_stream_t* stream, bool* out); +KernelStatusCode kernel_result_stream_complete_volume_operation( + kernel_result_stream_t* stream, const char* allowed_paths_json, + bool remove_not_found_is_success, int32_t* out_status); + KernelStatusCode kernel_result_stream_next_batch(kernel_result_stream_t* stream, struct ArrowArray* out_array, struct ArrowSchema* out_schema); diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index cd379412..3485fb78 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -531,33 +531,6 @@ func TestExecuteHandleLessOpContract(t *testing.T) { } } -// TestExecuteRejectsStaging drives Execute with a staging statement (not just the -// isStagingStatement detector in isolation) to pin the detector→Execute wiring: a -// refactor that dropped or reordered the check would silently reopen the -// silent-no-op data-loss path. Mirrors TestExecuteRejectsParams. -func TestExecuteRejectsStaging(t *testing.T) { - k := &KernelBackend{} - op, err := k.Execute(context.Background(), backend.ExecRequest{ - Query: "PUT '/tmp/f' INTO '/Volumes/main/s/e/f.csv'", - }) - if err == nil { - t.Fatal("expected an error for a staging statement, got nil") - } - if !errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - t.Errorf("staging rejection should wrap ErrNotSupportedByKernel, got %v", err) - } - if op == nil { - t.Fatal("Execute must return a non-nil Operation per the Backend contract") - } - closed, closeErr := op.Close(context.Background()) - if closeErr != nil { - t.Errorf("Close error = %v, want nil", closeErr) - } - if closed { - t.Error("Close on a handle-less op must report closed=false (no CLOSE_STATEMENT)") - } -} - // ExecutionError must satisfy the same public contract as the Thrift path so the // errors.Is → errors.As → SqlState()/QueryId() recipe documented in doc.go works // on the kernel backend too (it previously returned a bare *KernelError that diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index e2f355f0..fa8e130f 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -11,6 +11,7 @@ import "C" import ( "context" "database/sql/driver" + "encoding/json" "errors" "fmt" "sync" @@ -35,8 +36,7 @@ import ( // client-timeout path, let an in-flight cancel finish in the background so // it cannot extend the client deadline // -// Binds any query parameters (bindParams) before executing; staging statements -// are rejected up front by Execute, so none reach here. +// Binds any query parameters (bindParams) before executing. func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { // Log the SQL length, not the text: query bodies can carry PII/secrets in // WHERE/INSERT/SET, and this goes to stderr. Matches the driver's own @@ -384,11 +384,66 @@ func (o *kernelOp) Results(ctx context.Context, callbacks *dbsqlrows.TelemetryCa return newKernelRows(ctx, o, stream, callbacks) } -// IsStaging reports whether this is a staging (PUT/GET/REMOVE) operation. Always -// false on the kernel path: staging statements are rejected up front in -// Execute (isStagingStatement), so no kernelOp is ever produced for one — there -// is nothing for conn.execStagingOperation to act on here. -func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { return false, nil } +func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { + if o.exec == nil { + return false, nil + } + var marked C.bool + err := call(func() C.KernelStatusCode { + return C.kernel_executed_statement_is_volume_operation(o.exec, &marked) + }) + return bool(marked), err +} + +func (o *kernelOp) CompleteStaging(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + var stream *C.kernel_result_stream_t + if err := call(func() C.KernelStatusCode { + return C.kernel_executed_statement_get_result_stream(o.exec, &stream) + }); err != nil { + return toStatementError(err) + } + defer C.kernel_result_stream_close(stream) + var canceller *C.kernel_result_stream_canceller_t + if err := call(func() C.KernelStatusCode { + return C.kernel_result_stream_canceller_new(stream, &canceller) + }); err != nil { + return toStatementError(err) + } + defer C.kernel_result_stream_canceller_free(canceller) + done, stopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(stopped) + select { + case <-ctx.Done(): + C.kernel_result_stream_canceller_cancel(canceller) + case <-done: + } + }() + defer func() { close(done); <-stopped }() + encoded, err := json.Marshal(driverctx.StagingPathsFromContext(ctx)) + if err != nil { + return err + } + paths := newCStr(string(encoded)) + defer paths.free() + var outcome C.int32_t + err = call(func() C.KernelStatusCode { + return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) + }) + if err != nil { + if ctx.Err() != nil { + return fmt.Errorf("Volume transfer: %w: %w", ctx.Err(), toStatementError(err)) + } + return toStatementError(err) + } + if outcome != 2 { + return fmt.Errorf("Volume operation requires a local file instruction") + } + return nil +} // Close best-effort closes the executed statement and its statement handle. It is // idempotent (a second call, or a call after Rows.Close already tore the diff --git a/internal/backend/kernel/staging.go b/internal/backend/kernel/staging.go deleted file mode 100644 index c614169c..00000000 --- a/internal/backend/kernel/staging.go +++ /dev/null @@ -1,67 +0,0 @@ -package kernel - -import "strings" - -// This file is intentionally NOT behind the `cgo && databricks_kernel` build tag. -// Staging detection is pure Go (string inspection of the SQL), so keeping it -// untagged lets its test run under CGO_ENABLED=0. - -// isStagingStatement reports whether sql is a Unity Catalog volume staging -// command (PUT / GET / REMOVE). The Thrift backend learns a statement is staging -// from server result metadata (IsStagingOperation); the kernel C ABI exposes no -// such signal, and the kernel path cannot perform the local file transfer these -// commands require. We therefore detect them from the SQL and reject at execute -// time (see Execute) rather than silently returning success with no file moved — -// which is what returning false from IsStaging would do. Case-insensitive; skips -// leading whitespace AND leading SQL comments (`-- ...` and `/* ... */`), so a -// comment-prefixed staging command is still caught rather than slipping through -// as a silent no-op. Errs toward matching only a clear leading keyword — a false -// negative just yields the kernel's normal "unsupported syntax" server error, -// whereas a false positive would reject a legitimate query. -func isStagingStatement(sql string) bool { - s := stripLeadingSQLComments(sql) - for _, kw := range []string{"PUT", "GET", "REMOVE"} { - if len(s) > len(kw) && - strings.EqualFold(s[:len(kw)], kw) && - isSQLTokenBreak(s[len(kw)]) { - return true - } - } - return false -} - -// stripLeadingSQLComments removes leading whitespace and leading SQL comments -// (both `-- ...` line comments and `/* ... */` block comments), repeatedly, so -// the first real token is exposed. An unterminated block comment consumes the -// rest of the string (yielding ""). Comment bodies are not scanned for nested -// markers — SQL block comments do not nest — and this only touches the LEADING -// run, so a comment later in the statement is untouched. -func stripLeadingSQLComments(sql string) string { - s := strings.TrimSpace(sql) - for { - switch { - case strings.HasPrefix(s, "--"): - if i := strings.IndexByte(s, '\n'); i >= 0 { - s = s[i+1:] - } else { - return "" // line comment to end of input - } - case strings.HasPrefix(s, "/*"): - if i := strings.Index(s[2:], "*/"); i >= 0 { - s = s[2+i+2:] - } else { - return "" // unterminated block comment - } - default: - return s - } - s = strings.TrimSpace(s) - } -} - -// isSQLTokenBreak reports whether b ends a leading keyword — a space or a quote -// (staging commands take a quoted path, e.g. PUT '' INTO ''), so -// "PUT '..." matches but an identifier like "PUTS" or "GETTER" does not. -func isSQLTokenBreak(b byte) bool { - return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\'' || b == '"' -} diff --git a/internal/backend/kernel/staging_test.go b/internal/backend/kernel/staging_test.go deleted file mode 100644 index 91d69794..00000000 --- a/internal/backend/kernel/staging_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package kernel - -import "testing" - -func TestIsStagingStatement(t *testing.T) { - staging := []string{ - "PUT '/tmp/f.csv' INTO '/Volumes/main/s/e/f.csv' OVERWRITE", - "GET '/Volumes/main/s/e/f.csv' TO 'local.csv'", - "REMOVE '/Volumes/main/s/e/f.csv'", - " put '/tmp/f' into '/Volumes/x'", // leading space + lowercase - "Get\t'/Volumes/x' to 'y'", // tab after keyword, mixed case - "/* audit */ PUT '/local' INTO '/Volumes/main/s/v'", // leading block comment - "/* a */ /* b */ PUT '/l' INTO '/Volumes/x'", // multiple block comments - "-- upload the file\nPUT '/l' INTO '/Volumes/x'", // leading line comment, then staging - " /* c */\n -- d\n REMOVE '/Volumes/x'", // mixed comments + whitespace - } - for _, sql := range staging { - if !isStagingStatement(sql) { - t.Errorf("expected staging: %q", sql) - } - } - - notStaging := []string{ - "SELECT 1", - "PUTS ON A SHOW", // PUT is a prefix of an identifier, not a token - "GETTER()", // likewise for GET - "REMOVED_AT FROM t", // likewise for REMOVE - "INSERT INTO t VALUES (1)", // contains INTO but is not staging - "", // empty - "-- PUT '/x' INTO '/y'", // whole line commented out; nothing after - "/* PUT '/x' INTO '/y' */", // whole statement inside a block comment - "SELECT 'PUT ' AS col", // keyword only inside a literal - "/* unterminated PUT '/x'", // unterminated block comment consumes the rest - "/* c */ SELECT 1", // real leading comment, but not a staging command - } - for _, sql := range notStaging { - if isStagingStatement(sql) { - t.Errorf("did not expect staging: %q", sql) - } - } -} From 82ca1e1b0f0c5d3788300362a55867bb1341e7d5 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 00:07:11 +0000 Subject: [PATCH 02/38] Pin the native source for Volume completion Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index de1b52ca..f85bb468 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -0b5d0edfed32c0745d5abc2468fc3070e2eb2a17 +bdc4d35693ab94ee1991799d12a40503fdf66670 From 277929b2544cff3556171483d7a9b164f01f8b06 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 00:15:39 +0000 Subject: [PATCH 03/38] Select the Windows-compatible Volume kernel Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f85bb468..a34a99f0 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bdc4d35693ab94ee1991799d12a40503fdf66670 +c22bd606abc48599ab780a65a1389d141c3414bf From d90432bc1929eff796d02c529edd4755400bd9ce Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 02:53:00 +0000 Subject: [PATCH 04/38] Pin native Volume storage retry and terminal response fixes Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a34a99f0..8a2f8e59 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c22bd606abc48599ab780a65a1389d141c3414bf +22f96facc276836bcac7230c8a2c0987f734f2c7 From a87a3561c57cab068e00b1e926d7cb193b8b1d7d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 03:32:03 +0000 Subject: [PATCH 05/38] Preserve native staging error causes in dispatch checks Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- connection_test.go | 15 +++++++++++---- internal/backend/kernel/operation.go | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8a2f8e59..47d2f1c8 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -22f96facc276836bcac7230c8a2c0987f734f2c7 +08a921377dd79e5a301e007da7290c1984140952 diff --git a/connection_test.go b/connection_test.go index 6a69f3bf..c5ae12bd 100644 --- a/connection_test.go +++ b/connection_test.go @@ -1089,23 +1089,30 @@ func (o *nativeStagingStub) IsStaging(context.Context) (bool, error) { return o. func (o *nativeStagingStub) CompleteStaging(context.Context) error { o.completed = true; return o.err } func TestConnNativeStagingDispatch(t *testing.T) { + completionErr := errors.New("native completion failed") for _, tc := range []struct { name string marked, roots, wantCall, wantError bool + completionErr error }{ - {"ordinary query", false, true, false, false}, - {"missing roots", true, false, false, true}, - {"native completion", true, true, true, false}, + {"ordinary query", false, true, false, false, nil}, + {"missing roots", true, false, false, true, nil}, + {"native completion", true, true, true, false, nil}, + {"native completion error", true, true, true, true, completionErr}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() if tc.roots { ctx = driverctx.NewContextWithStagingInfo(ctx, []string{t.TempDir()}) } - op := &nativeStagingStub{marked: tc.marked} + op := &nativeStagingStub{marked: tc.marked, err: tc.completionErr} err := (&conn{}).execStagingOperation(op, ctx) assert.Equal(t, tc.wantError, err != nil) assert.Equal(t, tc.wantCall, op.completed) + if tc.completionErr != nil { + assert.ErrorIs(t, err, dbsqlerr.DriverError) + assert.ErrorIs(t, err, tc.completionErr) + } }) } } diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index fa8e130f..3a9db25c 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -435,12 +435,12 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { }) if err != nil { if ctx.Err() != nil { - return fmt.Errorf("Volume transfer: %w: %w", ctx.Err(), toStatementError(err)) + return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) } return toStatementError(err) } if outcome != 2 { - return fmt.Errorf("Volume operation requires a local file instruction") + return errors.New("kernel: volume operation requires a local file instruction") } return nil } From d445d87c4d7f59485c8af36d7e874c138b34e7dd Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 05:19:18 +0000 Subject: [PATCH 06/38] Apply session validity policy by Volume failure origin Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/cgo.go | 1 + internal/backend/kernel/errors_classify.go | 1 + .../kernel/include/databricks_kernel.h | 1 + internal/backend/kernel/operation.go | 4 + staging_kernel_test.go | 126 ++++++++++++++++++ 6 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 staging_kernel_test.go diff --git a/KERNEL_REV b/KERNEL_REV index 47d2f1c8..bc57504b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -08a921377dd79e5a301e007da7290c1984140952 +855847bdca71e07285cfc60332fb2f33f8e47786 diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index 9e31b65b..f6ba5999 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -173,6 +173,7 @@ func lastError(code C.KernelStatusCode) *KernelError { VendorCode: int32(e.vendor_code), HTTPStatus: uint16(e.http_status), Retryable: bool(e.retryable), + ErrorCode: C.GoString(C.kernel_get_last_error_code()), } if code := C.kernel_get_last_error_local_os_error(); code != 0 { ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} diff --git a/internal/backend/kernel/errors_classify.go b/internal/backend/kernel/errors_classify.go index 7d4fdd7b..7e07d065 100644 --- a/internal/backend/kernel/errors_classify.go +++ b/internal/backend/kernel/errors_classify.go @@ -49,6 +49,7 @@ type KernelError struct { HTTPStatus uint16 Retryable bool QueryID string + ErrorCode string cause error } diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index fe88926b..a88286dc 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -193,6 +193,7 @@ bool kernel_get_last_error(KernelError* out); /* These accessors preserve the last error. Strings are borrowed until the next * error-clearing kernel call; absent strings are NULL, absent OS code is zero. */ int32_t kernel_get_last_error_local_os_error(void); +const char* kernel_get_last_error_code(void); const char* kernel_get_last_error_local_kind(void); const char* kernel_get_last_error_local_path(void); diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 3a9db25c..cb98619a 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -437,6 +437,10 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { if ctx.Err() != nil { return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) } + var ke *KernelError + if !errors.As(err, &ke) || (ke.ErrorCode != "VOLUME_STORAGE_HTTP_ERROR" && ke.ErrorCode != "VOLUME_STORAGE_NETWORK_ERROR") { + o.backend.evictIfSessionFatal(err) + } return toStatementError(err) } if outcome != 2 { diff --git a/staging_kernel_test.go b/staging_kernel_test.go new file mode 100644 index 00000000..b9193f97 --- /dev/null +++ b/staging_kernel_test.go @@ -0,0 +1,126 @@ +//go:build cgo && databricks_kernel + +package dbsql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/databricks/databricks-sql-go/driverctx" + "github.com/databricks/databricks-sql-go/internal/backend/kernel" + "github.com/stretchr/testify/require" +) + +func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { + for _, mode := range []string{"instruction401", "storage401", "storageBody"} { + t.Run(mode, func(t *testing.T) { + directory := t.TempDir() + destination := filepath.Join(directory, "download.bin") + require.NoError(t, os.WriteFile(destination, []byte("old bytes"), 0600)) + var executions, storage atomic.Int64 + var result map[string]any + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/sessions"): + _, _ = io.WriteString(w, `{"session_id":"volume-session"}`) + case r.Method == "POST" && r.URL.Path == "/api/2.0/sql/statements": + executions.Add(1) + _ = json.NewEncoder(w).Encode(result) + case r.URL.Path == "/api/2.0/sql/statements/volume-stmt/result/chunks/1": + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) + case r.URL.Path == "/object": + storage.Add(1) + if mode == "storage401" { + w.WriteHeader(401) + } else { + w.Header().Set("Content-Length", "20") + _, _ = io.WriteString(w, "short") + } + default: + _, _ = io.WriteString(w, `{}`) + } + })) + defer server.Close() + names := []string{"operation", "presignedUrl", "headers", "localFile"} + values := []string{"GET", server.URL + "/object", "{}", destination} + fields, columns := make([]arrow.Field, 4), make([]arrow.Array, 4) + manifestColumns := make([]map[string]any, 4) + for i, name := range names { + fields[i] = arrow.Field{Name: name, Type: arrow.BinaryTypes.String, Nullable: true} + builder := array.NewStringBuilder(memory.DefaultAllocator) + builder.Append(values[i]) + columns[i] = builder.NewStringArray() + builder.Release() + defer columns[i].Release() + manifestColumns[i] = map[string]any{"name": name, "type_name": "STRING", "type_text": "STRING", "position": i} + } + schema := arrow.NewSchema(fields, nil) + record := array.NewRecordBatch(schema, columns, 1) + defer record.Release() + var attachment bytes.Buffer + writer := ipc.NewWriter(&attachment, ipc.WithSchema(schema)) + require.NoError(t, writer.Write(record)) + require.NoError(t, writer.Close()) + chunk := map[string]any{"chunk_index": 0, "row_offset": 0, "row_count": 1, "attachment": base64.StdEncoding.EncodeToString(attachment.Bytes())} + chunks := 1 + if mode == "instruction401" { + chunk["next_chunk_index"], chunks = 1, 2 + } + result = map[string]any{ + "statement_id": "volume-stmt", "status": map[string]any{"state": "SUCCEEDED"}, + "manifest": map[string]any{"format": "ARROW_STREAM", "is_volume_operation": true, "schema": map[string]any{"column_count": 4, "columns": manifestColumns}, "total_row_count": 1, "total_chunk_count": chunks}, + "result": chunk, + } + connector, err := NewConnector(WithServerHostname(server.URL), WithHTTPPath("/sql/1.0/warehouses/test"), + WithAccessToken("test-token"), WithUseKernel(true), WithSkipTLSHostVerify(), WithRetries(-1, time.Millisecond, time.Millisecond)) + require.NoError(t, err) + db := sql.OpenDB(connector) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + connection, err := db.Conn(ctx) + require.NoError(t, err) + defer connection.Close() + _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + require.Error(t, err) + require.NotErrorIs(t, err, driver.ErrBadConn) + var detail *kernel.KernelError + require.ErrorAs(t, err, &detail) + wantCode := map[string]string{"instruction401": "SESSION_EXPIRED", "storage401": "VOLUME_STORAGE_HTTP_ERROR", "storageBody": "VOLUME_STORAGE_NETWORK_ERROR"}[mode] + require.Equal(t, wantCode, detail.ErrorCode) + require.NoError(t, connection.Raw(func(raw any) error { + require.Equal(t, mode != "instruction401", raw.(driver.Validator).IsValid()) + return nil + })) + data, err := os.ReadFile(destination) + require.NoError(t, err) + require.Equal(t, "old bytes", string(data)) + require.Equal(t, int64(1), executions.Load()) + wantStorage := int64(1) + if mode == "instruction401" { + wantStorage = 0 + } + require.Equal(t, wantStorage, storage.Load()) + }) + } +} From 84ed9eebe731d16972b9e17a0313c230d6a80621 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 05:39:33 +0000 Subject: [PATCH 07/38] Invalidate fatal instruction sessions before returning cancellation Signed-off-by: Aakash Saravanan --- internal/backend/kernel/operation.go | 6 ++-- staging_kernel_test.go | 51 ++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index cb98619a..59956ffb 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -434,13 +434,13 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) }) if err != nil { - if ctx.Err() != nil { - return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) - } var ke *KernelError if !errors.As(err, &ke) || (ke.ErrorCode != "VOLUME_STORAGE_HTTP_ERROR" && ke.ErrorCode != "VOLUME_STORAGE_NETWORK_ERROR") { o.backend.evictIfSessionFatal(err) } + if ctx.Err() != nil { + return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) + } return toStatementError(err) } if outcome != 2 { diff --git a/staging_kernel_test.go b/staging_kernel_test.go index b9193f97..fb769cc5 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -24,12 +24,28 @@ import ( "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/databricks/databricks-sql-go/driverctx" + "github.com/databricks/databricks-sql-go/internal/backend" "github.com/databricks/databricks-sql-go/internal/backend/kernel" "github.com/stretchr/testify/require" ) +type cancelOnCompletionErrorContext struct { + context.Context + cancel context.CancelFunc + errCalls int +} + +func (c *cancelOnCompletionErrorContext) Err() error { + c.errCalls++ + // Deliver cancellation after native completion returns to make the error race deterministic. + if c.errCalls == 2 { + c.cancel() + } + return c.Context.Err() +} + func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { - for _, mode := range []string{"instruction401", "storage401", "storageBody"} { + for _, mode := range []string{"instruction401", "storage401", "storageBody", "instruction401Cancelled", "storage401Cancelled"} { t.Run(mode, func(t *testing.T) { directory := t.TempDir() destination := filepath.Join(directory, "download.bin") @@ -50,7 +66,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) case r.URL.Path == "/object": storage.Add(1) - if mode == "storage401" { + if strings.HasPrefix(mode, "storage401") { w.WriteHeader(401) } else { w.Header().Set("Content-Length", "20") @@ -83,7 +99,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.NoError(t, writer.Close()) chunk := map[string]any{"chunk_index": 0, "row_offset": 0, "row_count": 1, "attachment": base64.StdEncoding.EncodeToString(attachment.Bytes())} chunks := 1 - if mode == "instruction401" { + if strings.HasPrefix(mode, "instruction401") { chunk["next_chunk_index"], chunks = 1, 2 } result = map[string]any{ @@ -101,15 +117,36 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { connection, err := db.Conn(ctx) require.NoError(t, err) defer connection.Close() - _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + if strings.HasSuffix(mode, "Cancelled") { + err = connection.Raw(func(raw any) error { + operation, err := raw.(*conn).backend.Execute(ctx, backend.ExecRequest{Query: "GET instruction"}) + if err != nil { + return err + } + defer operation.Close(ctx) + cancelContext, cancelError := context.WithCancel(ctx) + defer cancelError() + stagingContext := driverctx.NewContextWithStagingInfo(&cancelOnCompletionErrorContext{Context: cancelContext, cancel: cancelError}, []string{directory}) + return operation.(interface{ CompleteStaging(context.Context) error }).CompleteStaging(stagingContext) + }) + require.ErrorIs(t, err, context.Canceled) + } else { + _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + } require.Error(t, err) require.NotErrorIs(t, err, driver.ErrBadConn) var detail *kernel.KernelError require.ErrorAs(t, err, &detail) - wantCode := map[string]string{"instruction401": "SESSION_EXPIRED", "storage401": "VOLUME_STORAGE_HTTP_ERROR", "storageBody": "VOLUME_STORAGE_NETWORK_ERROR"}[mode] + wantCode := "VOLUME_STORAGE_HTTP_ERROR" + if strings.HasPrefix(mode, "instruction401") { + wantCode = "SESSION_EXPIRED" + } + if mode == "storageBody" { + wantCode = "VOLUME_STORAGE_NETWORK_ERROR" + } require.Equal(t, wantCode, detail.ErrorCode) require.NoError(t, connection.Raw(func(raw any) error { - require.Equal(t, mode != "instruction401", raw.(driver.Validator).IsValid()) + require.Equal(t, !strings.HasPrefix(mode, "instruction401"), raw.(driver.Validator).IsValid()) return nil })) data, err := os.ReadFile(destination) @@ -117,7 +154,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.Equal(t, "old bytes", string(data)) require.Equal(t, int64(1), executions.Load()) wantStorage := int64(1) - if mode == "instruction401" { + if strings.HasPrefix(mode, "instruction401") { wantStorage = 0 } require.Equal(t, wantStorage, storage.Load()) From 23a569f4d08f030c702847f4d3fba8aa0b5412f2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 23:54:44 +0000 Subject: [PATCH 08/38] build(kernel): select corrected Volume completion Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index bc57504b..53687ee1 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -855847bdca71e07285cfc60332fb2f33f8e47786 +a543bbc572b627f059461fce351713da8cb59df9 From 9c710a84da26e7b166976516df291befeb9f843a Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 01:48:44 +0000 Subject: [PATCH 09/38] build: select native Volume cancellation and error handling Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 53687ee1..3232e69a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -a543bbc572b627f059461fce351713da8cb59df9 +566fe7f0a8a2fa246db95045a7000136ee6070d5 From ae67199ce7a5ae3b9448f4504dbb83777b8cb28f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 05:35:54 +0000 Subject: [PATCH 10/38] build: use native root authorization fix Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/include/databricks_kernel.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 3232e69a..e0f04c9a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -566fe7f0a8a2fa246db95045a7000136ee6070d5 +de28034ec75aa6773a5e4e904d77f85511e1aa78 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index a88286dc..b6e2ded7 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -375,6 +375,7 @@ KernelStatusCode kernel_session_config_set_sea_response_compression_enabled( /* Configure the total timeout for one HTTP request, from connection start * through response-body completion. A separate 30s connection timeout also * applies, so connection establishment is bounded by the shorter limit. + * Does not bound Volume storage requests or configure their inactivity limit. * `request_timeout_ms == 0` keeps the kernel default (120s). */ KernelStatusCode kernel_session_config_set_request_timeout(KernelSessionConfig* config, uint64_t request_timeout_ms); From 85738b8f7082f7ce4ad2c2e9f822eb1873bc4193 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 07:56:57 +0000 Subject: [PATCH 11/38] build: update kernel source for replayable uploads Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e0f04c9a..a85af789 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -de28034ec75aa6773a5e4e904d77f85511e1aa78 +8dd87c0ec0199cae6b3e5bf9cd973e7025409b45 From 32b0a16fc5d11069baf660e41e43619065a33962 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 08:21:05 +0000 Subject: [PATCH 12/38] build: update kernel for local stream close Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a85af789..e3eed1aa 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -8dd87c0ec0199cae6b3e5bf9cd973e7025409b45 +5b2b1da9aef7444f614d99f05cd7512a1afbe075 From 8d0c2c3d72ae268b98ef312b078db3fbc7c09d6a Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 23:44:26 +0000 Subject: [PATCH 13/38] fix(volume): select replayable file uploads and tighten validation Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e3eed1aa..d7abc8c7 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -5b2b1da9aef7444f614d99f05cd7512a1afbe075 +1347031d3c66e9179cec0a404efebe3685c26802 From 4c5cc0d31e85e6f09f39c9972977d10d7c1dada8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 00:49:23 +0000 Subject: [PATCH 14/38] fix(kernel): use permission-preserving Volume downloads Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d7abc8c7..8707bb23 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -1347031d3c66e9179cec0a404efebe3685c26802 +f265b7d78e045d7d28f082084d60e2b0dd3ba367 From e37844bc06a68e051be48dca85a994963dc21291 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 01:05:07 +0000 Subject: [PATCH 15/38] fix(kernel): remove obsolete staging rejection example Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- examples/kernel/main.go | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8707bb23..2df78f70 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f265b7d78e045d7d28f082084d60e2b0dd3ba367 +de19f0dd234c5dc0894c6267001630c923901f32 diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 51d99c31..e3f19775 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -87,14 +87,4 @@ func main() { log.Fatalf("query: %v", err) } fmt.Printf("server version: %s\n", version) - - // Options the kernel can't honor are rejected with an error wrapping - // ErrNotSupportedByKernel, never silently ignored. Staging (PUT/GET/REMOVE on a - // Unity Catalog volume) is one such feature, rejected at execute. - _, err = db.ExecContext(ctx, "PUT '/tmp/local.csv' INTO '/Volumes/main/default/vol/f.csv' OVERWRITE") - if errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - fmt.Println("staging is not supported on the kernel backend (use Thrift for staging)") - } else if err != nil { - fmt.Printf("staging: %v\n", err) - } } From aec341656e695cb203d33082ad85e12727d2af2c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 19:29:02 +0000 Subject: [PATCH 16/38] Pin kernel with statement lifecycle cleanup Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 2df78f70..1b8d987f 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -de19f0dd234c5dc0894c6267001630c923901f32 +728f6cf0931a9e3c321f63534fce240115bdb2f5 From b93dacd476a822977c940267793737b6c89405d0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 21:17:22 +0000 Subject: [PATCH 17/38] build: select kernel with async result cleanup Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 1b8d987f..d6c502ff 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -728f6cf0931a9e3c321f63534fce240115bdb2f5 +029c59cc42727d0294e1a2dc378c78929030a919 From c1afd7c85d23c1d472745a695e24089d46ab7653 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 21:51:22 +0000 Subject: [PATCH 18/38] build: select kernel with Volume instruction error telemetry Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d6c502ff..d5bed17a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -029c59cc42727d0294e1a2dc378c78929030a919 +ab4f133620fce52f9932508f0b245db630a57c48 From a2b3600e80dad20baff4232fa11ecfeda14f7ab5 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 23:57:11 +0000 Subject: [PATCH 19/38] build: pin kernel with complete Volume instruction handling Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d5bed17a..28a4d122 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ab4f133620fce52f9932508f0b245db630a57c48 +d9fcf834d36dc7f3de1621225db716200151fa0f From cfae1195e7055ab598d522d39ffca8796b814dbb Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 17 Nov 2026 19:30:12 +0000 Subject: [PATCH 20/38] build: use corrected kernel volume completion Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 28a4d122..d90f708d 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -d9fcf834d36dc7f3de1621225db716200151fa0f +4e90b0d058469dcb9cc67a8114edcbea1d2d6831 From 71e111fa83c94254924553da45cdf180bbf8b0d5 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 17 Nov 2026 22:01:13 +0000 Subject: [PATCH 21/38] build: select kernel with origin-safe Volume redirects Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d90f708d..9faa7f3b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -4e90b0d058469dcb9cc67a8114edcbea1d2d6831 +eaf1382d605dc18a0e5c220c3e502f9cbd7e9546 From f54641fbdea276c844b64a200a08682bd66f092f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 17 Nov 2026 22:30:09 +0000 Subject: [PATCH 22/38] build: select kernel with normalized upload lengths Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 9faa7f3b..496a772e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -eaf1382d605dc18a0e5c220c3e502f9cbd7e9546 +d3b85c23183c2415b2dbb4385b348f3fed6e21ae From 9c482279b58dc048e4ca6b3bde071bf0dc688ac8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 17 Nov 2026 23:26:00 +0000 Subject: [PATCH 23/38] docs: keep unreleased changelog entries in one list Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6db51418..2d9e9a10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ - **Behavior change:** Support Arrow TIME results on Thrift and SEA as UTC `time.Time` values on `0001-01-01`, preserving precision and using clock-only nested JSON. TIME metadata now reports `TIME`/`time.Time` with no length, including empty and NULL-only results; previously Thrift reported `STRING`/`string` with unbounded length, and SEA reported an empty name with `*interface{}` (databricks/databricks-sql-go#505). - **Behavior change:** Explicit `WithMaxDownloadThreads` / `maxDownloadThreads` settings tuned for Thrift also control kernel buffering and can change memory use and throughput. The kernel window defaults to 16 when unset; explicit chunk-setting overrides take precedence. - Support Volume PUT/GET/REMOVE through kernel ExecContext with staging roots, including resolved-symlink authorization and context cancellation. QueryContext retains raw instructions. - - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). From c7553ab1f27e79982dadd8e797b376d0c5766848 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Tue, 17 Nov 2026 23:56:14 +0000 Subject: [PATCH 24/38] build: pin kernel completion telemetry correction Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 496a772e..085b4541 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -d3b85c23183c2415b2dbb4385b348f3fed6e21ae +7af876f6782418f9c90d3d0c7dbf96fab278f75a From 5b61b6857a8aca95052120f54361f29545326764 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 00:34:13 +0000 Subject: [PATCH 25/38] Scope Volume timeout docs and isolate failure-origin fixture Signed-off-by: Aakash Saravanan --- doc.go | 5 +++-- staging_kernel_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc.go b/doc.go index dfc557e6..85eb8591 100644 --- a/doc.go +++ b/doc.go @@ -230,9 +230,10 @@ during setup. Nothing is silently ignored: WithTimeout and token-provider / exte static authenticators are rejected at connect. WithMaxRows is accepted but inert (the kernel manages fetching below the C ABI). -Kernel HTTP requests inherit the driver's ClientTimeout (900s by default). If +Kernel SQL HTTP requests inherit the driver's ClientTimeout (900s by default). If that value is zero, the kernel uses its own 120s default; zero is neither -unlimited nor an immediate timeout. +unlimited nor an immediate timeout. Use a context deadline to bound Volume storage +transfers; ClientTimeout does not apply to storage requests. OAuth U2M is interactive: on a cache miss, connecting launches the system browser and blocks until login completes or the kernel's ~120s callback timeout expires. Because diff --git a/staging_kernel_test.go b/staging_kernel_test.go index fb769cc5..7916f245 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -103,7 +103,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { chunk["next_chunk_index"], chunks = 1, 2 } result = map[string]any{ - "statement_id": "volume-stmt", "status": map[string]any{"state": "SUCCEEDED"}, + "statement_id": "volume-stmt", "status": map[string]any{"state": "CLOSED"}, "manifest": map[string]any{"format": "ARROW_STREAM", "is_volume_operation": true, "schema": map[string]any{"column_count": 4, "columns": manifestColumns}, "total_row_count": 1, "total_chunk_count": chunks}, "result": chunk, } From 6766d95b64da22c66eac3de2fa37fb1103242901 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 03:33:25 +0000 Subject: [PATCH 26/38] build: select kernel with restricted-directory GET support Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 085b4541..9c0856a4 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -7af876f6782418f9c90d3d0c7dbf96fab278f75a +21856dea28e87fdf61d328e7f21d5484d7b618b4 From 233f91a8946a315f46df7913cb2a41d79f3b3dd0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Wed, 18 Nov 2026 22:15:31 +0000 Subject: [PATCH 27/38] build: pin kernel with bounded symlink resolution Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 9c0856a4..8896a687 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -21856dea28e87fdf61d328e7f21d5484d7b618b4 +3c5342771df2df19d8a898c4deea2974e5c2383e From 6cdd53196c22ed1f9d49d4908c1a736f4bc6221c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 19 Nov 2026 04:03:09 +0000 Subject: [PATCH 28/38] refactor(volume): use typed completion outcomes Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- connection.go | 42 ++++++++++--------- connection_test.go | 7 ++-- .../kernel/include/databricks_kernel.h | 18 +++++--- internal/backend/kernel/kernel_test.go | 8 +++- internal/backend/kernel/operation.go | 4 +- staging_kernel_test.go | 15 ------- 7 files changed, 47 insertions(+), 49 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8896a687..bc49f880 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -3c5342771df2df19d8a898c4deea2974e5c2383e +2fe28af460d0db6eeb8c653e52eede954c90374c diff --git a/connection.go b/connection.go index 48cf6c31..1efd3ead 100644 --- a/connection.go +++ b/connection.go @@ -820,29 +820,31 @@ func (c *conn) execStagingOperation( return nil } - if len(driverctx.StagingPathsFromContext(ctx)) == 0 { - return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) - } - if native, ok := op.(interface{ CompleteStaging(context.Context) error }); ok { - if err := native.CompleteStaging(ctx); err != nil { - return dbsqlerrint.NewDriverError(ctx, "error performing staging operation", err) + if len(driverctx.StagingPathsFromContext(ctx)) != 0 { + if native, ok := op.(interface{ CompleteStaging(context.Context) error }); ok { + if err := native.CompleteStaging(ctx); err != nil { + return dbsqlerrint.NewDriverError(ctx, "error performing staging operation", err) + } + return nil } - return nil - } - // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). - telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { - if c.telemetry != nil { - c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) - c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) + // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). + telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { + if c.telemetry != nil { + c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) + c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) + } } + row, err = op.Results(ctx, &rows.TelemetryCallbacks{ + OnChunkFetched: telemetryUpdate, + }) + if err != nil { + return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) + } + defer row.Close() //nolint:errcheck + + } else { + return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) } - row, err = op.Results(ctx, &rows.TelemetryCallbacks{ - OnChunkFetched: telemetryUpdate, - }) - if err != nil { - return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) - } - defer row.Close() //nolint:errcheck var sqlRow []driver.Value colNames := row.Columns() diff --git a/connection_test.go b/connection_test.go index c5ae12bd..4ca640cd 100644 --- a/connection_test.go +++ b/connection_test.go @@ -1095,10 +1095,9 @@ func TestConnNativeStagingDispatch(t *testing.T) { marked, roots, wantCall, wantError bool completionErr error }{ - {"ordinary query", false, true, false, false, nil}, - {"missing roots", true, false, false, true, nil}, - {"native completion", true, true, true, false, nil}, - {"native completion error", true, true, true, true, completionErr}, + {name: "missing roots", marked: true, wantError: true}, + {name: "native completion", marked: true, roots: true, wantCall: true}, + {name: "native completion error", marked: true, roots: true, wantCall: true, wantError: true, completionErr: completionErr}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index b6e2ded7..6dc7380f 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -54,8 +54,9 @@ * `KernelStatusCode_InvalidStatementHandle`. Don't hold two live * executed handles from one statement. * - **A result stream borrows its executed handle.** Closing the executed - * handle invalidates any stream obtained from it: subsequent stream - * calls return `InvalidStatementHandle` (a defined error, not UB). + * handle invalidates any stream obtained from it: schema access, fetching + * and Volume completion return `InvalidStatementHandle` (a defined error, + * not UB). The cached Volume marker remains readable until stream close. * - **Close is awaited.** `kernel_session_close` waits for the server-side * delete to complete and flushes close-session telemetry before returning. * If close fails, the kernel still schedules best-effort cleanup when the @@ -194,7 +195,6 @@ bool kernel_get_last_error(KernelError* out); * error-clearing kernel call; absent strings are NULL, absent OS code is zero. */ int32_t kernel_get_last_error_local_os_error(void); const char* kernel_get_last_error_code(void); -const char* kernel_get_last_error_local_kind(void); const char* kernel_get_last_error_local_path(void); /* ─── Opaque handles ────────────────────────────────────────────────── @@ -973,16 +973,22 @@ KernelStatusCode kernel_executed_async_statement_close( */ KernelStatusCode kernel_result_stream_get_schema(kernel_result_stream_t* stream, struct ArrowSchema* out); -/* Volume completion preserves raw rows. Roots are a JSON array of strings; - * out_status is 0=ordinary result, 1=input bytes required, 2=completed. +typedef enum KernelVolumeCompletion { + KernelVolumeCompletion_NotVolume = 0, + KernelVolumeCompletion_NeedsInputStream = 1, + KernelVolumeCompletion_Completed = 2 +} KernelVolumeCompletion; + +/* Volume completion preserves raw rows. Roots are a JSON array of strings. * The detached result-stream canceller can interrupt completion. */ KernelStatusCode kernel_executed_statement_is_volume_operation( const kernel_executed_statement_t* exec, bool* out); +/* The cached marker does not establish whether the executed owner is valid. */ KernelStatusCode kernel_result_stream_is_volume_operation( const kernel_result_stream_t* stream, bool* out); KernelStatusCode kernel_result_stream_complete_volume_operation( kernel_result_stream_t* stream, const char* allowed_paths_json, - bool remove_not_found_is_success, int32_t* out_status); + bool remove_not_found_is_success, KernelVolumeCompletion* out_status); KernelStatusCode kernel_result_stream_next_batch(kernel_result_stream_t* stream, struct ArrowArray* out_array, diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index 3485fb78..0c0dc6eb 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -8,9 +8,11 @@ import ( "database/sql/driver" "encoding/json" "errors" + "io/fs" "os" "strings" "sync" + "syscall" "testing" "time" @@ -542,7 +544,8 @@ func TestExecutionErrorContract(t *testing.T) { t.Errorf("ExecutionError(nil) = %v, want nil", got) } - cause := &KernelError{Code: statusSqlError, Message: "boom", SQLState: "42000", QueryID: "q-123"} + cause := &KernelError{Code: statusSqlError, Message: "boom", SQLState: "42000", QueryID: "q-123", + cause: &os.PathError{Op: "file", Path: "missing.bin", Err: syscall.ENOENT}} err := o.ExecutionError(context.Background(), cause) if err == nil { t.Fatal("ExecutionError(cause) should not be nil") @@ -568,6 +571,9 @@ func TestExecutionErrorContract(t *testing.T) { if !errors.As(err, &ke) { t.Error("the *KernelError cause should remain reachable via errors.As") } + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("local file error should match fs.ErrNotExist; got %v", err) + } } // The execute error path must NEVER report retryable, even when the kernel marks diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 59956ffb..2dc1d8f0 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -429,7 +429,7 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { } paths := newCStr(string(encoded)) defer paths.free() - var outcome C.int32_t + var outcome C.KernelVolumeCompletion err = call(func() C.KernelStatusCode { return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) }) @@ -443,7 +443,7 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { } return toStatementError(err) } - if outcome != 2 { + if outcome != C.KernelVolumeCompletion_Completed { return errors.New("kernel: volume operation requires a local file instruction") } return nil diff --git a/staging_kernel_test.go b/staging_kernel_test.go index 7916f245..50edd67a 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -12,10 +12,8 @@ import ( "io" "net/http" "net/http/httptest" - "os" "path/filepath" "strings" - "sync/atomic" "testing" "time" @@ -49,8 +47,6 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { t.Run(mode, func(t *testing.T) { directory := t.TempDir() destination := filepath.Join(directory, "download.bin") - require.NoError(t, os.WriteFile(destination, []byte("old bytes"), 0600)) - var executions, storage atomic.Int64 var result map[string]any server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) @@ -59,13 +55,11 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/sessions"): _, _ = io.WriteString(w, `{"session_id":"volume-session"}`) case r.Method == "POST" && r.URL.Path == "/api/2.0/sql/statements": - executions.Add(1) _ = json.NewEncoder(w).Encode(result) case r.URL.Path == "/api/2.0/sql/statements/volume-stmt/result/chunks/1": w.WriteHeader(401) _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) case r.URL.Path == "/object": - storage.Add(1) if strings.HasPrefix(mode, "storage401") { w.WriteHeader(401) } else { @@ -149,15 +143,6 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.Equal(t, !strings.HasPrefix(mode, "instruction401"), raw.(driver.Validator).IsValid()) return nil })) - data, err := os.ReadFile(destination) - require.NoError(t, err) - require.Equal(t, "old bytes", string(data)) - require.Equal(t, int64(1), executions.Load()) - wantStorage := int64(1) - if strings.HasPrefix(mode, "instruction401") { - wantStorage = 0 - } - require.Equal(t, wantStorage, storage.Load()) }) } } From caf4afca344347393364094976513fd8031bbb1b Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Thu, 19 Nov 2026 17:06:20 +0000 Subject: [PATCH 29/38] build: select updated Volume kernel Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/include/databricks_kernel.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index bc49f880..17fa057e 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -2fe28af460d0db6eeb8c653e52eede954c90374c +c838ba9a3fce6fc5d4178640df26b26c7e6cabb3 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 6dc7380f..b41b99f5 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -986,6 +986,7 @@ KernelStatusCode kernel_executed_statement_is_volume_operation( /* The cached marker does not establish whether the executed owner is valid. */ KernelStatusCode kernel_result_stream_is_volume_operation( const kernel_result_stream_t* stream, bool* out); +/* out_status is written only on success. */ KernelStatusCode kernel_result_stream_complete_volume_operation( kernel_result_stream_t* stream, const char* allowed_paths_json, bool remove_not_found_is_success, KernelVolumeCompletion* out_status); From e3930d587ced68bf0d339659a6fe2e61f871dfd4 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 01:11:41 +0000 Subject: [PATCH 30/38] fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- connection.go | 2 +- connection_test.go | 1 + doc.go | 2 +- internal/backend/backend.go | 5 ++ internal/backend/kernel/backend.go | 4 +- internal/backend/kernel/cgo.go | 1 + internal/backend/kernel/errors_classify.go | 10 ++- .../backend/kernel/errors_classify_test.go | 4 +- internal/backend/kernel/operation.go | 23 +++-- staging_kernel_test.go | 84 ++++++++++++++----- 10 files changed, 96 insertions(+), 40 deletions(-) diff --git a/connection.go b/connection.go index 1efd3ead..40439341 100644 --- a/connection.go +++ b/connection.go @@ -821,7 +821,7 @@ func (c *conn) execStagingOperation( } if len(driverctx.StagingPathsFromContext(ctx)) != 0 { - if native, ok := op.(interface{ CompleteStaging(context.Context) error }); ok { + if native, ok := op.(backend.StagingCompleter); ok { if err := native.CompleteStaging(ctx); err != nil { return dbsqlerrint.NewDriverError(ctx, "error performing staging operation", err) } diff --git a/connection_test.go b/connection_test.go index 4ca640cd..566d0f40 100644 --- a/connection_test.go +++ b/connection_test.go @@ -1096,6 +1096,7 @@ func TestConnNativeStagingDispatch(t *testing.T) { completionErr error }{ {name: "missing roots", marked: true, wantError: true}, + {name: "unmarked operation with roots", roots: true}, {name: "native completion", marked: true, roots: true, wantCall: true}, {name: "native completion error", marked: true, roots: true, wantCall: true, wantError: true, completionErr: completionErr}, } { diff --git a/doc.go b/doc.go index 85eb8591..653ec5b4 100644 --- a/doc.go +++ b/doc.go @@ -375,7 +375,7 @@ With the kernel backend, ExecContext completes the transfer and reports zero rows affected. QueryContext returns raw instruction rows without file effects. Both the requested local path and its resolved symlink target must remain within the configured staging roots. Symlinks within those roots are permitted; -escapes are rejected, which is stricter than Thrift's requested-path check. +escapes are rejected. The staging context is required for REMOVE as well as PUT and GET. # Errors diff --git a/internal/backend/backend.go b/internal/backend/backend.go index 795c6be9..0ca41bf5 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -114,6 +114,11 @@ type ResultFormatReporter interface { ResultFormat() string } +// StagingCompleter completes server-marked staging operations in the backend. +type StagingCompleter interface { + CompleteStaging(context.Context) error +} + // ExecRequest is a backend-neutral statement-execution request. Per-backend wire // options (Arrow flags, CloudFetch, direct results, query tags, timeouts) are // derived from the config and context inside the backend implementation. diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index bdb8c54f..6dbd235d 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -65,7 +65,6 @@ import "C" import ( "context" - "errors" "fmt" "strconv" "strings" @@ -892,8 +891,7 @@ func (k *KernelBackend) markSessionDead() { k.valid = false } // Uses errors.As, not a bare type assertion, so it still fires if a caller wraps // the KernelError before passing it here. func (k *KernelBackend) evictIfSessionFatal(err error) { - var ke *KernelError - if errors.As(err, &ke) && isSessionFatal(ke.Code) { + if isSessionFatal(err) { k.markSessionDead() } } diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index f6ba5999..56985856 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -176,6 +176,7 @@ func lastError(code C.KernelStatusCode) *KernelError { ErrorCode: C.GoString(C.kernel_get_last_error_code()), } if code := C.kernel_get_last_error_local_os_error(); code != 0 { + // The C ABI reports a path and errno without a syscall name. ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} } if e.sql_state != nil { diff --git a/internal/backend/kernel/errors_classify.go b/internal/backend/kernel/errors_classify.go index 7e07d065..5c433403 100644 --- a/internal/backend/kernel/errors_classify.go +++ b/internal/backend/kernel/errors_classify.go @@ -133,15 +133,19 @@ func isBadConnection(code int) bool { } } -// isSessionFatal reports whether a status code means the server-side session is no +// isSessionFatal reports whether an error means the server-side session is no // longer usable, so the conn must be evicted from the pool rather than reused for // the next query. Broader than isBadConnection: it also covers Unauthenticated (an // expired/revoked token kills the session) — but unlike isBadConnection it is NOT // used to produce driver.ErrBadConn on the statement path, so eviction happens // without database/sql replaying the statement (see toStatementError + the // KernelBackend.markSessionDead call sites). -func isSessionFatal(code int) bool { - switch code { +func isSessionFatal(err error) bool { + var detail *KernelError + if !errors.As(err, &detail) || detail.ErrorCode == "VOLUME_STORAGE_HTTP_ERROR" || detail.ErrorCode == "VOLUME_STORAGE_NETWORK_ERROR" { + return false + } + switch detail.Code { case statusUnauthenticated, statusUnavailable, statusNetworkError: return true default: diff --git a/internal/backend/kernel/errors_classify_test.go b/internal/backend/kernel/errors_classify_test.go index 32eeec09..e2c38dd8 100644 --- a/internal/backend/kernel/errors_classify_test.go +++ b/internal/backend/kernel/errors_classify_test.go @@ -39,13 +39,13 @@ func TestIsBadConnection(t *testing.T) { func TestIsSessionFatal(t *testing.T) { fatal := []int{statusUnauthenticated, statusUnavailable, statusNetworkError} for _, code := range fatal { - if !isSessionFatal(code) { + if !isSessionFatal(&KernelError{Code: code}) { t.Errorf("code %d should be session-fatal", code) } } notFatal := []int{statusInvalidArgument, statusSqlError, statusTimeout} for _, code := range notFatal { - if isSessionFatal(code) { + if isSessionFatal(&KernelError{Code: code}) { t.Errorf("code %d should not be session-fatal", code) } } diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 2dc1d8f0..a7f00188 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -16,6 +16,7 @@ import ( "fmt" "sync" "time" + "unicode/utf8" "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" @@ -350,6 +351,7 @@ type kernelOp struct { } var _ backend.Operation = (*kernelOp)(nil) +var _ backend.StagingCompleter = (*kernelOp)(nil) // StatementID returns the server query id — captured at execute time on success, // or from the KernelError on an execute failure (empty only when the server @@ -384,6 +386,7 @@ func (o *kernelOp) Results(ctx context.Context, callbacks *dbsqlrows.TelemetryCa return newKernelRows(ctx, o, stream, callbacks) } +// IsStaging reports the server-authorized Volume marker. func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { if o.exec == nil { return false, nil @@ -395,10 +398,12 @@ func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { return bool(marked), err } +// CompleteStaging completes the marked transfer with the context roots. func (o *kernelOp) CompleteStaging(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } + // These local handle calls perform no I/O (databricks_kernel.h), so they cannot evict a session. var stream *C.kernel_result_stream_t if err := call(func() C.KernelStatusCode { return C.kernel_executed_statement_get_result_stream(o.exec, &stream) @@ -422,22 +427,24 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { case <-done: } }() + // Join the watcher before freeing its canceller and result stream. defer func() { close(done); <-stopped }() - encoded, err := json.Marshal(driverctx.StagingPathsFromContext(ctx)) - if err != nil { - return err + roots := driverctx.StagingPathsFromContext(ctx) + for _, root := range roots { + if !utf8.ValidString(root) { + return errors.New("kernel: staging root must be valid UTF-8") + } } + encoded, _ := json.Marshal(roots) paths := newCStr(string(encoded)) defer paths.free() var outcome C.KernelVolumeCompletion - err = call(func() C.KernelStatusCode { + err := call(func() C.KernelStatusCode { + // Go Thrift REMOVE treats a missing object as success. return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) }) if err != nil { - var ke *KernelError - if !errors.As(err, &ke) || (ke.ErrorCode != "VOLUME_STORAGE_HTTP_ERROR" && ke.ErrorCode != "VOLUME_STORAGE_NETWORK_ERROR") { - o.backend.evictIfSessionFatal(err) - } + o.backend.evictIfSessionFatal(err) if ctx.Err() != nil { return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) } diff --git a/staging_kernel_test.go b/staging_kernel_test.go index 50edd67a..3c27438f 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -22,6 +22,7 @@ import ( "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/databricks/databricks-sql-go/driverctx" + dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/databricks/databricks-sql-go/internal/backend" "github.com/databricks/databricks-sql-go/internal/backend/kernel" "github.com/stretchr/testify/require" @@ -35,7 +36,7 @@ type cancelOnCompletionErrorContext struct { func (c *cancelOnCompletionErrorContext) Err() error { c.errCalls++ - // Deliver cancellation after native completion returns to make the error race deterministic. + // CompleteStaging checks Err before completion and again after a native failure. if c.errCalls == 2 { c.cancel() } @@ -43,11 +44,26 @@ func (c *cancelOnCompletionErrorContext) Err() error { } func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { - for _, mode := range []string{"instruction401", "storage401", "storageBody", "instruction401Cancelled", "storage401Cancelled"} { - t.Run(mode, func(t *testing.T) { + for _, tc := range []struct { + name, operation, local, errorCode string + status int + instruction, cancelAfterError, inFlight, success, invalidRoot bool + }{ + {name: "instruction401", instruction: true, errorCode: "SESSION_EXPIRED"}, + {name: "storage401", status: 401, errorCode: "VOLUME_STORAGE_HTTP_ERROR"}, + {name: "storageBody", errorCode: "VOLUME_STORAGE_NETWORK_ERROR"}, + {name: "instruction401Cancelled", instruction: true, cancelAfterError: true, errorCode: "SESSION_EXPIRED"}, + {name: "getSuccess", success: true}, + {name: "removeNotFound", operation: "REMOVE", status: 404, success: true}, + {name: "inFlightCancel", inFlight: true}, + {name: "streamPut", operation: "PUT", local: "__input_stream__"}, + {name: "invalidUTF8Root", invalidRoot: true}, + } { + t.Run(tc.name, func(t *testing.T) { directory := t.TempDir() destination := filepath.Join(directory, "download.bin") var result map[string]any + storageStarted := make(chan struct{}) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) w.Header().Set("Content-Type", "application/json") @@ -60,8 +76,13 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { w.WriteHeader(401) _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) case r.URL.Path == "/object": - if strings.HasPrefix(mode, "storage401") { - w.WriteHeader(401) + if tc.inFlight { + close(storageStarted) + <-r.Context().Done() + } else if tc.status != 0 { + w.WriteHeader(tc.status) + } else if tc.success { + _, _ = io.WriteString(w, "downloaded") } else { w.Header().Set("Content-Length", "20") _, _ = io.WriteString(w, "short") @@ -72,7 +93,14 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { })) defer server.Close() names := []string{"operation", "presignedUrl", "headers", "localFile"} - values := []string{"GET", server.URL + "/object", "{}", destination} + operation, local := tc.operation, destination + if operation == "" { + operation = "GET" + } + if tc.local != "" { + local = tc.local + } + values := []string{operation, server.URL + "/object", "{}", local} fields, columns := make([]arrow.Field, 4), make([]arrow.Array, 4) manifestColumns := make([]map[string]any, 4) for i, name := range names { @@ -93,7 +121,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.NoError(t, writer.Close()) chunk := map[string]any{"chunk_index": 0, "row_offset": 0, "row_count": 1, "attachment": base64.StdEncoding.EncodeToString(attachment.Bytes())} chunks := 1 - if strings.HasPrefix(mode, "instruction401") { + if tc.instruction { chunk["next_chunk_index"], chunks = 1, 2 } result = map[string]any{ @@ -111,7 +139,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { connection, err := db.Conn(ctx) require.NoError(t, err) defer connection.Close() - if strings.HasSuffix(mode, "Cancelled") { + if tc.cancelAfterError { err = connection.Raw(func(raw any) error { operation, err := raw.(*conn).backend.Execute(ctx, backend.ExecRequest{Query: "GET instruction"}) if err != nil { @@ -121,26 +149,38 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { cancelContext, cancelError := context.WithCancel(ctx) defer cancelError() stagingContext := driverctx.NewContextWithStagingInfo(&cancelOnCompletionErrorContext{Context: cancelContext, cancel: cancelError}, []string{directory}) - return operation.(interface{ CompleteStaging(context.Context) error }).CompleteStaging(stagingContext) + return operation.(backend.StagingCompleter).CompleteStaging(stagingContext) }) require.ErrorIs(t, err, context.Canceled) } else { - _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") - } - require.Error(t, err) - require.NotErrorIs(t, err, driver.ErrBadConn) - var detail *kernel.KernelError - require.ErrorAs(t, err, &detail) - wantCode := "VOLUME_STORAGE_HTTP_ERROR" - if strings.HasPrefix(mode, "instruction401") { - wantCode = "SESSION_EXPIRED" + roots := []string{directory} + if tc.invalidRoot { + roots = []string{directory + string([]byte{0xff})} + } + if tc.inFlight { + go func() { <-storageStarted; cancel() }() + } + _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, roots), "Volume instruction") } - if mode == "storageBody" { - wantCode = "VOLUME_STORAGE_NETWORK_ERROR" + if tc.success { + require.NoError(t, err) + if tc.operation != "REMOVE" { + require.FileExists(t, destination) + } + } else if tc.errorCode == "" { + require.ErrorIs(t, err, dbsqlerr.DriverError) + if tc.inFlight { + require.ErrorIs(t, err, context.Canceled) + } + } else { + require.Error(t, err) + require.NotErrorIs(t, err, driver.ErrBadConn) + var detail *kernel.KernelError + require.ErrorAs(t, err, &detail) + require.Equal(t, tc.errorCode, detail.ErrorCode) } - require.Equal(t, wantCode, detail.ErrorCode) require.NoError(t, connection.Raw(func(raw any) error { - require.Equal(t, !strings.HasPrefix(mode, "instruction401"), raw.(driver.Validator).IsValid()) + require.Equal(t, !tc.instruction, raw.(driver.Validator).IsValid()) return nil })) }) From 72e981682a0301114ba857326d74535e5236d946 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 02:05:43 +0000 Subject: [PATCH 31/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- staging_kernel_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/staging_kernel_test.go b/staging_kernel_test.go index 3c27438f..9669dc7c 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -10,8 +10,10 @@ import ( "encoding/base64" "encoding/json" "io" + "io/fs" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -43,7 +45,7 @@ func (c *cancelOnCompletionErrorContext) Err() error { return c.Context.Err() } -func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { +func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { for _, tc := range []struct { name, operation, local, errorCode string status int @@ -165,7 +167,9 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { if tc.success { require.NoError(t, err) if tc.operation != "REMOVE" { - require.FileExists(t, destination) + content, readErr := fs.ReadFile(os.DirFS(directory), "download.bin") + require.NoError(t, readErr) + require.Equal(t, "downloaded", string(content)) } } else if tc.errorCode == "" { require.ErrorIs(t, err, dbsqlerr.DriverError) From deed11412ad0c2d7a93f32f69367d47ae94d2e9d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 03:51:56 +0000 Subject: [PATCH 32/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 +- KERNEL_REV | 2 +- .../kernel/include/databricks_kernel.h | 18 +++++++++---- staging_kernel_test.go | 26 +++++++++++++++++-- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d9e9a10..555e3ed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). - **Behavior change:** Support Arrow TIME results on Thrift and SEA as UTC `time.Time` values on `0001-01-01`, preserving precision and using clock-only nested JSON. TIME metadata now reports `TIME`/`time.Time` with no length, including empty and NULL-only results; previously Thrift reported `STRING`/`string` with unbounded length, and SEA reported an empty name with `*interface{}` (databricks/databricks-sql-go#505). - **Behavior change:** Explicit `WithMaxDownloadThreads` / `maxDownloadThreads` settings tuned for Thrift also control kernel buffering and can change memory use and throughput. The kernel window defaults to 16 when unset; explicit chunk-setting overrides take precedence. -- Support Volume PUT/GET/REMOVE through kernel ExecContext with staging roots, including resolved-symlink authorization and context cancellation. QueryContext retains raw instructions. +- **Behavior change:** Support Volume PUT/GET/REMOVE through kernel ExecContext with staging roots, including resolved-symlink authorization and context cancellation. QueryContext retains raw instructions (databricks/databricks-sql-go#560). - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/KERNEL_REV b/KERNEL_REV index 17fa057e..def3bb7c 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c838ba9a3fce6fc5d4178640df26b26c7e6cabb3 +4bb68f17eb2ab875db6629f7f84e7995a2cc4d63 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index b41b99f5..9ae08e46 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -375,11 +375,16 @@ KernelStatusCode kernel_session_config_set_sea_response_compression_enabled( /* Configure the total timeout for one HTTP request, from connection start * through response-body completion. A separate 30s connection timeout also * applies, so connection establishment is bounded by the shorter limit. - * Does not bound Volume storage requests or configure their inactivity limit. + * Volume storage inactivity uses the separate storage-read-timeout setter. * `request_timeout_ms == 0` keeps the kernel default (120s). */ KernelStatusCode kernel_session_config_set_request_timeout(KernelSessionConfig* config, uint64_t request_timeout_ms); +/* Configure Volume storage idle time for reads, upload progress and PUT response + * waits. Zero disables it. Redirected stream PUT resends have no idle bound. */ +KernelStatusCode kernel_session_config_set_storage_read_timeout(KernelSessionConfig* config, + uint64_t timeout_ms); + /* Configure the maximum number of idle HTTP connections retained per host. * `max_connections` must be greater than zero. Omitting this setter keeps the * kernel default (100). */ @@ -979,14 +984,17 @@ typedef enum KernelVolumeCompletion { KernelVolumeCompletion_Completed = 2 } KernelVolumeCompletion; -/* Volume completion preserves raw rows. Roots are a JSON array of strings. - * The detached result-stream canceller can interrupt completion. */ +/* out is written only on success. */ KernelStatusCode kernel_executed_statement_is_volume_operation( const kernel_executed_statement_t* exec, bool* out); -/* The cached marker does not establish whether the executed owner is valid. */ +/* The cached marker does not establish whether the executed owner is valid. + * out is written only on success. */ KernelStatusCode kernel_result_stream_is_volume_operation( const kernel_result_stream_t* stream, bool* out); -/* out_status is written only on success. */ +/* Complete a server-marked Volume operation while retaining its instruction rows. + * Roots are a JSON array of strings. + * The detached result-stream canceller can interrupt completion. + * out_status is written only on success. */ KernelStatusCode kernel_result_stream_complete_volume_operation( kernel_result_stream_t* stream, const char* allowed_paths_json, bool remove_not_found_is_success, KernelVolumeCompletion* out_status); diff --git a/staging_kernel_test.go b/staging_kernel_test.go index 9669dc7c..2c3786bd 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -140,7 +140,12 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { defer cancel() connection, err := db.Conn(ctx) require.NoError(t, err) - defer connection.Close() + defer func() { + cancel() + server.CloseClientConnections() + go connection.Close() + }() + var execResult sql.Result if tc.cancelAfterError { err = connection.Raw(func(raw any) error { operation, err := raw.(*conn).backend.Execute(ctx, backend.ExecRequest{Query: "GET instruction"}) @@ -162,11 +167,28 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { if tc.inFlight { go func() { <-storageStarted; cancel() }() } - _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, roots), "Volume instruction") + type executionResult struct { + result sql.Result + err error + } + done := make(chan executionResult, 1) + go func() { + result, err := connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, roots), "Volume instruction") + done <- executionResult{result, err} + }() + select { + case completed := <-done: + execResult, err = completed.result, completed.err + case <-time.After(10 * time.Second): + cancel() + t.Fatal("ExecContext did not return") + } } if tc.success { require.NoError(t, err) if tc.operation != "REMOVE" { + n, _ := execResult.RowsAffected() + require.Zero(t, n) content, readErr := fs.ReadFile(os.DirFS(directory), "download.bin") require.NoError(t, readErr) require.Equal(t, "downloaded", string(content)) From 1706da4566cbaee39f0cfdab67c34fc8b589de59 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 06:00:31 +0000 Subject: [PATCH 33/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/errors_classify_test.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index def3bb7c..1a5d3482 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -4bb68f17eb2ab875db6629f7f84e7995a2cc4d63 +9ed6b728dc609f0c96ab09f21cf90a27dd932553 diff --git a/internal/backend/kernel/errors_classify_test.go b/internal/backend/kernel/errors_classify_test.go index e2c38dd8..1b054e0b 100644 --- a/internal/backend/kernel/errors_classify_test.go +++ b/internal/backend/kernel/errors_classify_test.go @@ -49,6 +49,15 @@ func TestIsSessionFatal(t *testing.T) { t.Errorf("code %d should not be session-fatal", code) } } + for _, err := range []*KernelError{ + {Code: statusUnauthenticated, ErrorCode: "VOLUME_STORAGE_HTTP_ERROR"}, + {Code: statusNetworkError, ErrorCode: "VOLUME_STORAGE_NETWORK_ERROR"}, + } { + if isSessionFatal(err) { + t.Errorf("storage error %s should not be session-fatal", err.ErrorCode) + } + } + } // isUserFault picks the log level for a kernel-call failure: user/query faults From af3a000e8f17cba84869305b74af0d3289b8f4ad Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 08:47:59 +0000 Subject: [PATCH 34/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/errors_classify_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 1a5d3482..6af4073b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -9ed6b728dc609f0c96ab09f21cf90a27dd932553 +c2ddddc61bd41d9529dd19f36546d603ba21150d diff --git a/internal/backend/kernel/errors_classify_test.go b/internal/backend/kernel/errors_classify_test.go index 1b054e0b..ef40c416 100644 --- a/internal/backend/kernel/errors_classify_test.go +++ b/internal/backend/kernel/errors_classify_test.go @@ -39,7 +39,7 @@ func TestIsBadConnection(t *testing.T) { func TestIsSessionFatal(t *testing.T) { fatal := []int{statusUnauthenticated, statusUnavailable, statusNetworkError} for _, code := range fatal { - if !isSessionFatal(&KernelError{Code: code}) { + if !isSessionFatal(fmt.Errorf("call: %w", &KernelError{Code: code})) { t.Errorf("code %d should be session-fatal", code) } } @@ -57,7 +57,6 @@ func TestIsSessionFatal(t *testing.T) { t.Errorf("storage error %s should not be session-fatal", err.ErrorCode) } } - } // isUserFault picks the log level for a kernel-call failure: user/query faults From b2961001497ce14043bf9f90faaffc0f532846cd Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 20:01:42 +0000 Subject: [PATCH 35/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/backend.go | 3 ++- .../kernel/include/databricks_kernel.h | 24 ++++++++++--------- internal/backend/kernel/operation.go | 6 ++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 6af4073b..6697ab01 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c2ddddc61bd41d9529dd19f36546d603ba21150d +b633ebb59b1d7cbbd1b05925adf5baebf7777213 diff --git a/internal/backend/backend.go b/internal/backend/backend.go index 0ca41bf5..c679a866 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -114,7 +114,8 @@ type ResultFormatReporter interface { ResultFormat() string } -// StagingCompleter completes server-marked staging operations in the backend. +// StagingCompleter is implemented by operations that run the staging (Volume) +// file transfer themselves. type StagingCompleter interface { CompleteStaging(context.Context) error } diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 9ae08e46..4cf94ff9 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -56,7 +56,7 @@ * - **A result stream borrows its executed handle.** Closing the executed * handle invalidates any stream obtained from it: schema access, fetching * and Volume completion return `InvalidStatementHandle` (a defined error, - * not UB). The cached Volume marker remains readable until stream close. + * not UB). * - **Close is awaited.** `kernel_session_close` waits for the server-side * delete to complete and flushes close-session telemetry before returning. * If close fails, the kernel still schedules best-effort cleanup when the @@ -375,13 +375,13 @@ KernelStatusCode kernel_session_config_set_sea_response_compression_enabled( /* Configure the total timeout for one HTTP request, from connection start * through response-body completion. A separate 30s connection timeout also * applies, so connection establishment is bounded by the shorter limit. - * Volume storage inactivity uses the separate storage-read-timeout setter. + * Does not apply to Volume file transfers; see the storage read timeout. * `request_timeout_ms == 0` keeps the kernel default (120s). */ KernelStatusCode kernel_session_config_set_request_timeout(KernelSessionConfig* config, uint64_t request_timeout_ms); -/* Configure Volume storage idle time for reads, upload progress and PUT response - * waits. Zero disables it. Redirected stream PUT resends have no idle bound. */ +/* Idle limit for Volume file transfers: a transfer fails after this long without + * progress. Also caps connection setup. Zero sets no limit. */ KernelStatusCode kernel_session_config_set_storage_read_timeout(KernelSessionConfig* config, uint64_t timeout_ms); @@ -984,17 +984,19 @@ typedef enum KernelVolumeCompletion { KernelVolumeCompletion_Completed = 2 } KernelVolumeCompletion; -/* out is written only on success. */ +/* Whether the statement is a Volume PUT, GET or REMOVE. */ KernelStatusCode kernel_executed_statement_is_volume_operation( const kernel_executed_statement_t* exec, bool* out); -/* The cached marker does not establish whether the executed owner is valid. - * out is written only on success. */ +/* Same flag, read from the stream. Still succeeds after the executed statement + * is closed. */ KernelStatusCode kernel_result_stream_is_volume_operation( const kernel_result_stream_t* stream, bool* out); -/* Complete a server-marked Volume operation while retaining its instruction rows. - * Roots are a JSON array of strings. - * The detached result-stream canceller can interrupt completion. - * out_status is written only on success. */ +/* Runs the file transfer for a Volume PUT, GET or REMOVE result. Call before + * fetching; the rows stay readable afterwards. + * allowed_paths_json: JSON array of local directories the transfer may use. + * remove_not_found_is_success: report success when the remote file is missing. + * Returns NeedsInputStream for a stream upload, which this API cannot perform. + * kernel_result_stream_canceller_cancel interrupts it from another thread. */ KernelStatusCode kernel_result_stream_complete_volume_operation( kernel_result_stream_t* stream, const char* allowed_paths_json, bool remove_not_found_is_success, KernelVolumeCompletion* out_status); diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index a7f00188..34b4df32 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -386,7 +386,7 @@ func (o *kernelOp) Results(ctx context.Context, callbacks *dbsqlrows.TelemetryCa return newKernelRows(ctx, o, stream, callbacks) } -// IsStaging reports the server-authorized Volume marker. +// IsStaging reports whether the server marked the statement as a Volume PUT, GET or REMOVE. func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { if o.exec == nil { return false, nil @@ -398,12 +398,12 @@ func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { return bool(marked), err } -// CompleteStaging completes the marked transfer with the context roots. +// CompleteStaging runs the Volume file transfer, limited to the staging paths set on ctx. func (o *kernelOp) CompleteStaging(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - // These local handle calls perform no I/O (databricks_kernel.h), so they cannot evict a session. + // These two calls do no network I/O, so their errors never mean the session is dead. var stream *C.kernel_result_stream_t if err := call(func() C.KernelStatusCode { return C.kernel_executed_statement_get_result_stream(o.exec, &stream) From 82bf2e4ce13b9cf4a7eb8905bf43dfcf67200dd0 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 20 Nov 2026 21:24:38 +0000 Subject: [PATCH 36/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/include/databricks_kernel.h | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 6697ab01..b6153529 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -b633ebb59b1d7cbbd1b05925adf5baebf7777213 +ac32e64ee24b6fac30ba230a1ed32bbf3260e4b9 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 4cf94ff9..f1d27a86 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -984,11 +984,13 @@ typedef enum KernelVolumeCompletion { KernelVolumeCompletion_Completed = 2 } KernelVolumeCompletion; -/* Whether the statement is a Volume PUT, GET or REMOVE. */ +/* Whether the statement is a Volume PUT, GET or REMOVE. + * out is written only on success. */ KernelStatusCode kernel_executed_statement_is_volume_operation( const kernel_executed_statement_t* exec, bool* out); /* Same flag, read from the stream. Still succeeds after the executed statement - * is closed. */ + * is closed. + * out is written only on success. */ KernelStatusCode kernel_result_stream_is_volume_operation( const kernel_result_stream_t* stream, bool* out); /* Runs the file transfer for a Volume PUT, GET or REMOVE result. Call before @@ -996,7 +998,8 @@ KernelStatusCode kernel_result_stream_is_volume_operation( * allowed_paths_json: JSON array of local directories the transfer may use. * remove_not_found_is_success: report success when the remote file is missing. * Returns NeedsInputStream for a stream upload, which this API cannot perform. - * kernel_result_stream_canceller_cancel interrupts it from another thread. */ + * kernel_result_stream_canceller_cancel interrupts it from another thread. + * out_status is written only on success. */ KernelStatusCode kernel_result_stream_complete_volume_operation( kernel_result_stream_t* stream, const char* allowed_paths_json, bool remove_not_found_is_success, KernelVolumeCompletion* out_status); From c0c9301de9e43083f8ac1fecf8b444e7a8a53710 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 21 Nov 2026 01:08:54 +0000 Subject: [PATCH 37/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/cgo.go | 10 +++++----- staging_kernel_test.go | 31 +++++++++++++++++++++++++------ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index b6153529..8e16c26d 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ac32e64ee24b6fac30ba230a1ed32bbf3260e4b9 +254245b3b4fcc770f543b02e299b114666ac26a2 diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index 56985856..b0e1eac4 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -173,11 +173,6 @@ func lastError(code C.KernelStatusCode) *KernelError { VendorCode: int32(e.vendor_code), HTTPStatus: uint16(e.http_status), Retryable: bool(e.retryable), - ErrorCode: C.GoString(C.kernel_get_last_error_code()), - } - if code := C.kernel_get_last_error_local_os_error(); code != 0 { - // The C ABI reports a path and errno without a syscall name. - ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} } if e.sql_state != nil { ke.SQLState = C.GoString(e.sql_state) @@ -185,6 +180,11 @@ func lastError(code C.KernelStatusCode) *KernelError { if e.query_id != nil { ke.QueryID = C.GoString(e.query_id) } + ke.ErrorCode = C.GoString(C.kernel_get_last_error_code()) + if code := C.kernel_get_last_error_local_os_error(); code != 0 { + // The C ABI reports a path and errno without a syscall name. + ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} + } klog("kernel error: code=%d sqlstate=%q vendor=%d http=%d retryable=%v msg=%q", ke.Code, ke.SQLState, ke.VendorCode, ke.HTTPStatus, ke.Retryable, ke.Message) // Also emit through the driver's logger — no SQL text or PII, just the diff --git a/staging_kernel_test.go b/staging_kernel_test.go index 2c3786bd..372cc51c 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -47,9 +47,9 @@ func (c *cancelOnCompletionErrorContext) Err() error { func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { for _, tc := range []struct { - name, operation, local, errorCode string - status int - instruction, cancelAfterError, inFlight, success, invalidRoot bool + name, operation, local, errorCode, message string + status int + instruction, cancelAfterError, inFlight, success, invalidRoot, missingLocal bool }{ {name: "instruction401", instruction: true, errorCode: "SESSION_EXPIRED"}, {name: "storage401", status: 401, errorCode: "VOLUME_STORAGE_HTTP_ERROR"}, @@ -58,8 +58,9 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { {name: "getSuccess", success: true}, {name: "removeNotFound", operation: "REMOVE", status: 404, success: true}, {name: "inFlightCancel", inFlight: true}, - {name: "streamPut", operation: "PUT", local: "__input_stream__"}, - {name: "invalidUTF8Root", invalidRoot: true}, + {name: "streamPut", operation: "PUT", local: "__input_stream__", message: "requires a local file instruction"}, + {name: "invalidUTF8Root", invalidRoot: true, message: "valid UTF-8"}, + {name: "putMissingFile", operation: "PUT", missingLocal: true, errorCode: "VOLUME_LOCAL_FILE_ERROR"}, } { t.Run(tc.name, func(t *testing.T) { directory := t.TempDir() @@ -102,6 +103,9 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { if tc.local != "" { local = tc.local } + if tc.missingLocal { + local = filepath.Join(directory, "missing.bin") + } values := []string{operation, server.URL + "/object", "{}", local} fields, columns := make([]arrow.Field, 4), make([]arrow.Array, 4) manifestColumns := make([]map[string]any, 4) @@ -143,7 +147,16 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { defer func() { cancel() server.CloseClientConnections() - go connection.Close() + closed := make(chan struct{}) + go func() { + defer close(closed) + _ = connection.Close() + }() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Error("connection.Close did not return") + } }() var execResult sql.Result if tc.cancelAfterError { @@ -195,6 +208,9 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { } } else if tc.errorCode == "" { require.ErrorIs(t, err, dbsqlerr.DriverError) + if tc.message != "" { + require.ErrorContains(t, err, tc.message) + } if tc.inFlight { require.ErrorIs(t, err, context.Canceled) } @@ -204,6 +220,9 @@ func TestKernelVolumeCompletionControlsSessionValidity(t *testing.T) { var detail *kernel.KernelError require.ErrorAs(t, err, &detail) require.Equal(t, tc.errorCode, detail.ErrorCode) + if tc.missingLocal { + require.ErrorIs(t, err, fs.ErrNotExist) + } } require.NoError(t, connection.Raw(func(raw any) error { require.Equal(t, !tc.instruction, raw.(driver.Validator).IsValid()) From ef61222d3f965e1b9077a9c0edbcd3e9e5ad12ce Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 22 Nov 2026 07:12:10 +0000 Subject: [PATCH 38/38] fixup! fix(volume): enforce native staging dispatch and error policy Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8e16c26d..39d79c6f 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -254245b3b4fcc770f543b02e299b114666ac26a2 +23e45bafa9b0d1688fbffa5ecc975e1d91f40f01