From 842044168de07f74fc9739fa7005afc3c81dfee2 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 13 Nov 2026 16:05:57 +0000 Subject: [PATCH 1/3] fix(auth): bound the OAuth M2M token request The M2M authenticator built its token source on context.Background(), so oauth2 sent token requests through http.DefaultClient, which has no timeout. A token endpoint that accepted the connection but never answered blocked Authenticate, and every request queued behind its mutex, forever. Give the token exchange a copy of the context's oauth2 HTTP client (or http.DefaultClient) with its Timeout capped at 60s, the SQL kernel's default HTTP timeout. The token source keeps that context, so refreshes are bounded too. A client supplied on the context keeps its transport and any shorter timeout. The exported GetTokenSource is unchanged. Addresses #475 Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + auth/oauth/m2m/m2m.go | 31 ++++++- auth/oauth/m2m/m2m_test.go | 162 +++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f93aa25..fc7180c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (v2.0.0 prototype) +- Bound each OAuth M2M token request to 60 seconds, so a hung token endpoint no longer blocks every request on the connector indefinitely (databricks/databricks-sql-go#475). - Render non-finite FLOAT / DOUBLE values inside ARRAY / MAP / STRUCT results as the JSON strings `"NaN"`, `"Infinity"`, and `"-Infinity"` on both backends instead of failing row iteration. SQL NULL remains `null`; top-level floats remain numeric (databricks/databricks-sql-go#573). - **Behavior change:** Non-finite floating-point MAP keys use the same spellings as values on both backends. On the kernel backend, keys previously rendered as `"+Inf"` and `"-Inf"` now render as `"Infinity"` and `"-Infinity"`; `"NaN"` is unchanged. On Thrift, these keys previously failed row iteration (databricks/databricks-sql-go#573). - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). diff --git a/auth/oauth/m2m/m2m.go b/auth/oauth/m2m/m2m.go index 76b5e513..61ee4dc1 100644 --- a/auth/oauth/m2m/m2m.go +++ b/auth/oauth/m2m/m2m.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "sync" + "time" "github.com/databricks/databricks-sql-go/auth" "github.com/databricks/databricks-sql-go/auth/oauth" @@ -66,12 +67,14 @@ func (c *authClient) Authenticate(r *http.Request) error { return nil } - config, err := GetConfig(context.Background(), c.hostName, c.clientID, c.clientSecret, c.scopes) + ctx := withTokenRequestTimeout(context.Background()) + config, err := getConfig(ctx, c.hostName, c.clientID, c.clientSecret, c.scopes) if err != nil { return fmt.Errorf("unable to generate clientCredentials.Config: %w", err) } - c.tokenSource = GetTokenSource(config) + // The token source keeps ctx, so refreshes are bounded too. + c.tokenSource = config.TokenSource(ctx) token, err := c.tokenSource.Token() if err != nil { logger.Err(err).Msg("failed to get token") @@ -87,6 +90,30 @@ func (c *authClient) Authenticate(r *http.Request) error { } +var ( + // tokenRequestTimeout bounds each token request, matching the SQL kernel's + // default HTTP timeout. Until a fetch succeeds, oauth2's AuthStyle auto-detect + // retries each failed request once, so such a fetch can take up to twice this. + tokenRequestTimeout = 60 * time.Second + + // getConfig is replaced in tests to skip endpoint discovery. + getConfig = GetConfig +) + +// withTokenRequestTimeout returns ctx carrying a copy of its oauth2.HTTPClient +// (http.DefaultClient when unset) whose Timeout is at most tokenRequestTimeout. +func withTokenRequestTimeout(ctx context.Context) context.Context { + base := http.DefaultClient + if hc, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok && hc != nil { + base = hc + } + client := *base + if client.Timeout <= 0 || client.Timeout > tokenRequestTimeout { + client.Timeout = tokenRequestTimeout + } + return context.WithValue(ctx, oauth2.HTTPClient, &client) +} + func GetTokenSource(config clientcredentials.Config) oauth2.TokenSource { tokenSource := config.TokenSource(context.Background()) return tokenSource diff --git a/auth/oauth/m2m/m2m_test.go b/auth/oauth/m2m/m2m_test.go index 1ff8ce97..0a31c409 100644 --- a/auth/oauth/m2m/m2m_test.go +++ b/auth/oauth/m2m/m2m_test.go @@ -1,9 +1,20 @@ package m2m import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" + "time" + "github.com/databricks/databricks-sql-go/auth" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" ) func TestM2MScopes(t *testing.T) { @@ -38,3 +49,154 @@ func TestM2MScopes(t *testing.T) { assert.Equal(t, []string{"all-apis", "my-scope"}, auth.scopes) }) } + +// tokenServer answers the first `answered` requests with a short-lived token and +// holds every later request open until the test ends. +func tokenServer(t *testing.T, answered int32) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits.Add(1) <= answered { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"access_token":"tok","token_type":"Bearer","expires_in":1}`) + return + } + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { + close(release) + srv.Close() + }) + return srv, &hits +} + +// useTokenEndpoint skips endpoint discovery, sends token requests to tokenURL, +// and shortens the request bound for the test. +func useTokenEndpoint(t *testing.T, tokenURL string, bound time.Duration) { + t.Helper() + oldConfig, oldBound := getConfig, tokenRequestTimeout + getConfig = func(_ context.Context, _, clientID, clientSecret string, scopes []string) (clientcredentials.Config, error) { + return clientcredentials.Config{ClientID: clientID, ClientSecret: clientSecret, TokenURL: tokenURL, Scopes: scopes}, nil + } + tokenRequestTimeout = bound + t.Cleanup(func() { getConfig, tokenRequestTimeout = oldConfig, oldBound }) +} + +func newRequest(t *testing.T) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, "https://ws.cloud.databricks.com/sql/1.0/warehouses/x", nil) + require.NoError(t, err) + return req +} + +// authenticate fails the test if Authenticate has not returned within limit. +func authenticate(t *testing.T, authr auth.Authenticator, limit time.Duration) (*http.Request, error) { + t.Helper() + req := newRequest(t) + done := make(chan error, 1) + go func() { done <- authr.Authenticate(req) }() + select { + case err := <-done: + return req, err + case <-time.After(limit): + t.Fatalf("Authenticate did not return within %v", limit) + return nil, nil + } +} + +func requireTimeout(t *testing.T, err error) { + t.Helper() + var netErr net.Error + require.ErrorAs(t, err, &netErr) + require.True(t, netErr.Timeout(), "want a timeout error, got %v", err) +} + +func TestAuthenticateBoundsHungTokenEndpoint(t *testing.T) { + const bound = 200 * time.Millisecond + srv, hits := tokenServer(t, 0) + useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound) + authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com") + + start := time.Now() + _, err := authenticate(t, authr, 10*time.Second) + elapsed := time.Since(start) + + requireTimeout(t, err) + // oauth2's AuthStyle auto-detect retries a failed first request once. + assert.LessOrEqual(t, hits.Load(), int32(2)) + assert.GreaterOrEqual(t, elapsed, bound) + assert.Less(t, elapsed, 2*bound+time.Second) +} + +func TestAuthenticateBoundsConcurrentCallers(t *testing.T) { + const bound = 200 * time.Millisecond + srv, _ := tokenServer(t, 0) + useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound) + authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com") + + start := time.Now() + errs := make(chan error, 2) + for range 2 { + req := newRequest(t) + go func() { errs <- authr.Authenticate(req) }() + } + for range 2 { + select { + case err := <-errs: + requireTimeout(t, err) + case <-time.After(10 * time.Second): + t.Fatal("Authenticate did not return within 10s") + } + } + // The second caller waits for the first fetch, then makes its own. + assert.Less(t, time.Since(start), 4*bound+2*time.Second) +} + +func TestAuthenticateBoundsTokenRefresh(t *testing.T) { + const bound = 200 * time.Millisecond + srv, hits := tokenServer(t, 1) + useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound) + authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com") + + req, err := authenticate(t, authr, 10*time.Second) + require.NoError(t, err) + assert.Equal(t, "Bearer tok", req.Header.Get("Authorization")) + + // The token expires inside oauth2's 10s early-expiry window, so this refreshes. + start := time.Now() + _, err = authenticate(t, authr, 10*time.Second) + requireTimeout(t, err) + assert.Less(t, time.Since(start), bound+time.Second) + assert.Equal(t, int32(2), hits.Load(), "a refresh reuses the detected auth style") +} + +func TestWithTokenRequestTimeout(t *testing.T) { + clientOn := func(ctx context.Context) *http.Client { + return ctx.Value(oauth2.HTTPClient).(*http.Client) + } + + t.Run("default client gets the bound", func(t *testing.T) { + c := clientOn(withTokenRequestTimeout(context.Background())) + assert.Equal(t, tokenRequestTimeout, c.Timeout) + assert.Nil(t, c.Transport) + assert.Zero(t, http.DefaultClient.Timeout, "http.DefaultClient must not be mutated") + }) + + t.Run("supplied client keeps its transport and is capped", func(t *testing.T) { + base := &http.Client{Transport: &http.Transport{}, Timeout: 15 * time.Minute} + c := clientOn(withTokenRequestTimeout(context.WithValue(context.Background(), oauth2.HTTPClient, base))) + assert.Equal(t, tokenRequestTimeout, c.Timeout) + assert.Same(t, base.Transport, c.Transport) + assert.Equal(t, 15*time.Minute, base.Timeout, "supplied client must not be mutated") + }) + + t.Run("shorter supplied timeout is kept", func(t *testing.T) { + base := &http.Client{Timeout: 5 * time.Second} + c := clientOn(withTokenRequestTimeout(context.WithValue(context.Background(), oauth2.HTTPClient, base))) + assert.Equal(t, 5*time.Second, c.Timeout) + }) +} From 94e7f2e987affe900085ebd670e5f13f683fd19d Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 01:31:45 +0000 Subject: [PATCH 2/3] docs(auth): clarify M2M timeout and retry bounds Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7180c8..eb574e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased (v2.0.0 prototype) -- Bound each OAuth M2M token request to 60 seconds, so a hung token endpoint no longer blocks every request on the connector indefinitely (databricks/databricks-sql-go#475). +- **Behavior change:** OAuth M2M token requests use a snapshot of the HTTP client capped at 60 seconds, including longer `http.DefaultClient.Timeout` values. Until the first success, auth-style detection can make two requests per fetch; Thrift retries (`WithRetries`) repeat the fetch, so a single retryable RPC can take about 615 seconds with defaults, excluding discovery and time queued behind other callers (databricks/databricks-sql-go#475). - Render non-finite FLOAT / DOUBLE values inside ARRAY / MAP / STRUCT results as the JSON strings `"NaN"`, `"Infinity"`, and `"-Infinity"` on both backends instead of failing row iteration. SQL NULL remains `null`; top-level floats remain numeric (databricks/databricks-sql-go#573). - **Behavior change:** Non-finite floating-point MAP keys use the same spellings as values on both backends. On the kernel backend, keys previously rendered as `"+Inf"` and `"-Inf"` now render as `"Infinity"` and `"-Infinity"`; `"NaN"` is unchanged. On Thrift, these keys previously failed row iteration (databricks/databricks-sql-go#573). - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). From e533c15fdb73aa7a19d6fcb66f1dd14ea5546ae9 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 20 Nov 2026 04:06:47 +0000 Subject: [PATCH 3/3] docs: lead the M2M timeout changelog entry with the fix Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb574e1a..d5024d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased (v2.0.0 prototype) -- **Behavior change:** OAuth M2M token requests use a snapshot of the HTTP client capped at 60 seconds, including longer `http.DefaultClient.Timeout` values. Until the first success, auth-style detection can make two requests per fetch; Thrift retries (`WithRetries`) repeat the fetch, so a single retryable RPC can take about 615 seconds with defaults, excluding discovery and time queued behind other callers (databricks/databricks-sql-go#475). +- Fix OAuth M2M token requests hanging indefinitely when the token endpoint stops responding. Each request now uses a copy of `http.DefaultClient` with a timeout of at most 60s; `WithRetries` repeats failed fetches (databricks/databricks-sql-go#475). - Render non-finite FLOAT / DOUBLE values inside ARRAY / MAP / STRUCT results as the JSON strings `"NaN"`, `"Infinity"`, and `"-Infinity"` on both backends instead of failing row iteration. SQL NULL remains `null`; top-level floats remain numeric (databricks/databricks-sql-go#573). - **Behavior change:** Non-finite floating-point MAP keys use the same spellings as values on both backends. On the kernel backend, keys previously rendered as `"+Inf"` and `"-Inf"` now render as `"Infinity"` and `"-Infinity"`; `"NaN"` is unchanged. On Thrift, these keys previously failed row iteration (databricks/databricks-sql-go#573). - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162).