Skip to content

Commit 8420441

Browse files
vuanhphungIsaac
andcommitted
fix(auth): bound the OAuth M2M token request
The M2M authenticator built its token source on context.Background(), so oauth2 sent token requests through http.DefaultClient, which has no timeout. A token endpoint that accepted the connection but never answered blocked Authenticate, and every request queued behind its mutex, forever. Give the token exchange a copy of the context's oauth2 HTTP client (or http.DefaultClient) with its Timeout capped at 60s, the SQL kernel's default HTTP timeout. The token source keeps that context, so refreshes are bounded too. A client supplied on the context keeps its transport and any shorter timeout. The exported GetTokenSource is unchanged. Addresses #475 Signed-off-by: Vu Anh Phung <vu.phung@databricks.com> Co-authored-by: Isaac <no-reply@databricks.com>
1 parent d034123 commit 8420441

3 files changed

Lines changed: 192 additions & 2 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
## Unreleased (v2.0.0 prototype)
44

5+
- Bound each OAuth M2M token request to 60 seconds, so a hung token endpoint no longer blocks every request on the connector indefinitely (databricks/databricks-sql-go#475).
56
- Render non-finite FLOAT / DOUBLE values inside ARRAY / MAP / STRUCT results as the JSON strings `"NaN"`, `"Infinity"`, and `"-Infinity"` on both backends instead of failing row iteration. SQL NULL remains `null`; top-level floats remain numeric (databricks/databricks-sql-go#573).
67
- **Behavior change:** Non-finite floating-point MAP keys use the same spellings as values on both backends. On the kernel backend, keys previously rendered as `"+Inf"` and `"-Inf"` now render as `"Infinity"` and `"-Infinity"`; `"NaN"` is unchanged. On Thrift, these keys previously failed row iteration (databricks/databricks-sql-go#573).
78
- Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162).

‎auth/oauth/m2m/m2m.go‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"net/http"
77
"sync"
8+
"time"
89

910
"github.com/databricks/databricks-sql-go/auth"
1011
"github.com/databricks/databricks-sql-go/auth/oauth"
@@ -66,12 +67,14 @@ func (c *authClient) Authenticate(r *http.Request) error {
6667
return nil
6768
}
6869

69-
config, err := GetConfig(context.Background(), c.hostName, c.clientID, c.clientSecret, c.scopes)
70+
ctx := withTokenRequestTimeout(context.Background())
71+
config, err := getConfig(ctx, c.hostName, c.clientID, c.clientSecret, c.scopes)
7072
if err != nil {
7173
return fmt.Errorf("unable to generate clientCredentials.Config: %w", err)
7274
}
7375

74-
c.tokenSource = GetTokenSource(config)
76+
// The token source keeps ctx, so refreshes are bounded too.
77+
c.tokenSource = config.TokenSource(ctx)
7578
token, err := c.tokenSource.Token()
7679
if err != nil {
7780
logger.Err(err).Msg("failed to get token")
@@ -87,6 +90,30 @@ func (c *authClient) Authenticate(r *http.Request) error {
8790

8891
}
8992

93+
var (
94+
// tokenRequestTimeout bounds each token request, matching the SQL kernel's
95+
// default HTTP timeout. Until a fetch succeeds, oauth2's AuthStyle auto-detect
96+
// retries each failed request once, so such a fetch can take up to twice this.
97+
tokenRequestTimeout = 60 * time.Second
98+
99+
// getConfig is replaced in tests to skip endpoint discovery.
100+
getConfig = GetConfig
101+
)
102+
103+
// withTokenRequestTimeout returns ctx carrying a copy of its oauth2.HTTPClient
104+
// (http.DefaultClient when unset) whose Timeout is at most tokenRequestTimeout.
105+
func withTokenRequestTimeout(ctx context.Context) context.Context {
106+
base := http.DefaultClient
107+
if hc, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok && hc != nil {
108+
base = hc
109+
}
110+
client := *base
111+
if client.Timeout <= 0 || client.Timeout > tokenRequestTimeout {
112+
client.Timeout = tokenRequestTimeout
113+
}
114+
return context.WithValue(ctx, oauth2.HTTPClient, &client)
115+
}
116+
90117
func GetTokenSource(config clientcredentials.Config) oauth2.TokenSource {
91118
tokenSource := config.TokenSource(context.Background())
92119
return tokenSource

‎auth/oauth/m2m/m2m_test.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,20 @@
11
package m2m
22

33
import (
4+
"context"
5+
"io"
6+
"net"
7+
"net/http"
8+
"net/http/httptest"
9+
"sync/atomic"
410
"testing"
11+
"time"
512

13+
"github.com/databricks/databricks-sql-go/auth"
614
"github.com/stretchr/testify/assert"
15+
"github.com/stretchr/testify/require"
16+
"golang.org/x/oauth2"
17+
"golang.org/x/oauth2/clientcredentials"
718
)
819

920
func TestM2MScopes(t *testing.T) {
@@ -38,3 +49,154 @@ func TestM2MScopes(t *testing.T) {
3849
assert.Equal(t, []string{"all-apis", "my-scope"}, auth.scopes)
3950
})
4051
}
52+
53+
// tokenServer answers the first `answered` requests with a short-lived token and
54+
// holds every later request open until the test ends.
55+
func tokenServer(t *testing.T, answered int32) (*httptest.Server, *atomic.Int32) {
56+
t.Helper()
57+
var hits atomic.Int32
58+
release := make(chan struct{})
59+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
60+
if hits.Add(1) <= answered {
61+
w.Header().Set("Content-Type", "application/json")
62+
_, _ = io.WriteString(w, `{"access_token":"tok","token_type":"Bearer","expires_in":1}`)
63+
return
64+
}
65+
select {
66+
case <-release:
67+
case <-r.Context().Done():
68+
}
69+
}))
70+
t.Cleanup(func() {
71+
close(release)
72+
srv.Close()
73+
})
74+
return srv, &hits
75+
}
76+
77+
// useTokenEndpoint skips endpoint discovery, sends token requests to tokenURL,
78+
// and shortens the request bound for the test.
79+
func useTokenEndpoint(t *testing.T, tokenURL string, bound time.Duration) {
80+
t.Helper()
81+
oldConfig, oldBound := getConfig, tokenRequestTimeout
82+
getConfig = func(_ context.Context, _, clientID, clientSecret string, scopes []string) (clientcredentials.Config, error) {
83+
return clientcredentials.Config{ClientID: clientID, ClientSecret: clientSecret, TokenURL: tokenURL, Scopes: scopes}, nil
84+
}
85+
tokenRequestTimeout = bound
86+
t.Cleanup(func() { getConfig, tokenRequestTimeout = oldConfig, oldBound })
87+
}
88+
89+
func newRequest(t *testing.T) *http.Request {
90+
t.Helper()
91+
req, err := http.NewRequest(http.MethodPost, "https://ws.cloud.databricks.com/sql/1.0/warehouses/x", nil)
92+
require.NoError(t, err)
93+
return req
94+
}
95+
96+
// authenticate fails the test if Authenticate has not returned within limit.
97+
func authenticate(t *testing.T, authr auth.Authenticator, limit time.Duration) (*http.Request, error) {
98+
t.Helper()
99+
req := newRequest(t)
100+
done := make(chan error, 1)
101+
go func() { done <- authr.Authenticate(req) }()
102+
select {
103+
case err := <-done:
104+
return req, err
105+
case <-time.After(limit):
106+
t.Fatalf("Authenticate did not return within %v", limit)
107+
return nil, nil
108+
}
109+
}
110+
111+
func requireTimeout(t *testing.T, err error) {
112+
t.Helper()
113+
var netErr net.Error
114+
require.ErrorAs(t, err, &netErr)
115+
require.True(t, netErr.Timeout(), "want a timeout error, got %v", err)
116+
}
117+
118+
func TestAuthenticateBoundsHungTokenEndpoint(t *testing.T) {
119+
const bound = 200 * time.Millisecond
120+
srv, hits := tokenServer(t, 0)
121+
useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound)
122+
authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com")
123+
124+
start := time.Now()
125+
_, err := authenticate(t, authr, 10*time.Second)
126+
elapsed := time.Since(start)
127+
128+
requireTimeout(t, err)
129+
// oauth2's AuthStyle auto-detect retries a failed first request once.
130+
assert.LessOrEqual(t, hits.Load(), int32(2))
131+
assert.GreaterOrEqual(t, elapsed, bound)
132+
assert.Less(t, elapsed, 2*bound+time.Second)
133+
}
134+
135+
func TestAuthenticateBoundsConcurrentCallers(t *testing.T) {
136+
const bound = 200 * time.Millisecond
137+
srv, _ := tokenServer(t, 0)
138+
useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound)
139+
authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com")
140+
141+
start := time.Now()
142+
errs := make(chan error, 2)
143+
for range 2 {
144+
req := newRequest(t)
145+
go func() { errs <- authr.Authenticate(req) }()
146+
}
147+
for range 2 {
148+
select {
149+
case err := <-errs:
150+
requireTimeout(t, err)
151+
case <-time.After(10 * time.Second):
152+
t.Fatal("Authenticate did not return within 10s")
153+
}
154+
}
155+
// The second caller waits for the first fetch, then makes its own.
156+
assert.Less(t, time.Since(start), 4*bound+2*time.Second)
157+
}
158+
159+
func TestAuthenticateBoundsTokenRefresh(t *testing.T) {
160+
const bound = 200 * time.Millisecond
161+
srv, hits := tokenServer(t, 1)
162+
useTokenEndpoint(t, srv.URL+"/oidc/v1/token", bound)
163+
authr := NewAuthenticator("id", "secret", "ws.cloud.databricks.com")
164+
165+
req, err := authenticate(t, authr, 10*time.Second)
166+
require.NoError(t, err)
167+
assert.Equal(t, "Bearer tok", req.Header.Get("Authorization"))
168+
169+
// The token expires inside oauth2's 10s early-expiry window, so this refreshes.
170+
start := time.Now()
171+
_, err = authenticate(t, authr, 10*time.Second)
172+
requireTimeout(t, err)
173+
assert.Less(t, time.Since(start), bound+time.Second)
174+
assert.Equal(t, int32(2), hits.Load(), "a refresh reuses the detected auth style")
175+
}
176+
177+
func TestWithTokenRequestTimeout(t *testing.T) {
178+
clientOn := func(ctx context.Context) *http.Client {
179+
return ctx.Value(oauth2.HTTPClient).(*http.Client)
180+
}
181+
182+
t.Run("default client gets the bound", func(t *testing.T) {
183+
c := clientOn(withTokenRequestTimeout(context.Background()))
184+
assert.Equal(t, tokenRequestTimeout, c.Timeout)
185+
assert.Nil(t, c.Transport)
186+
assert.Zero(t, http.DefaultClient.Timeout, "http.DefaultClient must not be mutated")
187+
})
188+
189+
t.Run("supplied client keeps its transport and is capped", func(t *testing.T) {
190+
base := &http.Client{Transport: &http.Transport{}, Timeout: 15 * time.Minute}
191+
c := clientOn(withTokenRequestTimeout(context.WithValue(context.Background(), oauth2.HTTPClient, base)))
192+
assert.Equal(t, tokenRequestTimeout, c.Timeout)
193+
assert.Same(t, base.Transport, c.Transport)
194+
assert.Equal(t, 15*time.Minute, base.Timeout, "supplied client must not be mutated")
195+
})
196+
197+
t.Run("shorter supplied timeout is kept", func(t *testing.T) {
198+
base := &http.Client{Timeout: 5 * time.Second}
199+
c := clientOn(withTokenRequestTimeout(context.WithValue(context.Background(), oauth2.HTTPClient, base)))
200+
assert.Equal(t, 5*time.Second, c.Timeout)
201+
})
202+
}

0 commit comments

Comments
 (0)