Skip to content

Commit 40b0609

Browse files
feat(messagequeue): supervisor ticks use tenant IN-list (#702)
## Summary ### Why? Each subscription tick fanned out one goroutine and one `WHERE tenant = ?` round-trip per configured tenant. Tenants that hash onto the same Vitess shard still paid N Go timeouts and N queries. Partial-read failures in that new shared path also dropped poll workers or skipped lease renew, so one listing error could stall every tenant until the next successful tick. ### What? Replace supervisor reads and same-shape writes with `WHERE tenant IN (MQ_TENANTS)`. Go still groups rows by tenant and applies fair-share per `(tenant, topic)`. `TryAcquireLease` and per-partition `ReleaseLease` stay row-local. Poll workers and publish SQL are unchanged. If discovery or the leased-list read fails, cached discovery is kept and unconfirmed workers stop. If GetAllLeases or ActiveSubscribers fails after leases are known, skip acquire and reconcile from this tick's leased list. If the lease tick cannot list owned partitions or active peers, skip rebalance and still renew, heartbeat, and purge. Remove the superseded single-tenant supervisor store methods and their test-only callers so the private interfaces expose only the production paths. ## Test Plan - ✅ `make check-mocks && make check-gazelle && make check-tidy` - ✅ `./tool/bazel test //platform/extension/messagequeue/mysql:go_default_test --test_output=errors` - ✅ `./tool/bazel test //test/integration/extension/messagequeue/mysql:go_default_test --test_output=errors` - ✅ `./tool/bazel test //test/integration/extension/messagequeue/mysql/vitess:go_default_test --test_output=errors --strategy=TestRunner=local` Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e50c20e commit 40b0609

16 files changed

Lines changed: 1108 additions & 1367 deletions

doc/rfc/messagequeue-tenant-sharding.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,12 @@ Today partition discovery runs `SELECT DISTINCT partition_key FROM queue_message
6969
The subscriber takes an explicit configured tenant list from `MQ_TENANTS`. Consumer processes reject an empty list at startup; Stovepipe also rejects ingest requests for names outside the list. Discovery becomes:
7070

7171
```sql
72-
SELECT DISTINCT partition_key FROM queue_messages
73-
WHERE tenant = ? AND topic = ?
74-
ORDER BY partition_key
72+
SELECT DISTINCT tenant, partition_key FROM queue_messages
73+
WHERE tenant IN (MQ_TENANTS) AND topic = ?
74+
ORDER BY tenant, partition_key
7575
```
7676

77-
Fair-share, orphan sweep, and idle-lease release run per `(tenant, topic)`, not across all tenants on a topic. Discovery and shutdown attempt every configured tenant and aggregate errors so one unavailable shard does not block unrelated tenants.
77+
vtgate scatters only to shards that own those vindex values. Fair-share, orphan sweep, and idle-lease release still run per `(tenant, topic)` after grouping the result set in Go. One unavailable serving shard fails the tick for every listed tenant; the next interval retries. Poll workers stay scoped to leased `(tenant, partition_key)` rows. Discovery never uses an unscoped `WHERE topic = ?` predicate on Vitess.
7878

7979
## Publish
8080

platform/extension/messagequeue/mysql/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
"delivery_state_store.go",
88
"errors.go",
99
"identifier.go",
10+
"inlist.go",
1011
"message_store.go",
1112
"mock_stores.go",
1213
"offset_store.go",
@@ -35,6 +36,7 @@ go_test(
3536
name = "go_default_test",
3637
srcs = [
3738
"delivery_state_store_test.go",
39+
"inlist_test.go",
3840
"message_store_test.go",
3941
"offset_store_test.go",
4042
"partition_lease_store_test.go",

platform/extension/messagequeue/mysql/delivery_state_store.go

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -246,22 +246,21 @@ func (s *sqldeliveryStateStore) AdvanceWatermark(ctx context.Context, consumerGr
246246
}
247247

248248
// Batch-fetch delivery state for the provided offsets.
249-
placeholders := make([]byte, 0, len(offsets)*2-1)
249+
placeholders, ok := inListPlaceholders(len(offsets))
250+
if !ok {
251+
return currentWatermark, nil
252+
}
250253
args := make([]interface{}, 0, 4+len(offsets))
251254
args = append(args, tenant, consumerGroup, topic, partitionKey)
252-
for i, offset := range offsets {
253-
if i > 0 {
254-
placeholders = append(placeholders, ',')
255-
}
256-
placeholders = append(placeholders, '?')
255+
for _, offset := range offsets {
257256
args = append(args, offset)
258257
}
259258

260259
rows, err := s.db.QueryContext(ctx, fmt.Sprintf(`
261260
SELECT message_offset, acked FROM %s
262261
WHERE tenant = ? AND consumer_group = ? AND topic = ? AND partition_key = ?
263262
AND message_offset IN (%s)
264-
`, DeliveryStateTableName, string(placeholders)), args...)
263+
`, DeliveryStateTableName, placeholders), args...)
265264
if err != nil {
266265
return currentWatermark, fmt.Errorf("query delivery state for watermark tenant=%s topic=%s partition=%s: %w", tenant, topic, partitionKey, err)
267266
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package mysql
16+
17+
import "strings"
18+
19+
// inListPlaceholders returns "?,?,…" of length n. n < 1 is a no-op query.
20+
func inListPlaceholders(n int) (string, bool) {
21+
if n < 1 {
22+
return "", false
23+
}
24+
return strings.Repeat(",?", n)[1:], true
25+
}
26+
27+
func appendStrings(args []any, values []string) []any {
28+
for _, value := range values {
29+
args = append(args, value)
30+
}
31+
return args
32+
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package mysql
16+
17+
import (
18+
"testing"
19+
20+
"github.com/stretchr/testify/assert"
21+
)
22+
23+
func TestInListPlaceholders(t *testing.T) {
24+
tests := []struct {
25+
n int
26+
want string
27+
wantOK bool
28+
}{
29+
{n: 0},
30+
{n: -1},
31+
{n: 1, want: "?", wantOK: true},
32+
{n: 3, want: "?,?,?", wantOK: true},
33+
}
34+
for _, tt := range tests {
35+
got, ok := inListPlaceholders(tt.n)
36+
assert.Equal(t, tt.wantOK, ok)
37+
assert.Equal(t, tt.want, got)
38+
}
39+
}

platform/extension/messagequeue/mysql/message_store.go

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -144,22 +144,6 @@ func (s *sqlmessageStore) Insert(ctx context.Context, tenant string, topic strin
144144
return nil
145145
}
146146

147-
// Delete deletes a message by tenant, topic, partition key, and ID
148-
func (s *sqlmessageStore) Delete(ctx context.Context, tenant string, topic string, partitionKey string, messageID string) (retErr error) {
149-
op := metrics.Begin(s.scope, "delete", metrics.StorageLatencyBuckets, metrics.NewTag("topic", topic))
150-
defer func() { op.Complete(retErr) }()
151-
152-
_, err := s.db.ExecContext(ctx, fmt.Sprintf(`
153-
DELETE FROM %s WHERE tenant = ? AND topic = ? AND partition_key = ? AND id = ?
154-
`, MessagesTableName), tenant, topic, partitionKey, messageID)
155-
156-
if err != nil {
157-
return fmt.Errorf("delete message tenant=%s topic=%s partition=%s message=%s: %w", tenant, topic, partitionKey, messageID, err)
158-
}
159-
160-
return nil
161-
}
162-
163147
// FetchByOffset fetches messages with offset > currentOffset for a specific partition.
164148
// Messages are fetched from the immutable log; no per-message mutation occurs.
165149
func (s *sqlmessageStore) FetchByOffset(ctx context.Context, tenant string, topic string, partitionKey string, currentOffset int64, limit int) (_ []messageRow, retErr error) {

platform/extension/messagequeue/mysql/message_store_test.go

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -131,24 +131,6 @@ func TestMessageStore_Insert(t *testing.T) {
131131
}
132132
}
133133

134-
func TestMessageStore_Delete(t *testing.T) {
135-
db, mock, store := setupmessageStoreTest(t)
136-
defer db.Close()
137-
138-
ctx := context.Background()
139-
topic := "test_topic"
140-
partitionKey := "part1"
141-
messageID := "msg1"
142-
143-
mock.ExpectExec("DELETE FROM queue_messages").
144-
WithArgs(testTenant, topic, partitionKey, messageID).
145-
WillReturnResult(sqlmock.NewResult(0, 1))
146-
147-
err := store.Delete(ctx, testTenant, topic, partitionKey, messageID)
148-
require.NoError(t, err)
149-
require.NoError(t, mock.ExpectationsWereMet())
150-
}
151-
152134
func TestMessageStore_FetchByOffset(t *testing.T) {
153135
db, mock, store := setupmessageStoreTest(t)
154136
defer db.Close()

0 commit comments

Comments
 (0)