Skip to content

Commit b49618c

Browse files
JamyDevclaude
andcommitted
feat(platform): add pipeline.Construct engine
## Summary - Introduces `platform/pipeline` with the typed `Construct[D]` engine - `Stage[D]` struct declares pipeline topology as a typed table (key, name, consumer group, controller constructor, optional DLQ) - Engine builds topic registry, creates primary + DLQ consumers, eagerly constructs all controllers, pairs DLQ stages automatically - Returns a `lifecycle.Component` for ordered start/stop - Options: `TopicNames`, `Classifiers`, `PublishOnly`, `ExtraComponents` - Pure addition — no existing code changes Step 2 of the [Modular Queue Wiring RFC](https://github.com/uber/submitqueue/blob/main/doc/rfc/submitqueue/modular-queue-wiring.md). Stacked on #402. ## Test plan - [x] 10 unit tests: single stage, DLQ pairing, multiple stages, empty stages error, controller creation failure, DLQ creation failure, publish-only topics, topic name overrides, resolveTopicName table test, dlqTopicKey, buildTopicConfigs - [x] `bazel test //platform/pipeline:go_default_test` passes - [x] `make gazelle` — BUILD.bazel in sync - [x] `make fmt` — code formatted Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent e52af1b commit b49618c

3 files changed

Lines changed: 670 additions & 0 deletions

File tree

platform/pipeline/BUILD.bazel

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
load("@rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["pipeline.go"],
6+
importpath = "github.com/uber/submitqueue/platform/pipeline",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//platform/consumer:go_default_library",
10+
"//platform/errs:go_default_library",
11+
"//platform/extension/messagequeue:go_default_library",
12+
"//platform/lifecycle:go_default_library",
13+
"@com_github_uber_go_tally//:go_default_library",
14+
"@org_uber_go_zap//:go_default_library",
15+
],
16+
)
17+
18+
go_test(
19+
name = "go_default_test",
20+
srcs = ["pipeline_test.go"],
21+
embed = [":go_default_library"],
22+
deps = [
23+
"//platform/consumer:go_default_library",
24+
"//platform/extension/messagequeue:go_default_library",
25+
"//platform/extension/messagequeue/mock:go_default_library",
26+
"@com_github_stretchr_testify//assert:go_default_library",
27+
"@com_github_stretchr_testify//require:go_default_library",
28+
"@com_github_uber_go_tally//:go_default_library",
29+
"@org_uber_go_mock//gomock:go_default_library",
30+
"@org_uber_go_zap//:go_default_library",
31+
],
32+
)

platform/pipeline/pipeline.go

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
// Copyright (c) 2025 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package pipeline provides a typed engine for assembling queue-driven
16+
// service pipelines from declarative data. A service declares its topology
17+
// as a []Stage[D] table and its dependencies as a Deps struct; Construct
18+
// builds all consumers, registers controllers, pairs DLQ stages, and
19+
// returns a single lifecycle.Component the host drives with Start/Stop.
20+
package pipeline
21+
22+
import (
23+
"context"
24+
"fmt"
25+
"time"
26+
27+
"github.com/uber-go/tally"
28+
"github.com/uber/submitqueue/platform/consumer"
29+
"github.com/uber/submitqueue/platform/errs"
30+
extqueue "github.com/uber/submitqueue/platform/extension/messagequeue"
31+
"github.com/uber/submitqueue/platform/lifecycle"
32+
"go.uber.org/zap"
33+
)
34+
35+
// timeNow is a hook for tests to control time. Production uses time.Now.
36+
var timeNow = time.Now
37+
38+
// Stage is one row of a service's topology table. D is the service's Deps type.
39+
type Stage[D any] struct {
40+
// Key is the stage's logical topic key (e.g. topickey.TopicKeyStart).
41+
// The engine maps it to a physical topic name via the TopicNames option.
42+
Key consumer.TopicKey
43+
44+
// Name is the physical topic name for this stage (e.g. "start").
45+
// Used as the default when no TopicNames override is provided.
46+
Name string
47+
48+
// ConsumerGroup is the consumer group suffix for this stage's subscription
49+
// (e.g. "orchestrator-start").
50+
ConsumerGroup string
51+
52+
// New builds the stage's controller from the service's Deps. The engine
53+
// calls it once, eagerly, inside Construct — so a nil/missing dependency
54+
// fails at boot with the stage's name on it, never mid-delivery.
55+
New func(D) (consumer.Controller, error)
56+
57+
// DLQ, when non-nil, declares "this stage dead-letters". The engine then
58+
// derives the paired DLQ topic (<topic>_dlq, retry budget, DLQ-of-DLQ
59+
// disabled) AND registers this reconciler on the DLQ consumer. Declaring
60+
// one without getting the other is impossible — that's the invariant.
61+
DLQ func(D) (consumer.Controller, error)
62+
}
63+
64+
// PublishOnlyTopic declares a topic the service publishes to but does not
65+
// consume. The engine registers it in the TopicRegistry so controllers
66+
// can publish to it, but creates no subscription or controller.
67+
type PublishOnlyTopic struct {
68+
// Key is the logical topic key.
69+
Key consumer.TopicKey
70+
71+
// Name is the physical topic name.
72+
Name string
73+
}
74+
75+
// options holds the resolved configuration for a Construct call.
76+
type options struct {
77+
topicNames map[consumer.TopicKey]string
78+
classifiers []errs.Classifier
79+
publishOnly []PublishOnlyTopic
80+
extraComponents []lifecycle.Component
81+
}
82+
83+
// Option configures a Construct call.
84+
type Option func(*options)
85+
86+
// TopicNames provides a mapping from logical topic keys to physical topic
87+
// names. Keys not present in the map fall back to the Stage.Name default.
88+
func TopicNames(m map[consumer.TopicKey]string) Option {
89+
return func(o *options) { o.topicNames = m }
90+
}
91+
92+
// Classifiers sets the error classifiers for the primary consumer's
93+
// ErrorProcessor. DLQ consumers always use AlwaysRetryableProcessor.
94+
func Classifiers(c ...errs.Classifier) Option {
95+
return func(o *options) { o.classifiers = c }
96+
}
97+
98+
// PublishOnly adds topics the service publishes to but does not consume.
99+
func PublishOnly(topics ...PublishOnlyTopic) Option {
100+
return func(o *options) { o.publishOnly = append(o.publishOnly, topics...) }
101+
}
102+
103+
// ExtraComponents adds lifecycle components that are started before
104+
// consumers and stopped after them.
105+
func ExtraComponents(c ...lifecycle.Component) Option {
106+
return func(o *options) { o.extraComponents = append(o.extraComponents, c...) }
107+
}
108+
109+
// dlqTopicKey returns the DLQ topic key for a primary stage key.
110+
// Matches the convention in submitqueue/orchestrator/controller/dlq.TopicKey.
111+
const dlqTopicSuffix = "_dlq"
112+
113+
func dlqTopicKey(primary consumer.TopicKey) consumer.TopicKey {
114+
return consumer.TopicKey(string(primary) + dlqTopicSuffix)
115+
}
116+
117+
// Construct is the single assembly function for a queue-driven service.
118+
// It builds the topic registry, creates primary and DLQ consumers,
119+
// eagerly constructs all controllers, and returns a lifecycle.Component
120+
// that starts and stops everything in the correct order.
121+
//
122+
// The returned Component starts in this order:
123+
// 1. Extra components (infrastructure)
124+
// 2. Primary consumer (work-accepting)
125+
// 3. DLQ consumer (reconciliation)
126+
//
127+
// Stop reverses the order: DLQ consumer drains first, then primary, then
128+
// infrastructure.
129+
func Construct[D any](
130+
logger *zap.SugaredLogger,
131+
scope tally.Scope,
132+
queue extqueue.Queue,
133+
subscriberName string,
134+
deps D,
135+
stages []Stage[D],
136+
opts ...Option,
137+
) (lifecycle.Component, error) {
138+
if len(stages) == 0 {
139+
return nil, fmt.Errorf("pipeline: at least one stage is required")
140+
}
141+
142+
o := &options{}
143+
for _, opt := range opts {
144+
opt(o)
145+
}
146+
147+
// Build topic configs for the registry.
148+
configs, err := buildTopicConfigs(queue, subscriberName, stages, o)
149+
if err != nil {
150+
return nil, err
151+
}
152+
153+
registry, err := consumer.NewTopicRegistry(configs)
154+
if err != nil {
155+
return nil, fmt.Errorf("pipeline: failed to create topic registry: %w", err)
156+
}
157+
158+
// Create the primary consumer with user-provided classifiers.
159+
primaryProcessor := errs.NewClassifierProcessor(o.classifiers...)
160+
primary := consumer.New(logger, scope, registry, primaryProcessor)
161+
162+
// Create the DLQ consumer with always-retryable processor.
163+
dlq := consumer.New(logger, scope, registry, errs.AlwaysRetryableProcessor)
164+
165+
hasDLQ := false
166+
167+
// Eagerly construct and register all controllers.
168+
for _, s := range stages {
169+
ctl, err := s.New(deps)
170+
if err != nil {
171+
return nil, fmt.Errorf("pipeline: stage %s: failed to create controller: %w", s.Key, err)
172+
}
173+
if err := primary.Register(ctl); err != nil {
174+
return nil, fmt.Errorf("pipeline: stage %s: failed to register controller: %w", s.Key, err)
175+
}
176+
177+
if s.DLQ != nil {
178+
rec, err := s.DLQ(deps)
179+
if err != nil {
180+
return nil, fmt.Errorf("pipeline: stage %s dlq: failed to create controller: %w", s.Key, err)
181+
}
182+
if err := dlq.Register(rec); err != nil {
183+
return nil, fmt.Errorf("pipeline: stage %s dlq: failed to register controller: %w", s.Key, err)
184+
}
185+
hasDLQ = true
186+
}
187+
}
188+
189+
// Compose the lifecycle group.
190+
members := make([]lifecycle.Component, 0, len(o.extraComponents)+2)
191+
members = append(members, o.extraComponents...)
192+
members = append(members, &consumerComponent{name: "primary", c: primary})
193+
if hasDLQ {
194+
members = append(members, &consumerComponent{name: "dlq", c: dlq})
195+
}
196+
197+
return lifecycle.NewGroup(members...), nil
198+
}
199+
200+
// buildTopicConfigs constructs the []consumer.TopicConfig from stages and options.
201+
func buildTopicConfigs[D any](
202+
queue extqueue.Queue,
203+
subscriberName string,
204+
stages []Stage[D],
205+
o *options,
206+
) ([]consumer.TopicConfig, error) {
207+
// Pre-size: each stage gets a primary config + optional DLQ config,
208+
// plus publish-only topics.
209+
configs := make([]consumer.TopicConfig, 0, 2*len(stages)+len(o.publishOnly))
210+
211+
for _, s := range stages {
212+
topicName := resolveTopicName(s.Key, s.Name, o.topicNames)
213+
214+
configs = append(configs, consumer.TopicConfig{
215+
Key: s.Key,
216+
Name: topicName,
217+
Queue: queue,
218+
Subscription: extqueue.DefaultSubscriptionConfig(
219+
subscriberName, s.ConsumerGroup,
220+
),
221+
})
222+
223+
if s.DLQ != nil {
224+
configs = append(configs, consumer.TopicConfig{
225+
Key: dlqTopicKey(s.Key),
226+
Name: topicName + dlqTopicSuffix,
227+
Queue: queue,
228+
Subscription: extqueue.DLQSubscriptionConfig(
229+
subscriberName, s.ConsumerGroup+"-dlq",
230+
),
231+
})
232+
}
233+
}
234+
235+
for _, p := range o.publishOnly {
236+
topicName := resolveTopicName(p.Key, p.Name, o.topicNames)
237+
configs = append(configs, consumer.TopicConfig{
238+
Key: p.Key,
239+
Name: topicName,
240+
Queue: queue,
241+
})
242+
}
243+
244+
return configs, nil
245+
}
246+
247+
// resolveTopicName returns the override name if present, otherwise the default.
248+
func resolveTopicName(key consumer.TopicKey, defaultName string, overrides map[consumer.TopicKey]string) string {
249+
if overrides != nil {
250+
if name, ok := overrides[key]; ok {
251+
return name
252+
}
253+
}
254+
return defaultName
255+
}
256+
257+
// consumerComponent adapts consumer.Consumer to lifecycle.Component.
258+
// Consumer.Stop takes a timeoutMs int64; Component.Stop takes a context.
259+
// We derive the timeout from the context's deadline if set, defaulting to 30s.
260+
type consumerComponent struct {
261+
name string
262+
c consumer.Consumer
263+
}
264+
265+
func (a *consumerComponent) Start(ctx context.Context) error {
266+
return a.c.Start(ctx)
267+
}
268+
269+
func (a *consumerComponent) Stop(ctx context.Context) error {
270+
const defaultStopTimeoutMs = 30000
271+
timeoutMs := int64(defaultStopTimeoutMs)
272+
if deadline, ok := ctx.Deadline(); ok {
273+
remaining := deadline.Sub(timeNow())
274+
if remaining > 0 {
275+
timeoutMs = remaining.Milliseconds()
276+
} else {
277+
timeoutMs = 0
278+
}
279+
}
280+
return a.c.Stop(timeoutMs)
281+
}

0 commit comments

Comments
 (0)