diff --git a/AGENTS.md b/AGENTS.md index ac047121c..c39eca8ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -168,7 +168,7 @@ Paths follow the directory layout: shared packages live under `platform/` at the - Queue contracts: external `github.com/uber/submitqueue/api/{domain}/messagequeue`; internal `github.com/uber/submitqueue/{domain}/core/messagequeue` - Domain entities: `github.com/uber/submitqueue/{domain}/entity` (e.g. `.../submitqueue/entity`) - Domain extensions: `github.com/uber/submitqueue/{domain}/extension/{ext}[/{impl}]` (e.g. `.../submitqueue/extension/storage/mysql`) -- Cross-domain consumer framework: `github.com/uber/submitqueue/platform/consumer`; internal topic keys live with the owning domain contract (for example `submitqueue/core/topickey` and `stovepipe/core/messagequeue`); external queue topic keys live with their published contract (for example `api/runway/messagequeue`) +- Cross-domain consumer framework: `github.com/uber/submitqueue/platform/consumer`; internal topic keys live with the owning domain contract (for example `submitqueue/core/messagequeue` and `stovepipe/core/messagequeue`); external queue topic keys live with their published contract (for example `api/runway/messagequeue`) - Domain-internal infra: `github.com/uber/submitqueue/{domain}/core/{pkg}` (e.g. `.../submitqueue/core/request`) - Shared entities: `github.com/uber/submitqueue/platform/base/{pkg}` (e.g. `.../platform/base/messagequeue`) - Shared extensions: `github.com/uber/submitqueue/platform/extension/{ext}[/{impl}]` (e.g. `.../platform/extension/messagequeue/mysql`) @@ -200,7 +200,7 @@ New queue contracts are defined in **proto3** (`.proto` under `proto/`, generate The message types are generated; the contract package adds only generic `protojson` glue — `Marshal(m)` / `Unmarshal[T](b, m)` — owning the wire conventions: `UseProtoNames` (snake_case fields), UPPER_SNAKE enum values, int64-as-string, unknown fields discarded on read (additive evolution). The topic key(s) carrying a message are declared on the message via the `topic_keys` proto option — a `google.protobuf.MessageOptions` extension defined in `api/base/messagequeue`. A topic key is a stable logical name, not a concrete wire topic; each implementer maps it to its backend's topic name, and a `TopicKeys(msg)` reflection helper reads the option back. It is contract metadata, not the hot path — publish/consume still routes on `consumer.TopicKey` + `TopicRegistry`. The contract package owns both halves: the proto payload and the `TopicKey` constants for its topic keys. A contract test round-trips the payloads and asserts every topic key is bound to exactly one message. Shared field types (`Change`, `Strategy`) are shared protos under `api/base/{change,mergestrategy}`. `api/runway/messagequeue/` is the reference example. -SubmitQueue's internal pipeline predates the proto-backed convention. It continues to serialize domain entities with `encoding/json` and declares its logical keys in `submitqueue/core/topickey/`. Do not convert or mix these wire formats incidentally; treat migration as an explicit compatibility change. +SubmitQueue's internal pipeline contract lives at `submitqueue/core/messagequeue/` (proto3 + protojson, one message per topic key). `MarshalID`/`UnmarshalID` take the topic key so consume uses the bound message. Proto filenames that would collide with another domain in the protobuf registry are prefixed (`submitqueuemerge.proto`, `submitqueuebuild.proto`, `submitqueuebuildsignal.proto`). `submitqueue/core/topickey` re-exports the topic-key constants from that package. ### Naming Conventions diff --git a/Makefile b/Makefile index cae7104c3..b15be5161 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ GOIMPORTS_VERSION ?= v0.33.0 # (the out_dir convention in tool/proto/BUILD.bazel) and copied back here. A # package may hold multiple .proto files (e.g. an RPC contract plus messagequeue # contracts); all generated stubs land in the same protopb/ dir. -PROTO_PACKAGES = api/base/change api/base/hook api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/runway api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe stovepipe/core/messagequeue +PROTO_PACKAGES = api/base/change api/base/hook api/base/mergestrategy api/base/messagequeue api/runway/messagequeue api/runway api/submitqueue/gateway api/submitqueue/orchestrator api/stovepipe stovepipe/core/messagequeue submitqueue/core/messagequeue # Set REPO_ROOT for docker-compose export REPO_ROOT := $(shell pwd) diff --git a/submitqueue/core/messagequeue/BUILD.bazel b/submitqueue/core/messagequeue/BUILD.bazel new file mode 100644 index 000000000..7dc3faf37 --- /dev/null +++ b/submitqueue/core/messagequeue/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "id.go", + "map.go", + "messagequeue.go", + "topics.go", + ], + importpath = "github.com/uber/submitqueue/submitqueue/core/messagequeue", + visibility = [ + "//service/submitqueue:__subpackages__", + "//submitqueue:__subpackages__", + "//test:__subpackages__", + ], + deps = [ + "//api/base/change/protopb:go_default_library", + "//api/base/mergestrategy/protopb:go_default_library", + "//api/base/messagequeue/protopb:go_default_library", + "//platform/base/change:go_default_library", + "//platform/base/mergestrategy:go_default_library", + "//platform/consumer:go_default_library", + "//submitqueue/core/messagequeue/protopb:go_default_library", + "//submitqueue/entity:go_default_library", + "@org_golang_google_protobuf//encoding/protojson:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["messagequeue_test.go"], + embed = [":go_default_library"], + deps = [ + "//platform/base/change:go_default_library", + "//platform/base/mergestrategy:go_default_library", + "//submitqueue/entity:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", + ], +) diff --git a/submitqueue/core/messagequeue/README.md b/submitqueue/core/messagequeue/README.md new file mode 100644 index 000000000..f27e1be0d --- /dev/null +++ b/submitqueue/core/messagequeue/README.md @@ -0,0 +1,25 @@ +# SubmitQueue internal message-queue contract + +Wire payloads for the queues internal to the SubmitQueue pipeline (gateway and orchestrator). It is **internal** — used only within the SubmitQueue domain — so it lives under `submitqueue/core` rather than `api/` (Bazel visibility keeps it domain-scoped). + +Payloads are defined in proto3 (`proto/`, generated into `protopb/`) and serialized as **protobuf JSON** (protojson), so the MySQL-backed queue keeps storing self-describing JSON. The contract package adds protojson glue (`Marshal`/`Unmarshal`), `TopicKeys`, the pipeline `TopicKey` constants, and helpers that map generated payloads to `submitqueue/entity` types. `MarshalID`/`UnmarshalID` take the topic key so the bound message is used; unmarshalling through a different type would drop fields added later (`DiscardUnknown`). Each payload declares the topic key that carries it via the `topic_keys` proto option (defined in `api/base/messagequeue`); a contract test round-trips every payload and asserts each topic key is bound to exactly one message. + +Shared field types `Change` and `Strategy` come from `api/base/change` and `api/base/mergestrategy`. + +## Stages + +Each topic key has its own message, even when the first version is only an id and a queue, so a stage can grow fields without touching others. + +- **start** (`TopicKeyStart`, `Start`) — gateway publishes the minted request id and land inputs; start persists a `Request`. Full payload: this seam crosses services. +- **cancel** (`TopicKeyCancel`, `Cancel`) — gateway publishes the request id to cancel; cancel reloads the `Request`. Full payload across the gateway/orchestrator seam. +- **validate** (`TopicKeyValidate`, `Validate`) — start publishes the request id; validate reloads the `Request`. +- **batch** (`TopicKeyBatch`, `Batch`) — landconflictsignal publishes the request id; batch reloads the `Request`. +- **dependency-analysis** (`TopicKeyDependencyAnalysis`, `DependencyAnalysis`) — batch publishes the batch id; partitioned by queue. +- **speculate** (`TopicKeySpeculate`, `Speculate`) — dependency-analysis (and later stages) publish the batch id. +- **build** (`TopicKeyBuild`, `Build` in `submitqueuebuild.proto`) — speculate publishes a **batch** id; build reloads the `Batch`. The proto filename is not `build.proto` so it does not collide with Stovepipe's `stovepipe/core/messagequeue/proto/build.proto` in the protobuf filename registry. +- **buildsignal** (`TopicKeyBuildSignal`, `BuildSignal` in `submitqueuebuildsignal.proto`) — build publishes a **build** id; buildsignal polls and may hold the delivery. Same filename-registry reason as build. +- **submitqueue-land** (`TopicKeyLand`, `Merge` in `submitqueuemerge.proto`) — speculate publishes a batch id; land reloads the `Batch` before handing work to Runway. The proto filename is not `merge.proto`/`land.proto` so it does not collide with Runway's `api/runway/messagequeue/proto/merge.proto` in the protobuf filename registry. +- **conclude** (`TopicKeyConclude`, `Conclude`) — speculate/landsignal publish a batch id. A failed batch's reason travels in message metadata (`MetadataKeyFailureReason`), not the payload. +- **log** (`TopicKeyLog`, `Log`) — orchestrator publishes a full request-log entry; the gateway materializes it. `type`, `status`, and `event` are open strings matching the domain vocabularies. + +In-boundary stages (validate through conclude, except start/cancel/log) put only an id on the queue because producer and consumer share storage. diff --git a/submitqueue/core/messagequeue/id.go b/submitqueue/core/messagequeue/id.go new file mode 100644 index 000000000..d674f6b2a --- /dev/null +++ b/submitqueue/core/messagequeue/id.go @@ -0,0 +1,143 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messagequeue + +import ( + "fmt" + + "google.golang.org/protobuf/proto" + + "github.com/uber/submitqueue/submitqueue/entity" +) + +type idPayload interface { + proto.Message + GetId() string + GetQueue() string +} + +func idOnlyMessage(key TopicKey, id, queue string) (idPayload, error) { + switch key { + case TopicKeyValidate: + return &Validate{Id: id, Queue: queue}, nil + case TopicKeyBatch: + return &Batch{Id: id, Queue: queue}, nil + case TopicKeyDependencyAnalysis: + return &DependencyAnalysis{Id: id, Queue: queue}, nil + case TopicKeySpeculate: + return &Speculate{Id: id, Queue: queue}, nil + case TopicKeyBuild: + return &Build{Id: id, Queue: queue}, nil + case TopicKeyBuildSignal: + return &BuildSignal{Id: id, Queue: queue}, nil + case TopicKeyLand: + return &Merge{Id: id, Queue: queue}, nil + case TopicKeyConclude: + return &Conclude{Id: id, Queue: queue}, nil + default: + return nil, fmt.Errorf("topic %q does not carry an id-only payload", key) + } +} + +// MarshalID serializes the id-only payload bound to key. Start, cancel, and log +// are not id-only; callers marshal those messages directly. +func MarshalID(key TopicKey, id, queue string) ([]byte, error) { + m, err := idOnlyMessage(key, id, queue) + if err != nil { + return nil, err + } + return Marshal(m) +} + +// UnmarshalID reads id and queue from the id-only payload bound to key. +// Consumers pass the topic they subscribe to so a field added to that message +// is decoded rather than discarded as unknown on a different type. +func UnmarshalID(key TopicKey, b []byte) (id, queue string, err error) { + m, err := idOnlyMessage(key, "", "") + if err != nil { + return "", "", err + } + if err := Unmarshal(b, m); err != nil { + return "", "", err + } + return m.GetId(), m.GetQueue(), nil +} + +// UnmarshalLandRequest reads a start payload into the gateway-owned land request. +func UnmarshalLandRequest(b []byte) (entity.LandRequest, error) { + m := &Start{} + if err := Unmarshal(b, m); err != nil { + return entity.LandRequest{}, err + } + return LandRequestFromStart(m), nil +} + +// UnmarshalCancelRequest reads a cancel payload into the domain cancellation. +func UnmarshalCancelRequest(b []byte) (entity.CancelRequest, error) { + m := &Cancel{} + if err := Unmarshal(b, m); err != nil { + return entity.CancelRequest{}, err + } + return CancelToEntity(m), nil +} + +// UnmarshalRequestLog reads a log payload into a request-log entry. +func UnmarshalRequestLog(b []byte) (entity.RequestLog, error) { + m := &Log{} + if err := Unmarshal(b, m); err != nil { + return entity.RequestLog{}, err + } + return LogToEntity(m), nil +} + +// UnmarshalRequestID reads a request-scoped id-only payload (validate or batch). +func UnmarshalRequestID(key TopicKey, b []byte) (entity.RequestID, error) { + switch key { + case TopicKeyValidate, TopicKeyBatch: + default: + return entity.RequestID{}, fmt.Errorf("topic %q does not carry a request-id payload", key) + } + id, queue, err := UnmarshalID(key, b) + if err != nil { + return entity.RequestID{}, err + } + return entity.RequestID{ID: id, Queue: queue}, nil +} + +// UnmarshalBatchID reads a batch-scoped id-only payload. +func UnmarshalBatchID(key TopicKey, b []byte) (entity.BatchID, error) { + switch key { + case TopicKeyDependencyAnalysis, TopicKeySpeculate, TopicKeyBuild, TopicKeyLand, TopicKeyConclude: + default: + return entity.BatchID{}, fmt.Errorf("topic %q does not carry a batch-id payload", key) + } + id, queue, err := UnmarshalID(key, b) + if err != nil { + return entity.BatchID{}, err + } + return entity.BatchID{ID: id, Queue: queue}, nil +} + +// UnmarshalBuildID reads a buildsignal payload. +func UnmarshalBuildID(key TopicKey, b []byte) (entity.BuildID, error) { + if key != TopicKeyBuildSignal { + return entity.BuildID{}, fmt.Errorf("topic %q does not carry a build-id payload", key) + } + id, queue, err := UnmarshalID(key, b) + if err != nil { + return entity.BuildID{}, err + } + return entity.BuildID{ID: id, Queue: queue}, nil +} diff --git a/submitqueue/core/messagequeue/map.go b/submitqueue/core/messagequeue/map.go new file mode 100644 index 000000000..dcad58ec7 --- /dev/null +++ b/submitqueue/core/messagequeue/map.go @@ -0,0 +1,127 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messagequeue + +import ( + changepb "github.com/uber/submitqueue/api/base/change/protopb" + strategypb "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + "github.com/uber/submitqueue/platform/base/change" + "github.com/uber/submitqueue/platform/base/mergestrategy" + "github.com/uber/submitqueue/submitqueue/entity" +) + +// StartFromLandRequest copies a gateway-owned land request onto the start payload. +func StartFromLandRequest(r entity.LandRequest) *Start { + return &Start{ + Id: r.ID, + Queue: r.Queue, + Change: &changepb.Change{Uris: append([]string{}, r.Change.URIs...)}, + LandStrategy: landStrategyToProto(r.LandStrategy), + } +} + +// LandRequestFromStart copies a start payload onto the gateway-owned land request. +func LandRequestFromStart(m *Start) entity.LandRequest { + var uris []string + if m.GetChange() != nil { + uris = append([]string{}, m.GetChange().GetUris()...) + } + return entity.LandRequest{ + ID: m.GetId(), + Queue: m.GetQueue(), + Change: change.Change{URIs: uris}, + LandStrategy: landStrategyFromProto(m.GetLandStrategy()), + } +} + +// CancelFromEntity copies a cancellation onto the cancel payload. +func CancelFromEntity(r entity.CancelRequest) *Cancel { + return &Cancel{Id: r.ID, Queue: r.Queue, Reason: r.Reason} +} + +// CancelToEntity copies a cancel payload onto the domain cancellation. +func CancelToEntity(m *Cancel) entity.CancelRequest { + return entity.CancelRequest{ID: m.GetId(), Queue: m.GetQueue(), Reason: m.GetReason()} +} + +// LogFromEntity copies a request-log entry onto the log payload. +func LogFromEntity(r entity.RequestLog) *Log { + return &Log{ + RequestId: r.RequestID, + Queue: r.Queue, + TimestampMs: r.TimestampMs, + Type: string(r.Type), + Status: string(r.Status), + Event: string(r.Event), + RequestVersion: r.RequestVersion, + LastError: r.LastError, + Metadata: r.Metadata, + } +} + +// LogToEntity copies a log payload onto a request-log entry. An empty type is +// treated as a status entry, matching entries written before the type field +// existed. A nil metadata map becomes an empty map. +func LogToEntity(m *Log) entity.RequestLog { + meta := m.GetMetadata() + if meta == nil { + meta = make(map[string]string) + } + typ := entity.RequestLogType(m.GetType()) + if typ == "" { + typ = entity.RequestLogTypeStatus + } + return entity.RequestLog{ + RequestID: m.GetRequestId(), + Queue: m.GetQueue(), + TimestampMs: m.GetTimestampMs(), + Type: typ, + Status: entity.RequestStatus(m.GetStatus()), + Event: entity.RequestEvent(m.GetEvent()), + RequestVersion: m.GetRequestVersion(), + LastError: m.GetLastError(), + Metadata: meta, + } +} + +func landStrategyToProto(s mergestrategy.MergeStrategy) strategypb.Strategy { + switch s { + case mergestrategy.MergeStrategyRebase: + return strategypb.Strategy_REBASE + case mergestrategy.MergeStrategySquashRebase: + return strategypb.Strategy_SQUASH_REBASE + case mergestrategy.MergeStrategyMerge: + return strategypb.Strategy_MERGE + case mergestrategy.MergeStrategyPromote: + return strategypb.Strategy_PROMOTE + default: + return strategypb.Strategy_DEFAULT + } +} + +func landStrategyFromProto(s strategypb.Strategy) mergestrategy.MergeStrategy { + switch s { + case strategypb.Strategy_REBASE: + return mergestrategy.MergeStrategyRebase + case strategypb.Strategy_SQUASH_REBASE: + return mergestrategy.MergeStrategySquashRebase + case strategypb.Strategy_MERGE: + return mergestrategy.MergeStrategyMerge + case strategypb.Strategy_PROMOTE: + return mergestrategy.MergeStrategyPromote + default: + return mergestrategy.MergeStrategyUnknown + } +} diff --git a/submitqueue/core/messagequeue/messagequeue.go b/submitqueue/core/messagequeue/messagequeue.go new file mode 100644 index 000000000..17e1326d5 --- /dev/null +++ b/submitqueue/core/messagequeue/messagequeue.go @@ -0,0 +1,92 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package messagequeue holds SubmitQueue's internal message-queue contract: the +// wire payloads for the pipeline queues SubmitQueue owns, defined by the proto +// files in proto/ and generated into protopb/. The proto is the language-neutral +// authority; the generated Go types in protopb are the binding for Go callers. +// +// It is internal — used only within the SubmitQueue domain — so it lives under +// submitqueue/core rather than api/. The message types are generated into +// protopb; this package adds generic protojson glue (Marshal/Unmarshal), the +// topic-key reflection lookup (TopicKeys), the pipeline TopicKey constants, and +// helpers that map between generated payloads and submitqueue/entity types at +// the controller edge. Payloads are serialized as protobuf JSON, not binary, so +// the MySQL-backed queue keeps storing self-describing JSON. The topic key that +// carries each payload is declared on the message itself via the topic_keys +// proto option (see api/base/messagequeue). Proto filenames that would collide +// with another domain's contract in the protobuf filename registry are prefixed +// (submitqueuemerge.proto, submitqueuebuild.proto, submitqueuebuildsignal.proto). +package messagequeue + +import ( + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + basemqpb "github.com/uber/submitqueue/api/base/messagequeue/protopb" + "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb" +) + +// Wire payload types. These alias the generated protobuf bindings so callers +// reference the contract through this curated package rather than protopb. +type ( + Start = protopb.Start + Cancel = protopb.Cancel + Validate = protopb.Validate + Batch = protopb.Batch + DependencyAnalysis = protopb.DependencyAnalysis + Speculate = protopb.Speculate + Build = protopb.Build + BuildSignal = protopb.BuildSignal + Merge = protopb.Merge + Conclude = protopb.Conclude + Log = protopb.Log +) + +// marshalOpts keeps the JSON field names identical to the proto field names +// (snake_case), so the wire shape matches the declared contract rather than +// protojson's default lowerCamelCase. Zero-valued fields are omitted. +var marshalOpts = protojson.MarshalOptions{UseProtoNames: true} + +// unmarshalOpts tolerates unknown fields so an additive contract change (a new +// field a producer sends but this consumer does not yet know) is ignored rather +// than rejected. +var unmarshalOpts = protojson.UnmarshalOptions{DiscardUnknown: true} + +// Marshal serializes any contract message to protojson bytes for the queue +// payload, keeping the proto field names (snake_case) on the wire. +func Marshal(m proto.Message) ([]byte, error) { + return marshalOpts.Marshal(m) +} + +// Unmarshal deserializes protojson bytes into the contract message m, tolerating +// unknown fields so an additive contract change is ignored rather than rejected. +func Unmarshal[T proto.Message](b []byte, m T) error { + return unmarshalOpts.Unmarshal(b, m) +} + +// TopicKeys returns the stable logical topic keys bound to a message via the +// topic_keys proto option — not concrete wire names; a caller maps each key to +// its backend's topic name. Returns nil for a message that declares no keys. +func TopicKeys(m proto.Message) []string { + opts := m.ProtoReflect().Descriptor().Options() + if opts == nil { + return nil + } + keys, ok := proto.GetExtension(opts, basemqpb.E_TopicKeys).([]string) + if !ok { + return nil + } + return keys +} diff --git a/submitqueue/core/messagequeue/messagequeue_test.go b/submitqueue/core/messagequeue/messagequeue_test.go new file mode 100644 index 000000000..5661adad9 --- /dev/null +++ b/submitqueue/core/messagequeue/messagequeue_test.go @@ -0,0 +1,280 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messagequeue + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/uber/submitqueue/platform/base/change" + "github.com/uber/submitqueue/platform/base/mergestrategy" + "github.com/uber/submitqueue/submitqueue/entity" +) + +func TestStartRoundTrip(t *testing.T) { + req := StartFromLandRequest(entity.LandRequest{ + ID: "q/1", + Queue: "q", + Change: change.Change{URIs: []string{"github://github.example.com/org/repo/pull/1/0123456789abcdef0123456789abcdef01234567"}}, + LandStrategy: mergestrategy.MergeStrategySquashRebase, + }) + + data, err := Marshal(req) + require.NoError(t, err) + + got := &Start{} + require.NoError(t, Unmarshal(data, got)) + assert.True(t, proto.Equal(req, got)) + assert.Equal(t, mergestrategy.MergeStrategySquashRebase, LandRequestFromStart(got).LandStrategy) +} + +func TestCancelRoundTrip(t *testing.T) { + msg := CancelFromEntity(entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"}) + data, err := Marshal(msg) + require.NoError(t, err) + got := &Cancel{} + require.NoError(t, Unmarshal(data, got)) + assert.Equal(t, entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"}, CancelToEntity(got)) +} + +func TestIDMessageRoundTrip(t *testing.T) { + tests := []struct { + name string + msg proto.Message + into proto.Message + }{ + {name: "validate", msg: &Validate{Id: "q/1", Queue: "q"}, into: &Validate{}}, + {name: "batch", msg: &Batch{Id: "q/1", Queue: "q"}, into: &Batch{}}, + {name: "dependency-analysis", msg: &DependencyAnalysis{Id: "q/batch/1", Queue: "q"}, into: &DependencyAnalysis{}}, + {name: "speculate", msg: &Speculate{Id: "q/batch/1", Queue: "q"}, into: &Speculate{}}, + {name: "build", msg: &Build{Id: "q/batch/1", Queue: "q"}, into: &Build{}}, + {name: "buildsignal", msg: &BuildSignal{Id: "build-1", Queue: "q"}, into: &BuildSignal{}}, + {name: "merge", msg: &Merge{Id: "q/batch/1", Queue: "q"}, into: &Merge{}}, + {name: "conclude", msg: &Conclude{Id: "q/batch/1", Queue: "q"}, into: &Conclude{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := Marshal(tt.msg) + require.NoError(t, err) + require.NoError(t, Unmarshal(data, tt.into)) + assert.True(t, proto.Equal(tt.msg, tt.into)) + }) + } +} + +func TestLogRoundTrip(t *testing.T) { + entry := entity.NewRequestStatusLog("q", "q/1", entity.RequestStatusStarted, 1, "boom", map[string]string{"k": "v"}) + entry.TimestampMs = 1700000000000 + + data, err := Marshal(LogFromEntity(entry)) + require.NoError(t, err) + got := &Log{} + require.NoError(t, Unmarshal(data, got)) + assert.Equal(t, entry, LogToEntity(got)) +} + +func TestLogToEntityDefaults(t *testing.T) { + got := LogToEntity(&Log{RequestId: "q/1", Queue: "q"}) + assert.Equal(t, entity.RequestLogTypeStatus, got.Type) + assert.NotNil(t, got.Metadata) + assert.Empty(t, got.Metadata) +} + +func TestLogEventRoundTrip(t *testing.T) { + entry := entity.NewRequestEventLog("q", "q/1", entity.RequestEventBuilding, map[string]string{"build_id": "b/7"}) + entry.TimestampMs = 1700000000000 + + data, err := Marshal(LogFromEntity(entry)) + require.NoError(t, err) + got, err := UnmarshalRequestLog(data) + require.NoError(t, err) + assert.Equal(t, entry, got) + assert.Equal(t, entity.RequestLogTypeEvent, got.Type) + assert.Equal(t, entity.RequestStatusUnknown, got.Status) + assert.Equal(t, int32(0), got.RequestVersion) +} + +func TestLandStrategyMapping(t *testing.T) { + tests := []struct { + name string + in mergestrategy.MergeStrategy + want mergestrategy.MergeStrategy + }{ + {name: "rebase", in: mergestrategy.MergeStrategyRebase, want: mergestrategy.MergeStrategyRebase}, + {name: "squash", in: mergestrategy.MergeStrategySquashRebase, want: mergestrategy.MergeStrategySquashRebase}, + {name: "merge", in: mergestrategy.MergeStrategyMerge, want: mergestrategy.MergeStrategyMerge}, + {name: "promote", in: mergestrategy.MergeStrategyPromote, want: mergestrategy.MergeStrategyPromote}, + {name: "unknown_stays_unknown", in: mergestrategy.MergeStrategyUnknown, want: mergestrategy.MergeStrategyUnknown}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := LandRequestFromStart(StartFromLandRequest(entity.LandRequest{LandStrategy: tt.in})) + assert.Equal(t, tt.want, got.LandStrategy) + }) + } +} + +func TestStartNilChange(t *testing.T) { + got := LandRequestFromStart(&Start{Id: "q/1", Queue: "q"}) + assert.Nil(t, got.Change.URIs) + assert.Equal(t, mergestrategy.MergeStrategyUnknown, got.LandStrategy) +} + +// TestWireFormat locks protojson encoding: snake_case names, UPPER_SNAKE enums, +// and int64 as a JSON string. +func TestWireFormat(t *testing.T) { + start, err := Marshal(StartFromLandRequest(entity.LandRequest{ + ID: "q/1", + Queue: "q", + Change: change.Change{URIs: []string{"u"}}, + LandStrategy: mergestrategy.MergeStrategyRebase, + })) + require.NoError(t, err) + assert.Contains(t, string(start), `"id"`) + assert.Contains(t, string(start), `"queue"`) + assert.Contains(t, string(start), `"land_strategy"`) + assert.Contains(t, string(start), `"REBASE"`) + + logBytes, err := Marshal(LogFromEntity(entity.RequestLog{RequestID: "q/1", TimestampMs: 42})) + require.NoError(t, err) + assert.Contains(t, string(logBytes), `"timestamp_ms":"42"`) +} + +func TestMarshalIDRoundTripPerTopic(t *testing.T) { + keys := []TopicKey{ + TopicKeyValidate, + TopicKeyBatch, + TopicKeyDependencyAnalysis, + TopicKeySpeculate, + TopicKeyBuild, + TopicKeyBuildSignal, + TopicKeyLand, + TopicKeyConclude, + } + for _, key := range keys { + t.Run(key.String(), func(t *testing.T) { + data, err := MarshalID(key, "id-1", "q") + require.NoError(t, err) + id, queue, err := UnmarshalID(key, data) + require.NoError(t, err) + assert.Equal(t, "id-1", id) + assert.Equal(t, "q", queue) + }) + } + + rid, err := UnmarshalRequestID(TopicKeyValidate, []byte(`{"id":"r","queue":"q"}`)) + require.NoError(t, err) + assert.Equal(t, entity.RequestID{ID: "r", Queue: "q"}, rid) + + build, err := UnmarshalBuildID(TopicKeyBuildSignal, []byte(`{"id":"b","queue":"q"}`)) + require.NoError(t, err) + assert.Equal(t, entity.BuildID{ID: "b", Queue: "q"}, build) + + _, err = UnmarshalRequestID(TopicKeyValidate, []byte(`{`)) + require.Error(t, err) + _, err = UnmarshalBatchID(TopicKeySpeculate, []byte(`{`)) + require.Error(t, err) + _, err = UnmarshalBuildID(TopicKeyBuildSignal, []byte(`{`)) + require.Error(t, err) + _, err = UnmarshalLandRequest([]byte(`{`)) + require.Error(t, err) + _, err = UnmarshalCancelRequest([]byte(`{`)) + require.Error(t, err) + _, err = UnmarshalRequestLog([]byte(`{`)) + require.Error(t, err) + + land, err := UnmarshalLandRequest([]byte(`{"id":"q/1","queue":"q"}`)) + require.NoError(t, err) + assert.Equal(t, "q/1", land.ID) + cancel, err := UnmarshalCancelRequest([]byte(`{"id":"q/7","queue":"q","reason":"x"}`)) + require.NoError(t, err) + assert.Equal(t, "x", cancel.Reason) + logEntry, err := UnmarshalRequestLog([]byte(`{"request_id":"q/1","queue":"q"}`)) + require.NoError(t, err) + assert.Equal(t, entity.RequestLogTypeStatus, logEntry.Type) +} + +func TestMarshalIDRejectsUnknownTopic(t *testing.T) { + _, err := MarshalID(TopicKeyStart, "q/1", "q") + require.Error(t, err) +} + +func TestUnmarshalIDAndTypedIDs(t *testing.T) { + data, err := MarshalID(TopicKeyBuild, "q/batch/1", "q") + require.NoError(t, err) + + id, queue, err := UnmarshalID(TopicKeyBuild, data) + require.NoError(t, err) + assert.Equal(t, "q/batch/1", id) + assert.Equal(t, "q", queue) + + bid, err := UnmarshalBatchID(TopicKeyBuild, data) + require.NoError(t, err) + assert.Equal(t, entity.BatchID{ID: "q/batch/1", Queue: "q"}, bid) + + _, _, err = UnmarshalID(TopicKeyBuild, []byte(`{`)) + require.Error(t, err) +} + +func TestUnmarshalIDRejectsNonIDTopic(t *testing.T) { + _, _, err := UnmarshalID(TopicKeyStart, []byte(`{"id":"q/1","queue":"q"}`)) + require.Error(t, err) + _, err = UnmarshalRequestID(TopicKeySpeculate, []byte(`{"id":"q/1","queue":"q"}`)) + require.Error(t, err) + _, err = UnmarshalBatchID(TopicKeyValidate, []byte(`{"id":"q/1","queue":"q"}`)) + require.Error(t, err) + _, err = UnmarshalBuildID(TopicKeyBuild, []byte(`{"id":"b","queue":"q"}`)) + require.Error(t, err) +} + +func TestTopicKeysBindEveryTopicKey(t *testing.T) { + bound := map[string]int{} + for _, m := range []proto.Message{ + &Start{}, &Cancel{}, &Validate{}, &Batch{}, &DependencyAnalysis{}, + &Speculate{}, &Build{}, &BuildSignal{}, &Merge{}, &Conclude{}, &Log{}, + } { + keys := TopicKeys(m) + require.NotEmpty(t, keys, "message must declare a non-empty topic_keys option") + for _, key := range keys { + bound[key]++ + } + } + + keys := []TopicKey{ + TopicKeyStart, + TopicKeyCancel, + TopicKeyValidate, + TopicKeyBatch, + TopicKeyDependencyAnalysis, + TopicKeySpeculate, + TopicKeyBuild, + TopicKeyBuildSignal, + TopicKeyLand, + TopicKeyConclude, + TopicKeyLog, + } + + valid := map[string]bool{} + for _, k := range keys { + valid[k.String()] = true + assert.Equalf(t, 1, bound[k.String()], "topic key %q must be bound to exactly one message via the topic_keys option", k) + } + for key := range bound { + assert.Truef(t, valid[key], "topic_keys option names unknown key %q", key) + } +} diff --git a/submitqueue/core/messagequeue/proto/BUILD.bazel b/submitqueue/core/messagequeue/proto/BUILD.bazel new file mode 100644 index 000000000..469a76762 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/BUILD.bazel @@ -0,0 +1,16 @@ +exports_files( + [ + "batch.proto", + "cancel.proto", + "conclude.proto", + "dependencyanalysis.proto", + "log.proto", + "speculate.proto", + "start.proto", + "submitqueuebuild.proto", + "submitqueuebuildsignal.proto", + "submitqueuemerge.proto", + "validate.proto", + ], + visibility = ["//tool/proto:__pkg__"], +) diff --git a/submitqueue/core/messagequeue/proto/batch.proto b/submitqueue/core/messagequeue/proto/batch.proto new file mode 100644 index 000000000..4a8e465e0 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/batch.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "BatchProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Batch is the payload mergeconflictsignal publishes to the batch stage: only the request id travels. batch reloads the Request from storage. +message Batch { + option (uber.base.messagequeue.topic_keys) = "batch"; + + // id is the request id to enroll in a batch. Format: "/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/cancel.proto b/submitqueue/core/messagequeue/proto/cancel.proto new file mode 100644 index 000000000..d264396b3 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/cancel.proto @@ -0,0 +1,38 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "CancelProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Cancel is the payload the gateway publishes to the cancel stage. +message Cancel { + option (uber.base.messagequeue.topic_keys) = "cancel"; + + // id is the request id to cancel. Format: "/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; + // reason is an optional free-form explanation of why cancellation was requested. + string reason = 3; +} diff --git a/submitqueue/core/messagequeue/proto/conclude.proto b/submitqueue/core/messagequeue/proto/conclude.proto new file mode 100644 index 000000000..6c0273e95 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/conclude.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "ConcludeProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Conclude is the payload published to the conclude stage: the id is a batch id. conclude reloads the Batch from storage. A failed batch's reason travels in message metadata, not this payload. +message Conclude { + option (uber.base.messagequeue.topic_keys) = "conclude"; + + // id is the batch id to conclude. Format: "/batch/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/dependencyanalysis.proto b/submitqueue/core/messagequeue/proto/dependencyanalysis.proto new file mode 100644 index 000000000..0ffc142bb --- /dev/null +++ b/submitqueue/core/messagequeue/proto/dependencyanalysis.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "DependencyAnalysisProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// DependencyAnalysis is the payload batch publishes to the dependency-analysis stage: only the batch id travels. The consumer reloads the Batch from storage. +message DependencyAnalysis { + option (uber.base.messagequeue.topic_keys) = "dependency-analysis"; + + // id is the batch id to analyze. Format: "/batch/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/log.proto b/submitqueue/core/messagequeue/proto/log.proto new file mode 100644 index 000000000..ccfaaf407 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/log.proto @@ -0,0 +1,52 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "LogProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Log is the payload orchestrator stages publish to the log stage: a full +// request-log entry. The gateway materializes it; the seam crosses services, so +// the entry travels in full. type, status, and event are open strings matching +// the domain vocabularies (not proto enums), so additive values do not break +// existing consumers. +message Log { + option (uber.base.messagequeue.topic_keys) = "log"; + + // request_id is the request this entry belongs to. + string request_id = 1; + // queue is the name of the queue processing the request. + string queue = 2; + // timestamp_ms is when this entry was created, milliseconds since Unix epoch. + int64 timestamp_ms = 3; + // type is "status" or "event". An empty type is treated as "status". + string type = 4; + // status is the request status this entry records. Set only when type is "status". + string status = 5; + // event is the event this entry records. Set only when type is "event". + string event = 6; + // request_version is the request entity version at this entry, or zero. + int32 request_version = 7; + // last_error is the last error message associated with the status, or empty. + string last_error = 8; + // metadata is free-form additional context. + map metadata = 9; +} diff --git a/submitqueue/core/messagequeue/proto/speculate.proto b/submitqueue/core/messagequeue/proto/speculate.proto new file mode 100644 index 000000000..d3345ac44 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/speculate.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "SpeculateProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Speculate is the payload published to the speculate stage: only the batch id travels. speculate reloads the Batch from storage. +message Speculate { + option (uber.base.messagequeue.topic_keys) = "speculate"; + + // id is the batch id to speculate. Format: "/batch/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/start.proto b/submitqueue/core/messagequeue/proto/start.proto new file mode 100644 index 000000000..ea2189bc8 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/start.proto @@ -0,0 +1,42 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/change/proto/change.proto"; +import "api/base/mergestrategy/proto/mergestrategy.proto"; +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "StartProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Start is the payload the gateway publishes to the start stage: the +// gateway-owned land inputs and the minted request id. start persists a Request +// from these fields; producer and consumer do not share a store at this seam. +message Start { + option (uber.base.messagequeue.topic_keys) = "start"; + + // id is the minted request id. Format: "/". + string id = 1; + // queue is the name of the queue processing the land request. + string queue = 2; + // change is the set of code changes to land. + uber.base.change.Change change = 3; + // land_strategy is the source-control integration strategy for every URI of change. + uber.base.mergestrategy.Strategy land_strategy = 4; +} diff --git a/submitqueue/core/messagequeue/proto/submitqueuebuild.proto b/submitqueue/core/messagequeue/proto/submitqueuebuild.proto new file mode 100644 index 000000000..f84d8cdb0 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/submitqueuebuild.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "SubmitqueueBuildProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Build is the payload speculate publishes to the build stage: the id is a batch id. build reloads the Batch from storage. +message Build { + option (uber.base.messagequeue.topic_keys) = "build"; + + // id is the batch id whose speculated head should be built. Format: "/batch/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/submitqueuebuildsignal.proto b/submitqueue/core/messagequeue/proto/submitqueuebuildsignal.proto new file mode 100644 index 000000000..79cb057b4 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/submitqueuebuildsignal.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "SubmitqueueBuildSignalProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// BuildSignal is the payload build publishes to the buildsignal stage: the id is a build id. buildsignal reloads the Build from storage. +message BuildSignal { + option (uber.base.messagequeue.topic_keys) = "buildsignal"; + + // id is the build id to poll. + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/submitqueuemerge.proto b/submitqueue/core/messagequeue/proto/submitqueuemerge.proto new file mode 100644 index 000000000..88e9f1909 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/submitqueuemerge.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "SubmitqueueMergeProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Merge is the payload speculate publishes to the internal land stage: the id is a batch id. land reloads the Batch from storage before handing work to Runway. +message Merge { + option (uber.base.messagequeue.topic_keys) = "submitqueue-land"; + + // id is the batch id to land. Format: "/batch/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/proto/validate.proto b/submitqueue/core/messagequeue/proto/validate.proto new file mode 100644 index 000000000..ec22dc229 --- /dev/null +++ b/submitqueue/core/messagequeue/proto/validate.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.submitqueue.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "ValidateProto"; +option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; + +// Validate is the payload start publishes to the validate stage: only the request id travels. validate reloads the Request from storage. +message Validate { + option (uber.base.messagequeue.topic_keys) = "validate"; + + // id is the request id to validate. Format: "/". + string id = 1; + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + string queue = 2; +} diff --git a/submitqueue/core/messagequeue/protopb/BUILD.bazel b/submitqueue/core/messagequeue/protopb/BUILD.bazel new file mode 100644 index 000000000..f724951c4 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "batch.pb.go", + "cancel.pb.go", + "conclude.pb.go", + "dependencyanalysis.pb.go", + "log.pb.go", + "speculate.pb.go", + "start.pb.go", + "submitqueuebuild.pb.go", + "submitqueuebuildsignal.pb.go", + "submitqueuemerge.pb.go", + "validate.pb.go", + ], + importpath = "github.com/uber/submitqueue/submitqueue/core/messagequeue/protopb", + visibility = [ + "//service/submitqueue:__subpackages__", + "//submitqueue:__subpackages__", + "//test:__subpackages__", + ], + deps = [ + "//api/base/change/protopb:go_default_library", + "//api/base/mergestrategy/protopb:go_default_library", + "//api/base/messagequeue/protopb:go_default_library", + "@org_golang_google_protobuf//reflect/protoreflect:go_default_library", + "@org_golang_google_protobuf//runtime/protoimpl:go_default_library", + ], +) diff --git a/submitqueue/core/messagequeue/protopb/batch.pb.go b/submitqueue/core/messagequeue/protopb/batch.pb.go new file mode 100644 index 000000000..a0cd2cd04 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/batch.pb.go @@ -0,0 +1,154 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: batch.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Batch is the payload mergeconflictsignal publishes to the batch stage: only the request id travels. batch reloads the Request from storage. +type Batch struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the request id to enroll in a batch. Format: "/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Batch) Reset() { + *x = Batch{} + mi := &file_batch_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Batch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Batch) ProtoMessage() {} + +func (x *Batch) ProtoReflect() protoreflect.Message { + mi := &file_batch_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Batch.ProtoReflect.Descriptor instead. +func (*Batch) Descriptor() ([]byte, []int) { + return file_batch_proto_rawDescGZIP(), []int{0} +} + +func (x *Batch) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Batch) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_batch_proto protoreflect.FileDescriptor + +const file_batch_proto_rawDesc = "" + + "\n" + + "\vbatch.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"8\n" + + "\x05Batch\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\t\x8a\xb5\x18\x05batchB\x80\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\n" + + "BatchProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_batch_proto_rawDescOnce sync.Once + file_batch_proto_rawDescData []byte +) + +func file_batch_proto_rawDescGZIP() []byte { + file_batch_proto_rawDescOnce.Do(func() { + file_batch_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_batch_proto_rawDesc), len(file_batch_proto_rawDesc))) + }) + return file_batch_proto_rawDescData +} + +var file_batch_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_batch_proto_goTypes = []any{ + (*Batch)(nil), // 0: uber.submitqueue.messagequeue.Batch +} +var file_batch_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_batch_proto_init() } +func file_batch_proto_init() { + if File_batch_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_batch_proto_rawDesc), len(file_batch_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_batch_proto_goTypes, + DependencyIndexes: file_batch_proto_depIdxs, + MessageInfos: file_batch_proto_msgTypes, + }.Build() + File_batch_proto = out.File + file_batch_proto_goTypes = nil + file_batch_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/cancel.pb.go b/submitqueue/core/messagequeue/protopb/cancel.pb.go new file mode 100644 index 000000000..2ea16bc01 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/cancel.pb.go @@ -0,0 +1,164 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: cancel.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Cancel is the payload the gateway publishes to the cancel stage. +type Cancel struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the request id to cancel. Format: "/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + // reason is an optional free-form explanation of why cancellation was requested. + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Cancel) Reset() { + *x = Cancel{} + mi := &file_cancel_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Cancel) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Cancel) ProtoMessage() {} + +func (x *Cancel) ProtoReflect() protoreflect.Message { + mi := &file_cancel_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Cancel.ProtoReflect.Descriptor instead. +func (*Cancel) Descriptor() ([]byte, []int) { + return file_cancel_proto_rawDescGZIP(), []int{0} +} + +func (x *Cancel) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Cancel) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +func (x *Cancel) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +var File_cancel_proto protoreflect.FileDescriptor + +const file_cancel_proto_rawDesc = "" + + "\n" + + "\fcancel.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"R\n" + + "\x06Cancel\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue\x12\x16\n" + + "\x06reason\x18\x03 \x01(\tR\x06reason:\n" + + "\x8a\xb5\x18\x06cancelB\x81\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\vCancelProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_cancel_proto_rawDescOnce sync.Once + file_cancel_proto_rawDescData []byte +) + +func file_cancel_proto_rawDescGZIP() []byte { + file_cancel_proto_rawDescOnce.Do(func() { + file_cancel_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_cancel_proto_rawDesc), len(file_cancel_proto_rawDesc))) + }) + return file_cancel_proto_rawDescData +} + +var file_cancel_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_cancel_proto_goTypes = []any{ + (*Cancel)(nil), // 0: uber.submitqueue.messagequeue.Cancel +} +var file_cancel_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_cancel_proto_init() } +func file_cancel_proto_init() { + if File_cancel_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_cancel_proto_rawDesc), len(file_cancel_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_cancel_proto_goTypes, + DependencyIndexes: file_cancel_proto_depIdxs, + MessageInfos: file_cancel_proto_msgTypes, + }.Build() + File_cancel_proto = out.File + file_cancel_proto_goTypes = nil + file_cancel_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/conclude.pb.go b/submitqueue/core/messagequeue/protopb/conclude.pb.go new file mode 100644 index 000000000..543b8e0a3 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/conclude.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: conclude.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Conclude is the payload published to the conclude stage: the id is a batch id. conclude reloads the Batch from storage. A failed batch's reason travels in message metadata, not this payload. +type Conclude struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the batch id to conclude. Format: "/batch/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Conclude) Reset() { + *x = Conclude{} + mi := &file_conclude_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Conclude) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Conclude) ProtoMessage() {} + +func (x *Conclude) ProtoReflect() protoreflect.Message { + mi := &file_conclude_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Conclude.ProtoReflect.Descriptor instead. +func (*Conclude) Descriptor() ([]byte, []int) { + return file_conclude_proto_rawDescGZIP(), []int{0} +} + +func (x *Conclude) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Conclude) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_conclude_proto protoreflect.FileDescriptor + +const file_conclude_proto_rawDesc = "" + + "\n" + + "\x0econclude.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\">\n" + + "\bConclude\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\f\x8a\xb5\x18\bconcludeB\x83\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\rConcludeProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_conclude_proto_rawDescOnce sync.Once + file_conclude_proto_rawDescData []byte +) + +func file_conclude_proto_rawDescGZIP() []byte { + file_conclude_proto_rawDescOnce.Do(func() { + file_conclude_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_conclude_proto_rawDesc), len(file_conclude_proto_rawDesc))) + }) + return file_conclude_proto_rawDescData +} + +var file_conclude_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_conclude_proto_goTypes = []any{ + (*Conclude)(nil), // 0: uber.submitqueue.messagequeue.Conclude +} +var file_conclude_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_conclude_proto_init() } +func file_conclude_proto_init() { + if File_conclude_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_conclude_proto_rawDesc), len(file_conclude_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_conclude_proto_goTypes, + DependencyIndexes: file_conclude_proto_depIdxs, + MessageInfos: file_conclude_proto_msgTypes, + }.Build() + File_conclude_proto = out.File + file_conclude_proto_goTypes = nil + file_conclude_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go b/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go new file mode 100644 index 000000000..687490932 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: dependencyanalysis.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// DependencyAnalysis is the payload batch publishes to the dependency-analysis stage: only the batch id travels. The consumer reloads the Batch from storage. +type DependencyAnalysis struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the batch id to analyze. Format: "/batch/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DependencyAnalysis) Reset() { + *x = DependencyAnalysis{} + mi := &file_dependencyanalysis_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DependencyAnalysis) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependencyAnalysis) ProtoMessage() {} + +func (x *DependencyAnalysis) ProtoReflect() protoreflect.Message { + mi := &file_dependencyanalysis_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependencyAnalysis.ProtoReflect.Descriptor instead. +func (*DependencyAnalysis) Descriptor() ([]byte, []int) { + return file_dependencyanalysis_proto_rawDescGZIP(), []int{0} +} + +func (x *DependencyAnalysis) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *DependencyAnalysis) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_dependencyanalysis_proto protoreflect.FileDescriptor + +const file_dependencyanalysis_proto_rawDesc = "" + + "\n" + + "\x18dependencyanalysis.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"S\n" + + "\x12DependencyAnalysis\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\x17\x8a\xb5\x18\x13dependency-analysisB\x8d\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\x17DependencyAnalysisProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_dependencyanalysis_proto_rawDescOnce sync.Once + file_dependencyanalysis_proto_rawDescData []byte +) + +func file_dependencyanalysis_proto_rawDescGZIP() []byte { + file_dependencyanalysis_proto_rawDescOnce.Do(func() { + file_dependencyanalysis_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_dependencyanalysis_proto_rawDesc), len(file_dependencyanalysis_proto_rawDesc))) + }) + return file_dependencyanalysis_proto_rawDescData +} + +var file_dependencyanalysis_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_dependencyanalysis_proto_goTypes = []any{ + (*DependencyAnalysis)(nil), // 0: uber.submitqueue.messagequeue.DependencyAnalysis +} +var file_dependencyanalysis_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_dependencyanalysis_proto_init() } +func file_dependencyanalysis_proto_init() { + if File_dependencyanalysis_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_dependencyanalysis_proto_rawDesc), len(file_dependencyanalysis_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_dependencyanalysis_proto_goTypes, + DependencyIndexes: file_dependencyanalysis_proto_depIdxs, + MessageInfos: file_dependencyanalysis_proto_msgTypes, + }.Build() + File_dependencyanalysis_proto = out.File + file_dependencyanalysis_proto_goTypes = nil + file_dependencyanalysis_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/log.pb.go b/submitqueue/core/messagequeue/protopb/log.pb.go new file mode 100644 index 000000000..ed89bd6a0 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/log.pb.go @@ -0,0 +1,232 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: log.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Log is the payload orchestrator stages publish to the log stage: a full +// request-log entry. The gateway materializes it; the seam crosses services, so +// the entry travels in full. type, status, and event are open strings matching +// the domain vocabularies (not proto enums), so additive values do not break +// existing consumers. +type Log struct { + state protoimpl.MessageState `protogen:"open.v1"` + // request_id is the request this entry belongs to. + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // queue is the name of the queue processing the request. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + // timestamp_ms is when this entry was created, milliseconds since Unix epoch. + TimestampMs int64 `protobuf:"varint,3,opt,name=timestamp_ms,json=timestampMs,proto3" json:"timestamp_ms,omitempty"` + // type is "status" or "event". An empty type is treated as "status". + Type string `protobuf:"bytes,4,opt,name=type,proto3" json:"type,omitempty"` + // status is the request status this entry records. Set only when type is "status". + Status string `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + // event is the event this entry records. Set only when type is "event". + Event string `protobuf:"bytes,6,opt,name=event,proto3" json:"event,omitempty"` + // request_version is the request entity version at this entry, or zero. + RequestVersion int32 `protobuf:"varint,7,opt,name=request_version,json=requestVersion,proto3" json:"request_version,omitempty"` + // last_error is the last error message associated with the status, or empty. + LastError string `protobuf:"bytes,8,opt,name=last_error,json=lastError,proto3" json:"last_error,omitempty"` + // metadata is free-form additional context. + Metadata map[string]string `protobuf:"bytes,9,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Log) Reset() { + *x = Log{} + mi := &file_log_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Log) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Log) ProtoMessage() {} + +func (x *Log) ProtoReflect() protoreflect.Message { + mi := &file_log_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Log.ProtoReflect.Descriptor instead. +func (*Log) Descriptor() ([]byte, []int) { + return file_log_proto_rawDescGZIP(), []int{0} +} + +func (x *Log) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (x *Log) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +func (x *Log) GetTimestampMs() int64 { + if x != nil { + return x.TimestampMs + } + return 0 +} + +func (x *Log) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *Log) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *Log) GetEvent() string { + if x != nil { + return x.Event + } + return "" +} + +func (x *Log) GetRequestVersion() int32 { + if x != nil { + return x.RequestVersion + } + return 0 +} + +func (x *Log) GetLastError() string { + if x != nil { + return x.LastError + } + return "" +} + +func (x *Log) GetMetadata() map[string]string { + if x != nil { + return x.Metadata + } + return nil +} + +var File_log_proto protoreflect.FileDescriptor + +const file_log_proto_rawDesc = "" + + "\n" + + "\tlog.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"\xfb\x02\n" + + "\x03Log\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue\x12!\n" + + "\ftimestamp_ms\x18\x03 \x01(\x03R\vtimestampMs\x12\x12\n" + + "\x04type\x18\x04 \x01(\tR\x04type\x12\x16\n" + + "\x06status\x18\x05 \x01(\tR\x06status\x12\x14\n" + + "\x05event\x18\x06 \x01(\tR\x05event\x12'\n" + + "\x0frequest_version\x18\a \x01(\x05R\x0erequestVersion\x12\x1d\n" + + "\n" + + "last_error\x18\b \x01(\tR\tlastError\x12L\n" + + "\bmetadata\x18\t \x03(\v20.uber.submitqueue.messagequeue.Log.MetadataEntryR\bmetadata\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01:\a\x8a\xb5\x18\x03logB~\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\bLogProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_log_proto_rawDescOnce sync.Once + file_log_proto_rawDescData []byte +) + +func file_log_proto_rawDescGZIP() []byte { + file_log_proto_rawDescOnce.Do(func() { + file_log_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_log_proto_rawDesc), len(file_log_proto_rawDesc))) + }) + return file_log_proto_rawDescData +} + +var file_log_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_log_proto_goTypes = []any{ + (*Log)(nil), // 0: uber.submitqueue.messagequeue.Log + nil, // 1: uber.submitqueue.messagequeue.Log.MetadataEntry +} +var file_log_proto_depIdxs = []int32{ + 1, // 0: uber.submitqueue.messagequeue.Log.metadata:type_name -> uber.submitqueue.messagequeue.Log.MetadataEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_log_proto_init() } +func file_log_proto_init() { + if File_log_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_log_proto_rawDesc), len(file_log_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_log_proto_goTypes, + DependencyIndexes: file_log_proto_depIdxs, + MessageInfos: file_log_proto_msgTypes, + }.Build() + File_log_proto = out.File + file_log_proto_goTypes = nil + file_log_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/speculate.pb.go b/submitqueue/core/messagequeue/protopb/speculate.pb.go new file mode 100644 index 000000000..53c6407ef --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/speculate.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: speculate.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Speculate is the payload published to the speculate stage: only the batch id travels. speculate reloads the Batch from storage. +type Speculate struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the batch id to speculate. Format: "/batch/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Speculate) Reset() { + *x = Speculate{} + mi := &file_speculate_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Speculate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Speculate) ProtoMessage() {} + +func (x *Speculate) ProtoReflect() protoreflect.Message { + mi := &file_speculate_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Speculate.ProtoReflect.Descriptor instead. +func (*Speculate) Descriptor() ([]byte, []int) { + return file_speculate_proto_rawDescGZIP(), []int{0} +} + +func (x *Speculate) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Speculate) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_speculate_proto protoreflect.FileDescriptor + +const file_speculate_proto_rawDesc = "" + + "\n" + + "\x0fspeculate.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"@\n" + + "\tSpeculate\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\r\x8a\xb5\x18\tspeculateB\x84\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\x0eSpeculateProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_speculate_proto_rawDescOnce sync.Once + file_speculate_proto_rawDescData []byte +) + +func file_speculate_proto_rawDescGZIP() []byte { + file_speculate_proto_rawDescOnce.Do(func() { + file_speculate_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_speculate_proto_rawDesc), len(file_speculate_proto_rawDesc))) + }) + return file_speculate_proto_rawDescData +} + +var file_speculate_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_speculate_proto_goTypes = []any{ + (*Speculate)(nil), // 0: uber.submitqueue.messagequeue.Speculate +} +var file_speculate_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_speculate_proto_init() } +func file_speculate_proto_init() { + if File_speculate_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_speculate_proto_rawDesc), len(file_speculate_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_speculate_proto_goTypes, + DependencyIndexes: file_speculate_proto_depIdxs, + MessageInfos: file_speculate_proto_msgTypes, + }.Build() + File_speculate_proto = out.File + file_speculate_proto_goTypes = nil + file_speculate_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/start.pb.go b/submitqueue/core/messagequeue/protopb/start.pb.go new file mode 100644 index 000000000..343d9a6ac --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/start.pb.go @@ -0,0 +1,180 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: start.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protopb "github.com/uber/submitqueue/api/base/change/protopb" + protopb1 "github.com/uber/submitqueue/api/base/mergestrategy/protopb" + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Start is the payload the gateway publishes to the start stage: the +// gateway-owned land inputs and the minted request id. start persists a Request +// from these fields; producer and consumer do not share a store at this seam. +type Start struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the minted request id. Format: "/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing the land request. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + // change is the set of code changes to land. + Change *protopb.Change `protobuf:"bytes,3,opt,name=change,proto3" json:"change,omitempty"` + // land_strategy is the source-control integration strategy for every URI of change. + LandStrategy protopb1.Strategy `protobuf:"varint,4,opt,name=land_strategy,json=landStrategy,proto3,enum=uber.base.mergestrategy.Strategy" json:"land_strategy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Start) Reset() { + *x = Start{} + mi := &file_start_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Start) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Start) ProtoMessage() {} + +func (x *Start) ProtoReflect() protoreflect.Message { + mi := &file_start_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Start.ProtoReflect.Descriptor instead. +func (*Start) Descriptor() ([]byte, []int) { + return file_start_proto_rawDescGZIP(), []int{0} +} + +func (x *Start) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Start) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +func (x *Start) GetChange() *protopb.Change { + if x != nil { + return x.Change + } + return nil +} + +func (x *Start) GetLandStrategy() protopb1.Strategy { + if x != nil { + return x.LandStrategy + } + return protopb1.Strategy(0) +} + +var File_start_proto protoreflect.FileDescriptor + +const file_start_proto_rawDesc = "" + + "\n" + + "\vstart.proto\x12\x1duber.submitqueue.messagequeue\x1a\"api/base/change/proto/change.proto\x1a0api/base/mergestrategy/proto/mergestrategy.proto\x1a.api/base/messagequeue/proto/messagequeue.proto\"\xb2\x01\n" + + "\x05Start\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue\x120\n" + + "\x06change\x18\x03 \x01(\v2\x18.uber.base.change.ChangeR\x06change\x12F\n" + + "\rland_strategy\x18\x04 \x01(\x0e2!.uber.base.mergestrategy.StrategyR\flandStrategy:\t\x8a\xb5\x18\x05startB\x80\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\n" + + "StartProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_start_proto_rawDescOnce sync.Once + file_start_proto_rawDescData []byte +) + +func file_start_proto_rawDescGZIP() []byte { + file_start_proto_rawDescOnce.Do(func() { + file_start_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_start_proto_rawDesc), len(file_start_proto_rawDesc))) + }) + return file_start_proto_rawDescData +} + +var file_start_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_start_proto_goTypes = []any{ + (*Start)(nil), // 0: uber.submitqueue.messagequeue.Start + (*protopb.Change)(nil), // 1: uber.base.change.Change + (protopb1.Strategy)(0), // 2: uber.base.mergestrategy.Strategy +} +var file_start_proto_depIdxs = []int32{ + 1, // 0: uber.submitqueue.messagequeue.Start.change:type_name -> uber.base.change.Change + 2, // 1: uber.submitqueue.messagequeue.Start.land_strategy:type_name -> uber.base.mergestrategy.Strategy + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_start_proto_init() } +func file_start_proto_init() { + if File_start_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_start_proto_rawDesc), len(file_start_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_start_proto_goTypes, + DependencyIndexes: file_start_proto_depIdxs, + MessageInfos: file_start_proto_msgTypes, + }.Build() + File_start_proto = out.File + file_start_proto_goTypes = nil + file_start_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go b/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go new file mode 100644 index 000000000..c6ba889b3 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: submitqueuebuild.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Build is the payload speculate publishes to the build stage: the id is a batch id. build reloads the Batch from storage. +type Build struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the batch id whose speculated head should be built. Format: "/batch/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Build) Reset() { + *x = Build{} + mi := &file_submitqueuebuild_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Build) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Build) ProtoMessage() {} + +func (x *Build) ProtoReflect() protoreflect.Message { + mi := &file_submitqueuebuild_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Build.ProtoReflect.Descriptor instead. +func (*Build) Descriptor() ([]byte, []int) { + return file_submitqueuebuild_proto_rawDescGZIP(), []int{0} +} + +func (x *Build) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Build) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_submitqueuebuild_proto protoreflect.FileDescriptor + +const file_submitqueuebuild_proto_rawDesc = "" + + "\n" + + "\x16submitqueuebuild.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"8\n" + + "\x05Build\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\t\x8a\xb5\x18\x05buildB\x8b\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\x15SubmitqueueBuildProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_submitqueuebuild_proto_rawDescOnce sync.Once + file_submitqueuebuild_proto_rawDescData []byte +) + +func file_submitqueuebuild_proto_rawDescGZIP() []byte { + file_submitqueuebuild_proto_rawDescOnce.Do(func() { + file_submitqueuebuild_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_submitqueuebuild_proto_rawDesc), len(file_submitqueuebuild_proto_rawDesc))) + }) + return file_submitqueuebuild_proto_rawDescData +} + +var file_submitqueuebuild_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_submitqueuebuild_proto_goTypes = []any{ + (*Build)(nil), // 0: uber.submitqueue.messagequeue.Build +} +var file_submitqueuebuild_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_submitqueuebuild_proto_init() } +func file_submitqueuebuild_proto_init() { + if File_submitqueuebuild_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_submitqueuebuild_proto_rawDesc), len(file_submitqueuebuild_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_submitqueuebuild_proto_goTypes, + DependencyIndexes: file_submitqueuebuild_proto_depIdxs, + MessageInfos: file_submitqueuebuild_proto_msgTypes, + }.Build() + File_submitqueuebuild_proto = out.File + file_submitqueuebuild_proto_goTypes = nil + file_submitqueuebuild_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/submitqueuebuildsignal.pb.go b/submitqueue/core/messagequeue/protopb/submitqueuebuildsignal.pb.go new file mode 100644 index 000000000..4d62239b0 --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/submitqueuebuildsignal.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: submitqueuebuildsignal.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// BuildSignal is the payload build publishes to the buildsignal stage: the id is a build id. buildsignal reloads the Build from storage. +type BuildSignal struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the build id to poll. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BuildSignal) Reset() { + *x = BuildSignal{} + mi := &file_submitqueuebuildsignal_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BuildSignal) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BuildSignal) ProtoMessage() {} + +func (x *BuildSignal) ProtoReflect() protoreflect.Message { + mi := &file_submitqueuebuildsignal_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BuildSignal.ProtoReflect.Descriptor instead. +func (*BuildSignal) Descriptor() ([]byte, []int) { + return file_submitqueuebuildsignal_proto_rawDescGZIP(), []int{0} +} + +func (x *BuildSignal) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *BuildSignal) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_submitqueuebuildsignal_proto protoreflect.FileDescriptor + +const file_submitqueuebuildsignal_proto_rawDesc = "" + + "\n" + + "\x1csubmitqueuebuildsignal.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"D\n" + + "\vBuildSignal\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\x0f\x8a\xb5\x18\vbuildsignalB\x91\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\x1bSubmitqueueBuildSignalProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_submitqueuebuildsignal_proto_rawDescOnce sync.Once + file_submitqueuebuildsignal_proto_rawDescData []byte +) + +func file_submitqueuebuildsignal_proto_rawDescGZIP() []byte { + file_submitqueuebuildsignal_proto_rawDescOnce.Do(func() { + file_submitqueuebuildsignal_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_submitqueuebuildsignal_proto_rawDesc), len(file_submitqueuebuildsignal_proto_rawDesc))) + }) + return file_submitqueuebuildsignal_proto_rawDescData +} + +var file_submitqueuebuildsignal_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_submitqueuebuildsignal_proto_goTypes = []any{ + (*BuildSignal)(nil), // 0: uber.submitqueue.messagequeue.BuildSignal +} +var file_submitqueuebuildsignal_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_submitqueuebuildsignal_proto_init() } +func file_submitqueuebuildsignal_proto_init() { + if File_submitqueuebuildsignal_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_submitqueuebuildsignal_proto_rawDesc), len(file_submitqueuebuildsignal_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_submitqueuebuildsignal_proto_goTypes, + DependencyIndexes: file_submitqueuebuildsignal_proto_depIdxs, + MessageInfos: file_submitqueuebuildsignal_proto_msgTypes, + }.Build() + File_submitqueuebuildsignal_proto = out.File + file_submitqueuebuildsignal_proto_goTypes = nil + file_submitqueuebuildsignal_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go b/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go new file mode 100644 index 000000000..8b6119e4b --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: submitqueuemerge.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Merge is the payload speculate publishes to the internal land stage: the id is a batch id. land reloads the Batch from storage before handing work to Runway. +type Merge struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the batch id to land. Format: "/batch/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Merge) Reset() { + *x = Merge{} + mi := &file_submitqueuemerge_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Merge) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Merge) ProtoMessage() {} + +func (x *Merge) ProtoReflect() protoreflect.Message { + mi := &file_submitqueuemerge_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Merge.ProtoReflect.Descriptor instead. +func (*Merge) Descriptor() ([]byte, []int) { + return file_submitqueuemerge_proto_rawDescGZIP(), []int{0} +} + +func (x *Merge) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Merge) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_submitqueuemerge_proto protoreflect.FileDescriptor + +const file_submitqueuemerge_proto_rawDesc = "" + + "\n" + + "\x16submitqueuemerge.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"C\n" + + "\x05Merge\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\x14\x8a\xb5\x18\x10submitqueue-landB\x8b\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\x15SubmitqueueMergeProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_submitqueuemerge_proto_rawDescOnce sync.Once + file_submitqueuemerge_proto_rawDescData []byte +) + +func file_submitqueuemerge_proto_rawDescGZIP() []byte { + file_submitqueuemerge_proto_rawDescOnce.Do(func() { + file_submitqueuemerge_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_submitqueuemerge_proto_rawDesc), len(file_submitqueuemerge_proto_rawDesc))) + }) + return file_submitqueuemerge_proto_rawDescData +} + +var file_submitqueuemerge_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_submitqueuemerge_proto_goTypes = []any{ + (*Merge)(nil), // 0: uber.submitqueue.messagequeue.Merge +} +var file_submitqueuemerge_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_submitqueuemerge_proto_init() } +func file_submitqueuemerge_proto_init() { + if File_submitqueuemerge_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_submitqueuemerge_proto_rawDesc), len(file_submitqueuemerge_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_submitqueuemerge_proto_goTypes, + DependencyIndexes: file_submitqueuemerge_proto_depIdxs, + MessageInfos: file_submitqueuemerge_proto_msgTypes, + }.Build() + File_submitqueuemerge_proto = out.File + file_submitqueuemerge_proto_goTypes = nil + file_submitqueuemerge_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/protopb/validate.pb.go b/submitqueue/core/messagequeue/protopb/validate.pb.go new file mode 100644 index 000000000..d3ec8ec1b --- /dev/null +++ b/submitqueue/core/messagequeue/protopb/validate.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc v5.29.3 +// source: validate.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Validate is the payload start publishes to the validate stage: only the request id travels. validate reloads the Request from storage. +type Validate struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the request id to validate. Format: "/". + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue is the name of the queue processing this work, carried so the + // consumer can route by queue without loading state first. Empty on + // payloads written before the field existed. + Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Validate) Reset() { + *x = Validate{} + mi := &file_validate_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Validate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Validate) ProtoMessage() {} + +func (x *Validate) ProtoReflect() protoreflect.Message { + mi := &file_validate_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Validate.ProtoReflect.Descriptor instead. +func (*Validate) Descriptor() ([]byte, []int) { + return file_validate_proto_rawDescGZIP(), []int{0} +} + +func (x *Validate) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Validate) GetQueue() string { + if x != nil { + return x.Queue + } + return "" +} + +var File_validate_proto protoreflect.FileDescriptor + +const file_validate_proto_rawDesc = "" + + "\n" + + "\x0evalidate.proto\x12\x1duber.submitqueue.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\">\n" + + "\bValidate\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n" + + "\x05queue\x18\x02 \x01(\tR\x05queue:\f\x8a\xb5\x18\bvalidateB\x83\x01\n" + + "-com.uber.submitqueue.submitqueue.messagequeueB\rValidateProtoP\x01ZAgithub.com/uber/submitqueue/submitqueue/core/messagequeue/protopbb\x06proto3" + +var ( + file_validate_proto_rawDescOnce sync.Once + file_validate_proto_rawDescData []byte +) + +func file_validate_proto_rawDescGZIP() []byte { + file_validate_proto_rawDescOnce.Do(func() { + file_validate_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_validate_proto_rawDesc), len(file_validate_proto_rawDesc))) + }) + return file_validate_proto_rawDescData +} + +var file_validate_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_validate_proto_goTypes = []any{ + (*Validate)(nil), // 0: uber.submitqueue.messagequeue.Validate +} +var file_validate_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_validate_proto_init() } +func file_validate_proto_init() { + if File_validate_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_validate_proto_rawDesc), len(file_validate_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_validate_proto_goTypes, + DependencyIndexes: file_validate_proto_depIdxs, + MessageInfos: file_validate_proto_msgTypes, + }.Build() + File_validate_proto = out.File + file_validate_proto_goTypes = nil + file_validate_proto_depIdxs = nil +} diff --git a/submitqueue/core/messagequeue/topics.go b/submitqueue/core/messagequeue/topics.go new file mode 100644 index 000000000..a2bb0e5e6 --- /dev/null +++ b/submitqueue/core/messagequeue/topics.go @@ -0,0 +1,61 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messagequeue + +import "github.com/uber/submitqueue/platform/consumer" + +// TopicKey is the typed identifier used to look up a queue backend, topic name, +// and subscription config in a consumer.TopicRegistry. The constants below are +// the logical topic keys for SubmitQueue's internal pipeline stages; they are +// the same strings each message lists in its topic_keys option. +type TopicKey = consumer.TopicKey + +const ( + // TopicKeyStart carries new land requests from the gateway to start. + TopicKeyStart TopicKey = "start" + // TopicKeyCancel carries cancellation requests from the gateway to cancel. + TopicKeyCancel TopicKey = "cancel" + // TopicKeyValidate carries request ids from start to validate. + TopicKeyValidate TopicKey = "validate" + // TopicKeyBatch carries request ids from landconflictsignal to batch. + TopicKeyBatch TopicKey = "batch" + // TopicKeyDependencyAnalysis carries newly created batch ids for conflict + // analysis. Messages must be partitioned by queue: analysis reads the + // queue's dependency-eligible batches, so two batches of one queue analyzed + // concurrently would each miss the other. + TopicKeyDependencyAnalysis TopicKey = "dependency-analysis" + // TopicKeySpeculate carries batch ids for speculation. + TopicKeySpeculate TopicKey = "speculate" + // TopicKeyBuild carries batch ids whose speculated heads should be built. + TopicKeyBuild TopicKey = "build" + // TopicKeyBuildSignal carries build ids to poll. The consumer calls + // BuildRunner.Status, persists the latest status, publishes the batch id to + // TopicKeySpeculate so the state machine re-evaluates, and holds the + // delivery for the next poll when the build has not yet reached a terminal + // state. + TopicKeyBuildSignal TopicKey = "buildsignal" + // TopicKeyLand carries batch ids to the internal land stage before Runway. + TopicKeyLand TopicKey = "submitqueue-land" + // TopicKeyConclude carries batch ids for terminal request reconciliation. + TopicKeyConclude TopicKey = "conclude" + // TopicKeyLog carries per-request log entries from the orchestrator to the gateway. + TopicKeyLog TopicKey = "log" +) + +// MetadataKeyFailureReason is the conclude message's metadata attribute carrying +// a failed batch's human-readable reason. Set by the failure sites (land and +// speculate) on the conclude publish and read by conclude to stamp the request's +// terminal log; absent on the landed and cancelled paths. +const MetadataKeyFailureReason = "failure_reason" diff --git a/submitqueue/core/request/BUILD.bazel b/submitqueue/core/request/BUILD.bazel index 692184c4a..1d5db35e0 100644 --- a/submitqueue/core/request/BUILD.bazel +++ b/submitqueue/core/request/BUILD.bazel @@ -13,6 +13,7 @@ go_library( deps = [ "//platform/consumer:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", @@ -32,6 +33,7 @@ go_test( "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/core/request/log.go b/submitqueue/core/request/log.go index 94d606719..516612a75 100644 --- a/submitqueue/core/request/log.go +++ b/submitqueue/core/request/log.go @@ -20,6 +20,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" ) @@ -42,7 +43,7 @@ import ( // the build ID, or the batch and path the entry is about. Deriving it from the // wall clock or a random value would defeat the dedupe entirely. func PublishLog(ctx context.Context, registry consumer.TopicRegistry, logEntry entity.RequestLog, partitionKey string, occurrence string) error { - payload, err := logEntry.ToBytes() + payload, err := sqmq.Marshal(sqmq.LogFromEntity(logEntry)) if err != nil { return fmt.Errorf("failed to serialize request log: %w", err) } diff --git a/submitqueue/core/request/materializer_test.go b/submitqueue/core/request/materializer_test.go index 870db1fb3..ad51a4107 100644 --- a/submitqueue/core/request/materializer_test.go +++ b/submitqueue/core/request/materializer_test.go @@ -300,9 +300,8 @@ func TestLogWins(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Cases above leave Type unset when the entry is a status, so the - // table reads as being about the ordering rules under test. This - // mirrors production exactly: logWins only ever sees entries that - // came through RequestLogFromBytes, which applies the same default. + // table reads as being about the ordering rules under test. An + // untyped entry is a status, matching UnmarshalRequestLog. incoming := tt.incoming if incoming.Type == "" { incoming.Type = entity.RequestLogTypeStatus diff --git a/submitqueue/core/request/terminate_test.go b/submitqueue/core/request/terminate_test.go index fac4cfc09..63c6bd756 100644 --- a/submitqueue/core/request/terminate_test.go +++ b/submitqueue/core/request/terminate_test.go @@ -24,6 +24,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -45,7 +46,7 @@ func recordingRegistry(t *testing.T, ctrl *gomock.Controller, publishErr error) mockPub := queuemock.NewMockPublisher(ctrl) mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - log, err := entity.RequestLogFromBytes(msg.Payload) + log, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) *logs = append(*logs, log) return publishErr diff --git a/submitqueue/core/topickey/BUILD.bazel b/submitqueue/core/topickey/BUILD.bazel index 61271e9c0..a28b35fef 100644 --- a/submitqueue/core/topickey/BUILD.bazel +++ b/submitqueue/core/topickey/BUILD.bazel @@ -5,5 +5,5 @@ go_library( srcs = ["topickey.go"], importpath = "github.com/uber/submitqueue/submitqueue/core/topickey", visibility = ["//visibility:public"], - deps = ["//platform/consumer:go_default_library"], + deps = ["//submitqueue/core/messagequeue:go_default_library"], ) diff --git a/submitqueue/core/topickey/topickey.go b/submitqueue/core/topickey/topickey.go index 6bdae00f3..e7fbf92a4 100644 --- a/submitqueue/core/topickey/topickey.go +++ b/submitqueue/core/topickey/topickey.go @@ -12,48 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package topickey defines SubmitQueue pipeline stage identifiers. +// Package topickey re-exports SubmitQueue pipeline stage identifiers from +// submitqueue/core/messagequeue, the package that owns the topic-key constants +// and the payloads bound to them. package topickey -import "github.com/uber/submitqueue/platform/consumer" +import "github.com/uber/submitqueue/submitqueue/core/messagequeue" // TopicKey is the shared pipeline stage identifier type. -type TopicKey = consumer.TopicKey +type TopicKey = messagequeue.TopicKey const ( - // TopicKeyStart is the pipeline stage where new requests arrive from the gateway. - TopicKeyStart TopicKey = "start" - // TopicKeyCancel is the pipeline stage where cancellation requests arrive from the gateway. - TopicKeyCancel TopicKey = "cancel" - // TopicKeyValidate is the pipeline stage where requests are published for validation. - TopicKeyValidate TopicKey = "validate" - // TopicKeyBatch is the pipeline stage where validated requests are published for batching. - TopicKeyBatch TopicKey = "batch" - // TopicKeyDependencyAnalysis is the pipeline stage where newly created batches are - // published for conflict analysis. Messages must be partitioned by queue: - // analysis reads the queue's dependency-eligible batches, so two batches of - // one queue analyzed concurrently would each miss the other. - TopicKeyDependencyAnalysis TopicKey = "dependency-analysis" - // TopicKeySpeculate is the pipeline stage where batches are published for speculation. - TopicKeySpeculate TopicKey = "speculate" - // TopicKeyBuild is the pipeline stage where speculated batches are published for builds. - TopicKeyBuild TopicKey = "build" - // TopicKeyBuildSignal is the polling stage for triggered builds. Each - // message carries a Build; the consumer calls BuildRunner.Status, - // persists the latest status, publishes the batch ID to TopicKeySpeculate - // so the state machine re-evaluates, and holds the delivery for the next - // poll when the build has not yet reached a terminal state. - TopicKeyBuildSignal TopicKey = "buildsignal" - // TopicKeyLand is the pipeline stage where speculated batches are published for landing. - TopicKeyLand TopicKey = "submitqueue-land" - // TopicKeyConclude is the pipeline stage where landed requests are published for conclusion. - TopicKeyConclude TopicKey = "conclude" - // TopicKeyLog is the pipeline stage where per-request logs are written. - TopicKeyLog TopicKey = "log" + // TopicKeyStart carries new land requests from the gateway to start. + TopicKeyStart = messagequeue.TopicKeyStart + // TopicKeyCancel carries cancellation requests from the gateway to cancel. + TopicKeyCancel = messagequeue.TopicKeyCancel + // TopicKeyValidate carries request ids from start to validate. + TopicKeyValidate = messagequeue.TopicKeyValidate + // TopicKeyBatch carries request ids from landconflictsignal to batch. + TopicKeyBatch = messagequeue.TopicKeyBatch + // TopicKeyDependencyAnalysis carries newly created batch ids for conflict analysis. + TopicKeyDependencyAnalysis = messagequeue.TopicKeyDependencyAnalysis + // TopicKeySpeculate carries batch ids for speculation. + TopicKeySpeculate = messagequeue.TopicKeySpeculate + // TopicKeyBuild carries batch ids whose speculated heads should be built. + TopicKeyBuild = messagequeue.TopicKeyBuild + // TopicKeyBuildSignal carries build ids to poll. + TopicKeyBuildSignal = messagequeue.TopicKeyBuildSignal + // TopicKeyLand carries batch ids to the internal land stage before Runway. + TopicKeyLand = messagequeue.TopicKeyLand + // TopicKeyConclude carries batch ids for terminal request reconciliation. + TopicKeyConclude = messagequeue.TopicKeyConclude + // TopicKeyLog carries per-request log entries from the orchestrator to the gateway. + TopicKeyLog = messagequeue.TopicKeyLog + // MetadataKeyFailureReason is the conclude message metadata attribute for a failed batch's reason. + MetadataKeyFailureReason = messagequeue.MetadataKeyFailureReason ) - -// MetadataKeyFailureReason is the conclude message's metadata attribute carrying -// a failed batch's human-readable reason. Set by the failure sites (land and -// speculate) on the conclude publish and read by conclude to stamp the request's -// terminal log; absent on the landed and cancelled paths. -const MetadataKeyFailureReason = "failure_reason" diff --git a/submitqueue/entity/BUILD.bazel b/submitqueue/entity/BUILD.bazel index 419a98a61..6d1177f33 100644 --- a/submitqueue/entity/BUILD.bazel +++ b/submitqueue/entity/BUILD.bazel @@ -38,8 +38,6 @@ go_test( srcs = [ "batch_test.go", "build_test.go", - "cancel_test.go", - "land_test.go", "request_log_test.go", "request_test.go", "speculation_test.go", diff --git a/submitqueue/entity/batch.go b/submitqueue/entity/batch.go index 25f9d4af8..9a53eee59 100644 --- a/submitqueue/entity/batch.go +++ b/submitqueue/entity/batch.go @@ -194,15 +194,3 @@ type BatchID struct { // Queue is the name of the queue processing the batch. Empty on payloads written before the field existed. Queue string `json:"queue"` } - -// ToBytes serializes the BatchID to JSON bytes for queue message payload. -func (b BatchID) ToBytes() ([]byte, error) { - return json.Marshal(b) -} - -// BatchIDFromBytes deserializes a BatchID from JSON bytes. -func BatchIDFromBytes(data []byte) (BatchID, error) { - var bid BatchID - err := json.Unmarshal(data, &bid) - return bid, err -} diff --git a/submitqueue/entity/build.go b/submitqueue/entity/build.go index 0793faa22..0cb95577a 100644 --- a/submitqueue/entity/build.go +++ b/submitqueue/entity/build.go @@ -98,18 +98,6 @@ type BuildID struct { Queue string `json:"queue"` } -// ToBytes serializes the BuildID to JSON bytes for queue message payload. -func (b BuildID) ToBytes() ([]byte, error) { - return json.Marshal(b) -} - -// BuildIDFromBytes deserializes a BuildID from JSON bytes. -func BuildIDFromBytes(data []byte) (BuildID, error) { - var bid BuildID - err := json.Unmarshal(data, &bid) - return bid, err -} - // BuildMetadata carries provider-defined free-form metadata about a build // (e.g. build URL, duration, commit SHA). Keys and values are // implementation-defined; callers should not assume any particular schema. diff --git a/submitqueue/entity/cancel.go b/submitqueue/entity/cancel.go index 3c87c1964..e7ccd762d 100644 --- a/submitqueue/entity/cancel.go +++ b/submitqueue/entity/cancel.go @@ -14,8 +14,6 @@ package entity -import "encoding/json" - // CancelRequest represents a cancellation request sent over the queue from the gateway to the orchestrator. // It identifies the request to cancel by its ID and carries an optional human-readable reason for observability. type CancelRequest struct { @@ -26,15 +24,3 @@ type CancelRequest struct { // Reason is an optional free-form explanation of why the cancellation was requested. Reason string `json:"reason"` } - -// ToBytes serializes the CancelRequest to JSON bytes for queue message payload. -func (r CancelRequest) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// CancelRequestFromBytes deserializes a CancelRequest from JSON bytes. -func CancelRequestFromBytes(data []byte) (CancelRequest, error) { - var req CancelRequest - err := json.Unmarshal(data, &req) - return req, err -} diff --git a/submitqueue/entity/cancel_test.go b/submitqueue/entity/cancel_test.go deleted file mode 100644 index b82b5282e..000000000 --- a/submitqueue/entity/cancel_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCancelRequest_SerializationRoundTrip(t *testing.T) { - tests := []struct { - name string - req CancelRequest - }{ - { - name: "with reason", - req: CancelRequest{ID: "queue1/100", Reason: "obsolete change"}, - }, - { - name: "without reason", - req: CancelRequest{ID: "queue2/200"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err := tt.req.ToBytes() - require.NoError(t, err) - - deserialized, err := CancelRequestFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, tt.req, deserialized) - }) - } -} - -func TestCancelRequestFromBytes_InvalidJSON(t *testing.T) { - _, err := CancelRequestFromBytes([]byte(`{not json`)) - assert.Error(t, err) -} diff --git a/submitqueue/entity/land.go b/submitqueue/entity/land.go index edeaa60af..f579de1c5 100644 --- a/submitqueue/entity/land.go +++ b/submitqueue/entity/land.go @@ -15,8 +15,6 @@ package entity import ( - "encoding/json" - "github.com/uber/submitqueue/platform/base/change" "github.com/uber/submitqueue/platform/base/mergestrategy" ) @@ -36,18 +34,6 @@ type LandRequest struct { LandStrategy mergestrategy.MergeStrategy `json:"land_strategy"` } -// ToBytes serializes the LandRequest to JSON bytes for queue message payload. -func (r LandRequest) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// LandRequestFromBytes deserializes a LandRequest from JSON bytes. -func LandRequestFromBytes(data []byte) (LandRequest, error) { - var req LandRequest - err := json.Unmarshal(data, &req) - return req, err -} - // LandResult is the outcome of accepting a land request. It carries the ID the // controller assigned to the request so the transport layer can echo it back to // the caller. diff --git a/submitqueue/entity/land_test.go b/submitqueue/entity/land_test.go deleted file mode 100644 index 8820aea66..000000000 --- a/submitqueue/entity/land_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2025 Uber Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package entity - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/uber/submitqueue/platform/base/change" - "github.com/uber/submitqueue/platform/base/mergestrategy" -) - -func TestLandRequest_ToBytes(t *testing.T) { - req := LandRequest{ - ID: "test-queue/123", - Queue: "test-queue", - Change: change.Change{URIs: []string{ - "github://github.example.com/uber/submitqueue/pull/456/abcdef0123456789abcdef0123456789abcdef01", - "github://github.example.com/uber/submitqueue/pull/789/0123456789abcdef0123456789abcdef01234567", - }}, - LandStrategy: mergestrategy.MergeStrategyRebase, - } - - data, err := req.ToBytes() - require.NoError(t, err) - assert.NotEmpty(t, data) - - // Verify JSON contains expected fields - jsonStr := string(data) - assert.Contains(t, jsonStr, "test-queue/123") - assert.Contains(t, jsonStr, "github://github.example.com/uber/submitqueue/pull/456/abcdef0123456789abcdef0123456789abcdef01") - assert.Contains(t, jsonStr, "rebase") -} - -func TestLandRequestFromBytes(t *testing.T) { - original := LandRequest{ - ID: "my-queue/999", - Queue: "my-queue", - Change: change.Change{URIs: []string{"code.uber.internal.com/D111"}}, - LandStrategy: mergestrategy.MergeStrategyMerge, - } - - // Serialize - data, err := original.ToBytes() - require.NoError(t, err) - - // Deserialize - deserialized, err := LandRequestFromBytes(data) - require.NoError(t, err) - - // Verify all fields match - assert.Equal(t, original.ID, deserialized.ID) - assert.Equal(t, original.Queue, deserialized.Queue) - assert.Equal(t, original.Change.URIs, deserialized.Change.URIs) - assert.Equal(t, original.LandStrategy, deserialized.LandStrategy) -} - -func TestLandRequestFromBytes_InvalidJSON(t *testing.T) { - invalidJSON := []byte(`{"invalid": json"}`) - - _, err := LandRequestFromBytes(invalidJSON) - assert.Error(t, err) -} - -func TestLandRequestFromBytes_EmptyData(t *testing.T) { - emptyJSON := []byte(`{}`) - - req, err := LandRequestFromBytes(emptyJSON) - require.NoError(t, err) - - // Empty JSON should deserialize with zero values - assert.Empty(t, req.ID) - assert.Empty(t, req.Queue) - assert.Equal(t, mergestrategy.MergeStrategyUnknown, req.LandStrategy) -} - -func TestLandRequest_SerializationRoundTrip(t *testing.T) { - tests := []struct { - name string - req LandRequest - }{ - { - name: "github stacked diff", - req: LandRequest{ - ID: "queue1/100", - Queue: "queue1", - Change: change.Change{URIs: []string{ - "github://github.example.com/uber/repo-a/pull/101/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "github://github.example.com/uber/repo-a/pull/102/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "github://github.example.com/uber/repo-a/pull/103/cccccccccccccccccccccccccccccccccccccccc", - }}, - LandStrategy: mergestrategy.MergeStrategySquashRebase, - }, - }, - { - name: "phabricator revision", - req: LandRequest{ - ID: "queue2/200", - Queue: "queue2", - Change: change.Change{URIs: []string{"code.uber.internal.com/D12345"}}, - LandStrategy: mergestrategy.MergeStrategyRebase, - }, - }, - { - name: "github enterprise request", - req: LandRequest{ - ID: "queue3/300", - Queue: "queue3", - Change: change.Change{URIs: []string{"github.uber.com/internal/service/999/deadbeef12"}}, - LandStrategy: mergestrategy.MergeStrategyMerge, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Serialize - data, err := tt.req.ToBytes() - require.NoError(t, err) - - // Deserialize - deserialized, err := LandRequestFromBytes(data) - require.NoError(t, err) - - // Verify complete equality - assert.Equal(t, tt.req, deserialized) - }) - } -} diff --git a/submitqueue/entity/request.go b/submitqueue/entity/request.go index 5d970d25d..5182c8f83 100644 --- a/submitqueue/entity/request.go +++ b/submitqueue/entity/request.go @@ -115,15 +115,3 @@ type RequestID struct { // Queue is the name of the queue processing the land request. Empty on payloads written before the field existed. Queue string `json:"queue"` } - -// ToBytes serializes the RequestID to JSON bytes for queue message payload. -func (r RequestID) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// RequestIDFromBytes deserializes a RequestID from JSON bytes. -func RequestIDFromBytes(data []byte) (RequestID, error) { - var rid RequestID - err := json.Unmarshal(data, &rid) - return rid, err -} diff --git a/submitqueue/entity/request_log.go b/submitqueue/entity/request_log.go index a1be20fc3..ea29018f6 100644 --- a/submitqueue/entity/request_log.go +++ b/submitqueue/entity/request_log.go @@ -15,7 +15,6 @@ package entity import ( - "encoding/json" "time" ) @@ -218,26 +217,3 @@ func (r RequestLog) Value() string { } return string(r.Status) } - -// ToBytes serializes the RequestLog to JSON bytes for queue message payload. -func (r RequestLog) ToBytes() ([]byte, error) { - return json.Marshal(r) -} - -// RequestLogFromBytes deserializes a RequestLog from JSON bytes. -// If metadata is absent from the JSON, it will be initialized as an empty map. -// An entry without a type predates the field, when every entry recorded a status. -func RequestLogFromBytes(data []byte) (RequestLog, error) { - var log RequestLog - err := json.Unmarshal(data, &log) - if err != nil { - return log, err - } - if log.Metadata == nil { - log.Metadata = make(map[string]string) - } - if log.Type == "" { - log.Type = RequestLogTypeStatus - } - return log, nil -} diff --git a/submitqueue/entity/request_log_test.go b/submitqueue/entity/request_log_test.go index 3e2f6cc6d..e6e33f4fe 100644 --- a/submitqueue/entity/request_log_test.go +++ b/submitqueue/entity/request_log_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestNewRequestLog_NilMetadata(t *testing.T) { @@ -28,135 +27,9 @@ func TestNewRequestLog_NilMetadata(t *testing.T) { assert.Empty(t, log.Metadata) } -func TestRequestLog_ToBytes(t *testing.T) { - log := RequestLog{ - RequestID: "test-queue/123", - TimestampMs: 1709568000000, - Status: RequestStatusStarted, - RequestVersion: 1, - LastError: "", - Metadata: map[string]string{"source": "gateway"}, - } - - data, err := log.ToBytes() - require.NoError(t, err) - assert.NotEmpty(t, data) - - jsonStr := string(data) - assert.Contains(t, jsonStr, "test-queue/123") - assert.Contains(t, jsonStr, "1709568000000") - assert.Contains(t, jsonStr, "gateway") -} - -func TestRequestLogFromBytes(t *testing.T) { - original := RequestLog{ - RequestID: "my-queue/999", - TimestampMs: 1709568000000, - Status: RequestStatusSpeculating, - RequestVersion: 3, - LastError: "timeout", - Metadata: map[string]string{"step": "validation", "attempt": "2"}, - } - - data, err := original.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestLogFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, original.RequestID, deserialized.RequestID) - assert.Equal(t, original.TimestampMs, deserialized.TimestampMs) - assert.Equal(t, original.Status, deserialized.Status) - assert.Equal(t, original.RequestVersion, deserialized.RequestVersion) - assert.Equal(t, original.LastError, deserialized.LastError) - assert.Equal(t, original.Metadata, deserialized.Metadata) -} - -func TestRequestLogFromBytes_InvalidJSON(t *testing.T) { - invalidJSON := []byte(`{"invalid": json"}`) - - _, err := RequestLogFromBytes(invalidJSON) - assert.Error(t, err) -} - -func TestRequestLogFromBytes_EmptyData(t *testing.T) { - emptyJSON := []byte(`{}`) - - log, err := RequestLogFromBytes(emptyJSON) - require.NoError(t, err) - - assert.Empty(t, log.RequestID) - assert.Equal(t, int64(0), log.TimestampMs) - assert.Empty(t, log.Status) - assert.Equal(t, int32(0), log.RequestVersion) - assert.Empty(t, log.LastError) - assert.NotNil(t, log.Metadata) - assert.Empty(t, log.Metadata) -} - -func TestRequestLog_SerializationRoundTrip(t *testing.T) { - tests := []struct { - name string - log RequestLog - }{ - { - name: "with all fields populated", - log: RequestLog{ - RequestID: "queue1/100", - TimestampMs: 1709568000000, - Type: RequestLogTypeStatus, - Status: RequestStatusLanded, - RequestVersion: 5, - LastError: "", - Metadata: map[string]string{"source": "orchestrator", "batch_id": "b-1"}, - }, - }, - { - name: "with error", - log: RequestLog{ - RequestID: "queue2/200", - TimestampMs: 1709568001000, - Type: RequestLogTypeStatus, - Status: RequestStatusError, - RequestVersion: 2, - LastError: "merge conflict detected", - Metadata: map[string]string{}, - }, - }, - { - name: "with zero version", - log: RequestLog{ - RequestID: "queue3/300", - TimestampMs: 1709568002000, - Type: RequestLogTypeStatus, - Status: RequestStatusStarted, - RequestVersion: 0, - LastError: "", - Metadata: map[string]string{"key": "value"}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - data, err := tt.log.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestLogFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, tt.log, deserialized) - }) - } -} - // An entry records a status or an event, never both. The constructors are what // hold that up — the struct can express the invalid combinations, nothing is // meant to build them — so this pins what each one sets and leaves unset. -// -// The old classifier this replaces asked "is this status really an event?", a -// question the split makes unaskable: an event is a RequestEvent and cannot be -// assigned to Status or to RequestSummary.Status at all. func TestRequestLogConstructors(t *testing.T) { t.Run("status entry carries a status and no event", func(t *testing.T) { log := NewRequestStatusLog("q", "q/1", RequestStatusSpeculating, 4, "boom", map[string]string{"k": "v"}) @@ -176,34 +49,6 @@ func TestRequestLogConstructors(t *testing.T) { assert.Equal(t, RequestEventBuilding, log.Event) assert.Equal(t, RequestStatusUnknown, log.Status) assert.Equal(t, "building", log.Value()) - // An event is not a state transition, so it can carry no version for a - // reader to mistake for a reconcilable one. assert.Zero(t, log.RequestVersion) }) } - -// An entry written before the type column existed recorded a status, so it must -// read back as one rather than as the zero type — otherwise every historical -// entry would stop counting towards the current status the moment this shipped. -func TestRequestLogFromBytes_UntypedEntryIsAStatus(t *testing.T) { - legacy := []byte(`{"request_id":"q/1","queue":"q","timestamp_ms":10,"status":"landed","request_version":3}`) - - log, err := RequestLogFromBytes(legacy) - require.NoError(t, err) - - assert.Equal(t, RequestLogTypeStatus, log.Type) - assert.Equal(t, RequestStatusLanded, log.Status) - assert.Equal(t, "landed", log.Value()) -} - -func TestRequestLog_EventSerializationRoundTrip(t *testing.T) { - original := NewRequestEventLog("q", "q/1", RequestEventBuilt, map[string]string{"build_id": "b/7"}) - - data, err := original.ToBytes() - require.NoError(t, err) - - deserialized, err := RequestLogFromBytes(data) - require.NoError(t, err) - - assert.Equal(t, original, deserialized) -} diff --git a/submitqueue/gateway/controller/BUILD.bazel b/submitqueue/gateway/controller/BUILD.bazel index 377136749..0faaa4164 100644 --- a/submitqueue/gateway/controller/BUILD.bazel +++ b/submitqueue/gateway/controller/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "//platform/extension/counter:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -52,6 +53,7 @@ go_test( "//platform/extension/counter:go_default_library", "//platform/extension/counter/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", diff --git a/submitqueue/gateway/controller/cancel.go b/submitqueue/gateway/controller/cancel.go index f835d61fe..678953502 100644 --- a/submitqueue/gateway/controller/cancel.go +++ b/submitqueue/gateway/controller/cancel.go @@ -23,6 +23,7 @@ import ( "github.com/uber/submitqueue/platform/errs" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" requestcore "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -129,7 +130,7 @@ func (c *cancelController) Cancel(ctx context.Context, req entity.CancelRequest) // publishToQueue publishes a cancel request to the cancel queue for async processing. func (c *cancelController) publishToQueue(ctx context.Context, cancelRequest entity.CancelRequest) error { - payload, err := cancelRequest.ToBytes() + payload, err := sqmq.Marshal(sqmq.CancelFromEntity(cancelRequest)) if err != nil { return fmt.Errorf("failed to serialize cancel request: %w", err) } diff --git a/submitqueue/gateway/controller/cancel_test.go b/submitqueue/gateway/controller/cancel_test.go index 554079906..2935549ca 100644 --- a/submitqueue/gateway/controller/cancel_test.go +++ b/submitqueue/gateway/controller/cancel_test.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/errs" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" storagemock "github.com/uber/submitqueue/submitqueue/extension/storage/mock" @@ -156,7 +157,7 @@ func TestCancel_PublishesToQueue(t *testing.T) { assert.Equal(t, "my-queue", publishedMessage.Tenant) assert.Equal(t, "my-queue/7", publishedMessage.PartitionKey) - deserialized, err := entity.CancelRequestFromBytes(publishedMessage.Payload) + deserialized, err := sqmq.UnmarshalCancelRequest(publishedMessage.Payload) require.NoError(t, err) assert.Equal(t, "my-queue/7", deserialized.ID) assert.Equal(t, "obsolete change", deserialized.Reason) diff --git a/submitqueue/gateway/controller/land.go b/submitqueue/gateway/controller/land.go index d03b537a8..63d81a003 100644 --- a/submitqueue/gateway/controller/land.go +++ b/submitqueue/gateway/controller/land.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/extension/counter" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" requestcore "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -201,8 +202,7 @@ func (c *landController) Land(ctx context.Context, req entity.LandRequest) (resu // publishToQueue publishes a land request to the request queue for async processing. func (c *landController) publishToQueue(ctx context.Context, landRequest entity.LandRequest) error { - // Serialize land request entity to JSON - payload, err := landRequest.ToBytes() + payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(landRequest)) if err != nil { return fmt.Errorf("failed to serialize land request: %w", err) } @@ -210,7 +210,6 @@ func (c *landController) publishToQueue(ctx context.Context, landRequest entity. // Publish the request into the pipeline: // - Message ID: landRequest.ID with no cause — a request enters once, so a // retry of this same publish dedups instead of enqueuing it twice - // - Payload: serialized LandRequest entity // - Partition key: landRequest.Queue (ensures ordering per queue) if err := publish.Message(ctx, c.registry, topickey.TopicKeyStart, publish.MessageParams{ Tenant: landRequest.Queue, diff --git a/submitqueue/gateway/controller/land_test.go b/submitqueue/gateway/controller/land_test.go index 52b25f8ee..a0066dead 100644 --- a/submitqueue/gateway/controller/land_test.go +++ b/submitqueue/gateway/controller/land_test.go @@ -31,6 +31,7 @@ import ( "github.com/uber/submitqueue/platform/extension/counter" countermock "github.com/uber/submitqueue/platform/extension/counter/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" requestcore "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -463,7 +464,7 @@ func TestLand_PublishesToQueue(t *testing.T) { assert.Equal(t, "test-queue", publishedMessage.PartitionKey) // Verify payload can be deserialized - deserializedReq, err := entity.LandRequestFromBytes(publishedMessage.Payload) + deserializedReq, err := sqmq.UnmarshalLandRequest(publishedMessage.Payload) require.NoError(t, err) assert.Equal(t, "test-queue/123", deserializedReq.ID) assert.Equal(t, "test-queue", deserializedReq.Queue) diff --git a/submitqueue/gateway/controller/log/BUILD.bazel b/submitqueue/gateway/controller/log/BUILD.bazel index 3c22824ae..cec2124a0 100644 --- a/submitqueue/gateway/controller/log/BUILD.bazel +++ b/submitqueue/gateway/controller/log/BUILD.bazel @@ -9,8 +9,8 @@ go_library( "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", - "//submitqueue/entity:go_default_library", "@com_github_uber_go_tally//:go_default_library", "@org_uber_go_zap//:go_default_library", ], @@ -23,6 +23,7 @@ go_test( deps = [ "//platform/base/messagequeue:go_default_library", "//platform/consumer/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", diff --git a/submitqueue/gateway/controller/log/log.go b/submitqueue/gateway/controller/log/log.go index ef02de7ff..856b182d9 100644 --- a/submitqueue/gateway/controller/log/log.go +++ b/submitqueue/gateway/controller/log/log.go @@ -22,8 +22,8 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" requestcore "github.com/uber/submitqueue/submitqueue/core/request" - "github.com/uber/submitqueue/submitqueue/entity" "go.uber.org/zap" ) @@ -70,8 +70,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er msg := delivery.Message() - // Deserialize request log entry - logEntry, err := entity.RequestLogFromBytes(msg.Payload) + logEntry, err := sqmq.UnmarshalRequestLog(msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) // Non-retryable: malformed messages will never succeed regardless of retry count diff --git a/submitqueue/gateway/controller/log/log_test.go b/submitqueue/gateway/controller/log/log_test.go index 01cea18f1..ec00912c1 100644 --- a/submitqueue/gateway/controller/log/log_test.go +++ b/submitqueue/gateway/controller/log/log_test.go @@ -23,6 +23,7 @@ import ( "github.com/uber-go/tally" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" consumermock "github.com/uber/submitqueue/platform/consumer/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" requestcore "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -98,7 +99,7 @@ func TestController_Process(t *testing.T) { payload := tt.rawPayload if tt.logEntry != nil { var err error - payload, err = tt.logEntry.ToBytes() + payload, err = sqmq.Marshal(sqmq.LogFromEntity(*tt.logEntry)) require.NoError(t, err) } controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, tt.setupStore(ctrl), topickey.TopicKeyLog, "gateway-log") @@ -123,7 +124,7 @@ func TestController_Process(t *testing.T) { func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) logEntry := newRequestLog("test-queue/1", entity.RequestStatusStarted, 1, "", nil) - payload, err := logEntry.ToBytes() + payload, err := sqmq.Marshal(sqmq.LogFromEntity(*logEntry)) require.NoError(t, err) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, newUnusedMaterializer(ctrl), topickey.TopicKeyLog, "gateway-log") msg := entityqueue.NewMessage(logEntry.RequestID, payload, logEntry.Queue, nil) diff --git a/submitqueue/orchestrator/controller/batch/BUILD.bazel b/submitqueue/orchestrator/controller/batch/BUILD.bazel index aac1f9e63..ae667d36d 100644 --- a/submitqueue/orchestrator/controller/batch/BUILD.bazel +++ b/submitqueue/orchestrator/controller/batch/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//platform/extension/counter:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -33,6 +34,7 @@ go_test( "//platform/extension/counter:go_default_library", "//platform/extension/counter/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/batch/batch.go b/submitqueue/orchestrator/controller/batch/batch.go index dec1c73fc..00d93a5d9 100644 --- a/submitqueue/orchestrator/controller/batch/batch.go +++ b/submitqueue/orchestrator/controller/batch/batch.go @@ -24,6 +24,7 @@ import ( "github.com/uber/submitqueue/platform/extension/counter" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -82,8 +83,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - // Deserialize request ID from payload - rid, err := entity.RequestIDFromBytes(msg.Payload) + rid, err := sqmq.UnmarshalRequestID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize request ID: %w", err) @@ -210,7 +210,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // The message ID is the bare batch ID, with no cause: a batch is handed over // once in its life, so a redelivery that re-sends it is meant to be dropped. func (c *Controller) publishToDependencyAnalysis(ctx context.Context, batch entity.Batch) error { - payload, err := entity.BatchID{ID: batch.ID, Queue: batch.Queue}.ToBytes() + payload, err := sqmq.MarshalID(topickey.TopicKeyDependencyAnalysis, batch.ID, batch.Queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/batch/batch_test.go b/submitqueue/orchestrator/controller/batch/batch_test.go index 4227deb8b..d597a42f3 100644 --- a/submitqueue/orchestrator/controller/batch/batch_test.go +++ b/submitqueue/orchestrator/controller/batch/batch_test.go @@ -31,6 +31,7 @@ import ( "github.com/uber/submitqueue/platform/extension/counter" countermock "github.com/uber/submitqueue/platform/extension/counter/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -39,9 +40,9 @@ import ( "go.uber.org/zap/zaptest" ) -// requestIDPayload serializes a RequestID to JSON bytes for test message payloads. +// requestIDPayload serializes a Batch payload for the batch controller. func requestIDPayload(t *testing.T, id, queue string) []byte { - payload, err := entity.RequestID{ID: id, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, id, queue) require.NoError(t, err) return payload } @@ -130,7 +131,7 @@ func newDelivery(t *testing.T, ctrl *gomock.Controller, request entity.Request, payload := requestIDPayload(t, request.ID, request.Queue) tenant := request.Queue if payloadQueue != "" { - bytes, err := entity.RequestID{ID: request.ID, Queue: payloadQueue}.ToBytes() + bytes, err := sqmq.MarshalID(sqmq.TopicKeyBatch, request.ID, payloadQueue) require.NoError(t, err) payload = bytes tenant = payloadQueue @@ -227,7 +228,7 @@ func TestController_Process_StampsQueueOnHandoffPayload(t *testing.T) { require.NoError(t, controller.Process(context.Background(), newDelivery(t, ctrl, request, ""))) require.Len(t, handoffs, 1) - bid, err := entity.BatchIDFromBytes(handoffs[0].Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeyDependencyAnalysis, handoffs[0].Payload) require.NoError(t, err) assert.Equal(t, request.Queue, bid.Queue) assert.Equal(t, request.Queue, handoffs[0].PartitionKey) @@ -411,7 +412,7 @@ func TestController_Process_PublishesBatchingStatus(t *testing.T) { publisher.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, topic string, msg entityqueue.Message) error { if topic == "log" { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) logs = append(logs, entry) } diff --git a/submitqueue/orchestrator/controller/build/BUILD.bazel b/submitqueue/orchestrator/controller/build/BUILD.bazel index 1803101cf..cb4c9fad5 100644 --- a/submitqueue/orchestrator/controller/build/BUILD.bazel +++ b/submitqueue/orchestrator/controller/build/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/buildrunner:go_default_library", @@ -28,6 +29,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/buildrunner:go_default_library", diff --git a/submitqueue/orchestrator/controller/build/build.go b/submitqueue/orchestrator/controller/build/build.go index 207646e3c..5dc66aed4 100644 --- a/submitqueue/orchestrator/controller/build/build.go +++ b/submitqueue/orchestrator/controller/build/build.go @@ -38,6 +38,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/buildrunner" @@ -101,7 +102,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize batch ID: %w", err) @@ -380,7 +381,7 @@ func (c *Controller) loadBase(ctx context.Context, store storage.Storage, path e // away while the original signal is still in the queue's un-GC'd window (see // publish.IntentID). func (c *Controller) publishBuildSignal(ctx context.Context, buildID, queue string) error { - payload, err := entity.BuildID{ID: buildID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, buildID, queue) if err != nil { return fmt.Errorf("failed to serialize build ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/build/build_test.go b/submitqueue/orchestrator/controller/build/build_test.go index 74f23690b..0dd3f1305 100644 --- a/submitqueue/orchestrator/controller/build/build_test.go +++ b/submitqueue/orchestrator/controller/build/build_test.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/buildrunner" @@ -58,7 +59,7 @@ func (f staticBuildRunnerFactory) For(buildrunner.Config) (buildrunner.BuildRunn func batchIDPayload(t *testing.T, id string) []byte { t.Helper() - payload, err := entity.BatchID{ID: id, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuild, id, "test-queue") require.NoError(t, err) return payload } @@ -184,7 +185,7 @@ func expectSignal(t *testing.T, deps *testDeps, buildID string) { t.Helper() deps.publisher.EXPECT().Publish(gomock.Any(), "buildsignal", gomock.Any()). DoAndReturn(func(_ context.Context, _ string, msg entityqueue.Message) error { - got, err := entity.BuildIDFromBytes(msg.Payload) + got, err := sqmq.UnmarshalBuildID(sqmq.TopicKeyBuildSignal, msg.Payload) require.NoError(t, err) assert.Equal(t, buildID, got.ID) assert.Equal(t, "test-queue", msg.Tenant) @@ -248,7 +249,7 @@ func TestProcess_TriggersWithThePathsBase(t *testing.T) { }).Return(nil), deps.publisher.EXPECT().Publish(gomock.Any(), "buildsignal", gomock.Any()). DoAndReturn(func(_ context.Context, _ string, msg entityqueue.Message) error { - got, err := entity.BuildIDFromBytes(msg.Payload) + got, err := sqmq.UnmarshalBuildID(sqmq.TopicKeyBuildSignal, msg.Payload) require.NoError(t, err) assert.Equal(t, "build-1", got.ID) assert.Equal(t, "build-1", msg.PartitionKey, diff --git a/submitqueue/orchestrator/controller/buildsignal/BUILD.bazel b/submitqueue/orchestrator/controller/buildsignal/BUILD.bazel index 9022b3dda..fb40c8c2f 100644 --- a/submitqueue/orchestrator/controller/buildsignal/BUILD.bazel +++ b/submitqueue/orchestrator/controller/buildsignal/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -29,6 +30,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/buildrunner/mock:go_default_library", diff --git a/submitqueue/orchestrator/controller/buildsignal/buildsignal.go b/submitqueue/orchestrator/controller/buildsignal/buildsignal.go index 18ce0e052..65bf4c990 100644 --- a/submitqueue/orchestrator/controller/buildsignal/buildsignal.go +++ b/submitqueue/orchestrator/controller/buildsignal/buildsignal.go @@ -50,6 +50,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -138,7 +139,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - buildID, err := entity.BuildIDFromBytes(msg.Payload) + buildID, err := sqmq.UnmarshalBuildID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) // Non-retryable: malformed messages will never succeed. @@ -440,7 +441,7 @@ func findEntry(set entity.SpeculationPathSet, pathID string) (entity.Speculation // publishBatchID publishes a batch ID to the topic identified by key under // msgID, stamped with and partitioned by the batch's queue. func (c *Controller) publishBatchID(ctx context.Context, key consumer.TopicKey, msgID, batchID, queue string) error { - payload, err := entity.BatchID{ID: batchID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(key, batchID, queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go b/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go index c9184c91f..543531a67 100644 --- a/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go +++ b/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go @@ -27,6 +27,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" buildrunnermock "github.com/uber/submitqueue/submitqueue/extension/buildrunner/mock" @@ -90,7 +91,7 @@ func newTestHarness(t *testing.T, ctrl *gomock.Controller, batchState entity.Bat logPub := queuemock.NewMockPublisher(ctrl) logPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) h.logs = append(h.logs, entry) return nil @@ -166,7 +167,7 @@ func (h *testHarness) wanted() { // is returned so tests can expect a Hold for the next poll. func delivery(t *testing.T, ctrl *gomock.Controller) *consumermock.MockDelivery { t.Helper() - payload, err := entity.BuildID{ID: testBuildID, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, testBuildID, "test-queue") require.NoError(t, err) msg := entityqueue.NewMessage(testBuildID, payload, testBuildID, nil) msg.Tenant = "test-queue" @@ -201,7 +202,7 @@ func TestController_Identity(t *testing.T) { func TestProcess_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) h := newTestHarness(t, ctrl, entity.BatchStateSpeculating) - payload, err := entity.BuildID{ID: testBuildID, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, testBuildID, "test-queue") require.NoError(t, err) msg := entityqueue.NewMessage(testBuildID, payload, testBuildID, nil) msg.Tenant = "other-queue" diff --git a/submitqueue/orchestrator/controller/cancel/BUILD.bazel b/submitqueue/orchestrator/controller/cancel/BUILD.bazel index 8dfe860ae..3ab90a062 100644 --- a/submitqueue/orchestrator/controller/cancel/BUILD.bazel +++ b/submitqueue/orchestrator/controller/cancel/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//platform/metrics:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -29,6 +30,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/cancel/cancel.go b/submitqueue/orchestrator/controller/cancel/cancel.go index 66d18d932..786ea106c 100644 --- a/submitqueue/orchestrator/controller/cancel/cancel.go +++ b/submitqueue/orchestrator/controller/cancel/cancel.go @@ -60,6 +60,7 @@ import ( "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -106,7 +107,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - cancelReq, err := entity.CancelRequestFromBytes(msg.Payload) + cancelReq, err := sqmq.UnmarshalCancelRequest(msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize cancel request: %w", err) @@ -350,7 +351,7 @@ func (c *Controller) cancelBatch(ctx context.Context, store storage.Storage, bat // redelivery re-publish documented above a silent no-op — leaving a batch // Cancelling with nothing driving it to terminal. func (c *Controller) publishBatchID(ctx context.Context, key consumer.TopicKey, batchID string, queue string) error { - payload, err := entity.BatchID{ID: batchID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(key, batchID, queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/cancel/cancel_test.go b/submitqueue/orchestrator/controller/cancel/cancel_test.go index a35267e27..32a400c59 100644 --- a/submitqueue/orchestrator/controller/cancel/cancel_test.go +++ b/submitqueue/orchestrator/controller/cancel/cancel_test.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -61,7 +62,7 @@ func requestWithState(request entity.Request, state entity.RequestState) entity. // cancelPayload serializes a CancelRequest to JSON bytes for test message payloads. func cancelPayload(t *testing.T, id, reason string) []byte { - payload, err := entity.CancelRequest{ID: id, Queue: "q", Reason: reason}.ToBytes() + payload, err := sqmq.Marshal(sqmq.CancelFromEntity(entity.CancelRequest{ID: id, Queue: "q", Reason: reason})) require.NoError(t, err) return payload } @@ -392,7 +393,7 @@ func TestProcess_BatchPath_HandsOffToSpeculate(t *testing.T) { var records []pubRec pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, topic string, msg entityqueue.Message) error { - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, msg.Payload) require.NoError(t, err) records = append(records, pubRec{topic: topic, msgID: msg.ID, payloadID: bid.ID}) return nil @@ -465,7 +466,7 @@ func TestProcess_CancelsEveryApplicableBatch(t *testing.T) { func(_ context.Context, _ string, msg entityqueue.Message) error { // The message ID is the queue's dedup key and is distinct per // publish; the payload carries the batch ID the consumer acts on. - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, msg.Payload) require.NoError(t, err) operations = append(operations, "publish:"+bid.ID) return nil @@ -506,7 +507,7 @@ func TestProcess_BatchFailureDoesNotPreventLaterCancellation(t *testing.T) { batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch2, entity.BatchStateCancelling), int32(2), int32(3)).Return(nil) publisher.EXPECT().Publish(gomock.Any(), "speculate", gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, msg.Payload) require.NoError(t, err) assert.Equal(t, batch2.ID, bid.ID) return nil diff --git a/submitqueue/orchestrator/controller/conclude/BUILD.bazel b/submitqueue/orchestrator/controller/conclude/BUILD.bazel index 493cdfc06..44a064048 100644 --- a/submitqueue/orchestrator/controller/conclude/BUILD.bazel +++ b/submitqueue/orchestrator/controller/conclude/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -28,6 +29,7 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/conclude/conclude.go b/submitqueue/orchestrator/controller/conclude/conclude.go index 2c21c5bb2..5b99d0660 100644 --- a/submitqueue/orchestrator/controller/conclude/conclude.go +++ b/submitqueue/orchestrator/controller/conclude/conclude.go @@ -22,6 +22,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -69,8 +70,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - // Deserialize batch ID from payload - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, "process", "deserialize_errors", 1) return fmt.Errorf("failed to deserialize batch ID: %w", err) diff --git a/submitqueue/orchestrator/controller/conclude/conclude_test.go b/submitqueue/orchestrator/controller/conclude/conclude_test.go index 03fad71b3..604187cdf 100644 --- a/submitqueue/orchestrator/controller/conclude/conclude_test.go +++ b/submitqueue/orchestrator/controller/conclude/conclude_test.go @@ -27,6 +27,7 @@ import ( consumermock "github.com/uber/submitqueue/platform/consumer/mock" "github.com/uber/submitqueue/platform/errs" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -48,7 +49,7 @@ func requestWithState(request entity.Request, state entity.RequestState) entity. // batchIDPayload serializes a BatchID to JSON bytes for test message payloads. func batchIDPayload(t *testing.T, id string) []byte { - payload, err := entity.BatchID{ID: id, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyConclude, id, "test-queue") require.NoError(t, err) return payload } @@ -495,7 +496,7 @@ func TestController_Process_FailedBatchCarriesReasonToRequestLog(t *testing.T) { var logged entity.RequestLog pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - log, err := entity.RequestLogFromBytes(msg.Payload) + log, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) logged = log return nil diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/BUILD.bazel b/submitqueue/orchestrator/controller/dependencyanalysis/BUILD.bazel index 79c891d48..a8d24de8c 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/BUILD.bazel +++ b/submitqueue/orchestrator/controller/dependencyanalysis/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//platform/metrics:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -30,6 +31,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/conflict:go_default_library", diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go index 4f7a604e2..b018972aa 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go +++ b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis.go @@ -59,6 +59,7 @@ import ( "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -110,7 +111,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize batch ID: %w", err) @@ -439,7 +440,7 @@ func (c *Controller) writeDependentIndexes(ctx context.Context, store storage.St // dropped. Every later publish about the same batch names its cause and so // cannot collide with this row — see publish.IntentID. func (c *Controller) publishToSpeculate(ctx context.Context, batch entity.Batch) error { - payload, err := entity.BatchID{ID: batch.ID, Queue: batch.Queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, batch.ID, batch.Queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go index e49b372f5..514635db0 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go +++ b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go @@ -27,6 +27,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/conflict" @@ -73,7 +74,7 @@ func liveRequest() entity.Request { func batchIDPayload(t *testing.T, id, queue string) []byte { t.Helper() - payload, err := entity.BatchID{ID: id, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyDependencyAnalysis, id, queue) require.NoError(t, err) return payload } @@ -494,7 +495,7 @@ func TestController_Process_StampsQueueOnAnnouncement(t *testing.T) { require.NoError(t, controller.Process(context.Background(), newDelivery(t, ctrl, batch.ID, testQueue))) require.Len(t, announced, 1) - bid, err := entity.BatchIDFromBytes(announced[0].Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, announced[0].Payload) require.NoError(t, err) assert.Equal(t, batch.ID, bid.ID) assert.Equal(t, testQueue, bid.Queue) @@ -658,7 +659,7 @@ func TestController_Process_PublishesBatchedLogOnPromotion(t *testing.T) { publisher.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, topic string, msg entityqueue.Message) error { if topic == "log" { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) logs = append(logs, entry) } diff --git a/submitqueue/orchestrator/controller/dlq/BUILD.bazel b/submitqueue/orchestrator/controller/dlq/BUILD.bazel index a58385885..7f839936e 100644 --- a/submitqueue/orchestrator/controller/dlq/BUILD.bazel +++ b/submitqueue/orchestrator/controller/dlq/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//platform/metrics:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -53,6 +54,7 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/dlq/batch.go b/submitqueue/orchestrator/controller/dlq/batch.go index 2b8ab2a8c..c790e84d5 100644 --- a/submitqueue/orchestrator/controller/dlq/batch.go +++ b/submitqueue/orchestrator/controller/dlq/batch.go @@ -22,7 +22,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/submitqueue/entity" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/extension/storage" "go.uber.org/zap" ) @@ -82,7 +82,7 @@ func (c *batchController) Process(ctx context.Context, delivery consumer.Deliver msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(primaryTopicKey(c.topicKey), msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to decode batch id from dlq payload: %w", err) diff --git a/submitqueue/orchestrator/controller/dlq/batch_test.go b/submitqueue/orchestrator/controller/dlq/batch_test.go index 9bcebe0b8..544e67588 100644 --- a/submitqueue/orchestrator/controller/dlq/batch_test.go +++ b/submitqueue/orchestrator/controller/dlq/batch_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber/submitqueue/platform/consumer" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" storagemock "github.com/uber/submitqueue/submitqueue/extension/storage/mock" @@ -69,7 +70,7 @@ func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - payload, err := entity.BatchID{ID: "q/batch/9", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "q/batch/9", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -81,7 +82,7 @@ func TestDLQBatchController_Process_TenantPayloadQueueMismatchAcks(t *testing.T) store := storagemock.NewMockStorage(ctrl) c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - payload, err := entity.BatchID{ID: "q/batch/9", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "q/batch/9", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) @@ -106,7 +107,7 @@ func TestDLQBatchController_Process_EmptyIDFails(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - payload, err := entity.BatchID{ID: "", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) diff --git a/submitqueue/orchestrator/controller/dlq/buildsignal.go b/submitqueue/orchestrator/controller/dlq/buildsignal.go index 21089b869..a25ab4aa9 100644 --- a/submitqueue/orchestrator/controller/dlq/buildsignal.go +++ b/submitqueue/orchestrator/controller/dlq/buildsignal.go @@ -23,7 +23,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" - "github.com/uber/submitqueue/submitqueue/entity" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/extension/storage" "go.uber.org/zap" ) @@ -74,7 +74,7 @@ func (c *buildSignalController) Process(ctx context.Context, delivery consumer.D msg := delivery.Message() - buildID, err := entity.BuildIDFromBytes(msg.Payload) + buildID, err := sqmq.UnmarshalBuildID(primaryTopicKey(c.topicKey), msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to decode build id from dlq payload: %w", err) diff --git a/submitqueue/orchestrator/controller/dlq/buildsignal_test.go b/submitqueue/orchestrator/controller/dlq/buildsignal_test.go index f3bf64768..39ae7df45 100644 --- a/submitqueue/orchestrator/controller/dlq/buildsignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/buildsignal_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/uber/submitqueue/platform/consumer" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -77,7 +78,7 @@ func TestDLQBuildSignalController_Process_FansOutToBatch(t *testing.T) { c := NewDLQBuildSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(topickey.TopicKeyBuildSignal), "orchestrator-buildsignal-dlq") - payload, err := entity.BuildID{ID: "build-1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, "build-1", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -89,7 +90,7 @@ func TestDLQBuildSignalController_Process_TenantPayloadQueueMismatchAcks(t *test store := storagemock.NewMockStorage(ctrl) c := NewDLQBuildSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyBuildSignal), "orchestrator-buildsignal-dlq") - payload, err := entity.BuildID{ID: "build-1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, "build-1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) @@ -107,7 +108,7 @@ func TestDLQBuildSignalController_Process_BuildNotFoundIsNoOp(t *testing.T) { c := NewDLQBuildSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyBuildSignal), "orchestrator-buildsignal-dlq") - payload, err := entity.BuildID{ID: "build-1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, "build-1", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -128,7 +129,7 @@ func TestDLQBuildSignalController_Process_BuildMissingBatchIsNoOp(t *testing.T) c := NewDLQBuildSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyBuildSignal), "orchestrator-buildsignal-dlq") - payload, err := entity.BuildID{ID: "build-1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBuildSignal, "build-1", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) diff --git a/submitqueue/orchestrator/controller/dlq/dlq.go b/submitqueue/orchestrator/controller/dlq/dlq.go index 3d99e4d1c..47960177d 100644 --- a/submitqueue/orchestrator/controller/dlq/dlq.go +++ b/submitqueue/orchestrator/controller/dlq/dlq.go @@ -63,6 +63,12 @@ func TopicKey(main consumer.TopicKey) consumer.TopicKey { return consumer.TopicKey(string(main) + topicSuffix) } +// primaryTopicKey strips the DLQ suffix so payload decode uses the originating +// topic's message type. DLQ rows keep the primary payload bytes verbatim. +func primaryTopicKey(dlqKey consumer.TopicKey) consumer.TopicKey { + return consumer.TopicKey(strings.TrimSuffix(string(dlqKey), topicSuffix)) +} + // failureContext reads everything the queue recorded about a dead-lettered // message: the human-readable reason, and a metadata map to carry alongside it // on the terminal request log. diff --git a/submitqueue/orchestrator/controller/dlq/publisher_test.go b/submitqueue/orchestrator/controller/dlq/publisher_test.go index e767cbd78..0e56e73c3 100644 --- a/submitqueue/orchestrator/controller/dlq/publisher_test.go +++ b/submitqueue/orchestrator/controller/dlq/publisher_test.go @@ -21,6 +21,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "go.uber.org/mock/gomock" @@ -35,7 +36,7 @@ func newTestLogRegistry( publisher := queuemock.NewMockPublisher(ctrl) publisher.EXPECT().Publish(gomock.Any(), "log", gomock.Any()).DoAndReturn( func(_ context.Context, _ string, message entityqueue.Message) error { - logEntry, err := entity.RequestLogFromBytes(message.Payload) + logEntry, err := sqmq.UnmarshalRequestLog(message.Payload) require.NoError(t, err) require.Equal(t, logEntry.Queue, message.Tenant) return publishFn(logEntry) diff --git a/submitqueue/orchestrator/controller/dlq/request.go b/submitqueue/orchestrator/controller/dlq/request.go index 9a078dcf9..153c55afc 100644 --- a/submitqueue/orchestrator/controller/dlq/request.go +++ b/submitqueue/orchestrator/controller/dlq/request.go @@ -24,6 +24,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" "go.uber.org/zap" @@ -40,7 +41,7 @@ type RequestIDDecoder func(payload []byte) (entity.RequestID, error) // DecodeLandRequestID extracts the request ID from a LandRequest payload // (the shape used by the start topic). func DecodeLandRequestID(payload []byte) (entity.RequestID, error) { - lr, err := entity.LandRequestFromBytes(payload) + lr, err := sqmq.UnmarshalLandRequest(payload) if err != nil { return entity.RequestID{}, err } @@ -50,17 +51,19 @@ func DecodeLandRequestID(payload []byte) (entity.RequestID, error) { // DecodeCancelRequestID extracts the request ID from a CancelRequest payload // (the shape used by the cancel topic). func DecodeCancelRequestID(payload []byte) (entity.RequestID, error) { - cr, err := entity.CancelRequestFromBytes(payload) + cr, err := sqmq.UnmarshalCancelRequest(payload) if err != nil { return entity.RequestID{}, err } return entity.RequestID{ID: cr.ID, Queue: cr.Queue}, nil } -// DecodeRequestID extracts the request ID from a RequestID payload (the shape -// used by the validate and batch topics). -func DecodeRequestID(payload []byte) (entity.RequestID, error) { - return entity.RequestIDFromBytes(payload) +// DecodeRequestID extracts the request ID from a validate or batch payload. +// primary is the originating topic (TopicKeyValidate or TopicKeyBatch), not the DLQ key. +func DecodeRequestID(primary consumer.TopicKey) RequestIDDecoder { + return func(payload []byte) (entity.RequestID, error) { + return sqmq.UnmarshalRequestID(primary, payload) + } } // requestController is the DLQ reconciler for request-scoped pipeline stages. diff --git a/submitqueue/orchestrator/controller/dlq/request_test.go b/submitqueue/orchestrator/controller/dlq/request_test.go index 94258a7c2..b130e7f39 100644 --- a/submitqueue/orchestrator/controller/dlq/request_test.go +++ b/submitqueue/orchestrator/controller/dlq/request_test.go @@ -24,6 +24,7 @@ import ( queue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" storagemock "github.com/uber/submitqueue/submitqueue/extension/storage/mock" @@ -37,7 +38,7 @@ func TestDLQRequestController_InterfaceAndAccessors(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestBatchStore().Return(noBatchAssociations(ctrl)).AnyTimes() - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") assert.Equal(t, "validate_dlq", c.Name()) assert.Equal(t, consumer.TopicKey("validate_dlq"), c.TopicKey()) @@ -65,7 +66,7 @@ func TestDLQRequestController_Process_LandRequestPayload(t *testing.T) { c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeLandRequestID, TopicKey(topickey.TopicKeyStart), "orchestrator-start-dlq") - payload, err := entity.LandRequest{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(entity.LandRequest{ID: "q/1", Queue: "q"})) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -75,9 +76,9 @@ func TestDLQRequestController_Process_LandRequestPayload(t *testing.T) { func TestDLQRequestController_Process_TenantPayloadQueueMismatchAcks(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockStorage(ctrl) - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "q/1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) @@ -104,7 +105,7 @@ func TestDLQRequestController_Process_CancelRequestPayload(t *testing.T) { c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeCancelRequestID, TopicKey(topickey.TopicKeyCancel), "orchestrator-cancel-dlq") - payload, err := entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"}.ToBytes() + payload, err := sqmq.Marshal(sqmq.CancelFromEntity(entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"})) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -131,9 +132,9 @@ func TestDLQRequestController_Process_RequestIDPayload(t *testing.T) { store.EXPECT().GetRequestBatchStore().Return(noBatchAssociations(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeRequestID, TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := entity.RequestID{ID: "q/3", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/3", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -153,9 +154,9 @@ func TestDLQRequestController_Process_DifferentTerminalOutcomeSkips(t *testing.T store.EXPECT().GetRequestBatchStore().Return(noBatchAssociations(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "q/1", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -170,7 +171,7 @@ func TestDLQRequestController_Process_MalformedPayloadFails(t *testing.T) { store.EXPECT().GetRequestBatchStore().Return(noBatchAssociations(ctrl)).AnyTimes() // no store calls expected - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") delivery := newMockDelivery(ctrl, []byte("not json")) err := c.Process(context.Background(), delivery) @@ -185,9 +186,9 @@ func TestDLQRequestController_Process_EmptyIDFails(t *testing.T) { store.EXPECT().GetRequestBatchStore().Return(noBatchAssociations(ctrl)).AnyTimes() // no store calls expected - c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") + c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") - payload, err := entity.RequestID{ID: "", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -255,9 +256,9 @@ func TestDLQRequestController_Process_SkipsRequestOwnedByLiveBatch(t *testing.T) store.EXPECT().GetRequestStore().Return(storagemock.NewMockRequestStore(ctrl)).AnyTimes() c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, - consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") + consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -294,9 +295,9 @@ func TestDLQRequestController_Process_FailsWhenEveryBatchIsTerminal(t *testing.T store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, - registry, DecodeRequestID, TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") + registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -338,9 +339,9 @@ func TestDLQRequestController_Process_FailsWhenCreatingBatchNeverClaimed(t *test store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, - registry, DecodeRequestID, TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") + registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -376,9 +377,9 @@ func TestDLQRequestController_Process_SkipsWhenCreatingBatchAlreadyClaimed(t *te store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, - consumer.TopicRegistry{}, DecodeRequestID, TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") + consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := entity.RequestID{ID: "q/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) diff --git a/submitqueue/orchestrator/controller/dlq/speculate.go b/submitqueue/orchestrator/controller/dlq/speculate.go index 1801ba31b..705fca4a1 100644 --- a/submitqueue/orchestrator/controller/dlq/speculate.go +++ b/submitqueue/orchestrator/controller/dlq/speculate.go @@ -24,6 +24,7 @@ import ( "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -86,7 +87,7 @@ func (c *speculateController) Process(ctx context.Context, delivery consumer.Del msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(primaryTopicKey(c.topicKey), msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to decode batch id from dlq payload: %w", err) @@ -200,7 +201,7 @@ func (c *speculateController) retrigger(ctx context.Context, store storage.Stora } } - payload, err := entity.BatchID{ID: next, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, next, queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/dlq/speculate_test.go b/submitqueue/orchestrator/controller/dlq/speculate_test.go index a23e2252b..ddc62e12c 100644 --- a/submitqueue/orchestrator/controller/dlq/speculate_test.go +++ b/submitqueue/orchestrator/controller/dlq/speculate_test.go @@ -24,6 +24,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" storagemock "github.com/uber/submitqueue/submitqueue/extension/storage/mock" @@ -44,7 +45,7 @@ func speculateRegistry(t *testing.T, ctrl *gomock.Controller, logPublishes int, logPublisher := queuemock.NewMockPublisher(ctrl) logPublisher.EXPECT().Publish(gomock.Any(), "log", gomock.Any()).DoAndReturn( func(_ context.Context, _ string, message entityqueue.Message) error { - entry, err := entity.RequestLogFromBytes(message.Payload) + entry, err := sqmq.UnmarshalRequestLog(message.Payload) require.NoError(t, err) if logs != nil { *logs = append(*logs, entry) @@ -58,7 +59,7 @@ func speculateRegistry(t *testing.T, ctrl *gomock.Controller, logPublishes int, specPublisher := queuemock.NewMockPublisher(ctrl) specPublisher.EXPECT().Publish(gomock.Any(), "speculate", gomock.Any()).DoAndReturn( func(_ context.Context, _ string, message entityqueue.Message) error { - bid, err := entity.BatchIDFromBytes(message.Payload) + bid, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, message.Payload) require.NoError(t, err) *captured = append(*captured, bid.ID) return nil @@ -148,7 +149,7 @@ func TestDLQSpeculateController_Process_Attribution(t *testing.T) { registry := speculateRegistry(t, ctrl, 1, &republished, &logs) c := newSpeculateController(registry, store, t) - payload, err := entity.BatchID{ID: "q/batch/named", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/named", "q") require.NoError(t, err) delivery := newMockDeliveryWithFailure(ctrl, payload, tt.recordedFailure, tt.failed) @@ -201,7 +202,7 @@ func TestDLQSpeculateController_Process_RetriggersQueue(t *testing.T) { registry := speculateRegistry(t, ctrl, 0, &republished, nil) c := newSpeculateController(registry, store, t) - payload, err := entity.BatchID{ID: "q/batch/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/1", "q") require.NoError(t, err) delivery := newMockDeliveryWithFailure(ctrl, payload, failure.New("boom"), true) @@ -240,7 +241,7 @@ func TestDLQSpeculateController_Process_NoRetriggerWithoutProgress(t *testing.T) registry := speculateRegistry(t, ctrl, 0, &republished, nil) c := newSpeculateController(registry, store, t) - payload, err := entity.BatchID{ID: "q/batch/1", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/1", "q") require.NoError(t, err) delivery := newMockDeliveryWithFailure(ctrl, payload, failure.New("boom"), true) @@ -275,7 +276,7 @@ func TestDLQSpeculateController_Process_TenantPayloadQueueMismatchAcks(t *testin store := storagemock.NewMockStorage(ctrl) c := newSpeculateController(consumer.TopicRegistry{}, store, t) - payload, err := entity.BatchID{ID: "q/batch/named", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/named", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) @@ -287,7 +288,7 @@ func TestDLQSpeculateController_Process_EmptyIDFails(t *testing.T) { c := newSpeculateController(consumer.TopicRegistry{}, store, t) - payload, err := entity.BatchID{ID: "", Queue: "q"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) diff --git a/submitqueue/orchestrator/controller/land/BUILD.bazel b/submitqueue/orchestrator/controller/land/BUILD.bazel index 4bde9554c..502628526 100644 --- a/submitqueue/orchestrator/controller/land/BUILD.bazel +++ b/submitqueue/orchestrator/controller/land/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", @@ -36,6 +37,7 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/land/land.go b/submitqueue/orchestrator/controller/land/land.go index 47552b0ac..696cf72f3 100644 --- a/submitqueue/orchestrator/controller/land/land.go +++ b/submitqueue/orchestrator/controller/land/land.go @@ -36,6 +36,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -97,7 +98,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize batch ID: %w", err) diff --git a/submitqueue/orchestrator/controller/land/land_test.go b/submitqueue/orchestrator/controller/land/land_test.go index 450f919df..2c05a02f3 100644 --- a/submitqueue/orchestrator/controller/land/land_test.go +++ b/submitqueue/orchestrator/controller/land/land_test.go @@ -34,13 +34,13 @@ import ( consumermock "github.com/uber/submitqueue/platform/consumer/mock" "github.com/uber/submitqueue/platform/errs" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" storagemock "github.com/uber/submitqueue/submitqueue/extension/storage/mock" ) -// batchIDPayload serializes a BatchID to JSON bytes for test message payloads. // staticStorageFactory resolves every queue to one fixed store aggregate. type staticStorageFactory struct{ store storage.Storage } @@ -48,7 +48,7 @@ type staticStorageFactory struct{ store storage.Storage } func (f staticStorageFactory) For(storage.Config) (storage.Storage, error) { return f.store, nil } func batchIDPayload(t *testing.T, id, queue string) []byte { - payload, err := entity.BatchID{ID: id, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, id, queue) require.NoError(t, err) return payload } @@ -306,7 +306,7 @@ func TestProcess_ReportsLandingBeforeDispatch(t *testing.T) { logs := rec.byTopic["log"] require.Len(t, logs, 2) for i, requestID := range []string{req1.ID, req2.ID} { - entry, err := entity.RequestLogFromBytes(logs[i].Payload) + entry, err := sqmq.UnmarshalRequestLog(logs[i].Payload) require.NoError(t, err) assert.Equal(t, requestID, entry.RequestID) assert.Equal(t, entity.RequestStatusLanding, entry.Status) diff --git a/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel b/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel index 442f71002..589c316ae 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel +++ b/submitqueue/orchestrator/controller/landconflictsignal/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -32,6 +33,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go index bb4ce0843..46178bbf4 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal.go @@ -30,6 +30,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -206,7 +207,7 @@ func (c *Controller) failRequest(ctx context.Context, store storage.Storage, req // per conflict-check result, so a redelivery that re-hands it is meant to dedup // away. func (c *Controller) publishRequestID(ctx context.Context, key consumer.TopicKey, requestID string, queue string) error { - payload, err := entity.RequestID{ID: requestID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(key, requestID, queue) if err != nil { return fmt.Errorf("failed to serialize request ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index 9a372c1fe..64093cbfb 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -28,6 +28,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -125,14 +126,14 @@ func TestProcess_LandablePublishesToBatch(t *testing.T) { // First publish: validated log entry. assert.Equal(t, "log", gotTopics[0]) - logEntry, err := entity.RequestLogFromBytes(gotPayloads[0]) + logEntry, err := sqmq.UnmarshalRequestLog(gotPayloads[0]) require.NoError(t, err) assert.Equal(t, entity.RequestStatusValidated, logEntry.Status) assert.Equal(t, int32(2), logEntry.RequestVersion) // Second publish: request ID to batch topic. assert.Equal(t, "batch", gotTopics[1]) - rid, err := entity.RequestIDFromBytes(gotPayloads[1]) + rid, err := sqmq.UnmarshalRequestID(sqmq.TopicKeyBatch, gotPayloads[1]) require.NoError(t, err) assert.Equal(t, testRequestID, rid.ID) } @@ -181,7 +182,7 @@ func TestProcess_NotLandableMarksRequestError(t *testing.T) { // The single publish is the terminal log entry carrying the conflict reason. assert.Equal(t, "log", gotTopic) - logEntry, err := entity.RequestLogFromBytes(gotPayload) + logEntry, err := sqmq.UnmarshalRequestLog(gotPayload) require.NoError(t, err) assert.Equal(t, entity.RequestStatusError, logEntry.Status) assert.Equal(t, int32(2), logEntry.RequestVersion) diff --git a/submitqueue/orchestrator/controller/landsignal/BUILD.bazel b/submitqueue/orchestrator/controller/landsignal/BUILD.bazel index 5ff7a0416..855374dab 100644 --- a/submitqueue/orchestrator/controller/landsignal/BUILD.bazel +++ b/submitqueue/orchestrator/controller/landsignal/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//platform/metrics:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/landsignal/landsignal.go b/submitqueue/orchestrator/controller/landsignal/landsignal.go index c4efad351..efc52365d 100644 --- a/submitqueue/orchestrator/controller/landsignal/landsignal.go +++ b/submitqueue/orchestrator/controller/landsignal/landsignal.go @@ -33,6 +33,7 @@ import ( "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -206,7 +207,7 @@ func (c *Controller) fanout(ctx context.Context, batchID, queue, failureReason s // with and partitioned by the batch's queue. metadata rides the message as // side-band headers (nil for none). func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, msgID, batchID, queue string, metadata map[string]string) error { - payload, err := entity.BatchID{ID: batchID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(key, batchID, queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/speculate/BUILD.bazel b/submitqueue/orchestrator/controller/speculate/BUILD.bazel index c223ee7e2..0d9ada4c3 100644 --- a/submitqueue/orchestrator/controller/speculate/BUILD.bazel +++ b/submitqueue/orchestrator/controller/speculate/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//platform/metrics:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -48,6 +49,7 @@ go_test( "//platform/consumer:go_default_library", "//platform/consumer/mock:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/speculation/speculator:go_default_library", diff --git a/submitqueue/orchestrator/controller/speculate/run_test.go b/submitqueue/orchestrator/controller/speculate/run_test.go index 3525de0d4..32597f238 100644 --- a/submitqueue/orchestrator/controller/speculate/run_test.go +++ b/submitqueue/orchestrator/controller/speculate/run_test.go @@ -25,6 +25,7 @@ import ( entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/speculation/speculator" @@ -172,7 +173,7 @@ func newRunHarness(t *testing.T, ctrl *gomock.Controller, spec *scriptedSpeculat return assert.AnError } if topic == "log" { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) h.logs = append(h.logs, entry) return nil @@ -267,7 +268,7 @@ func TestRun_DispatchStampsQueueAndPartitionsByHead(t *testing.T) { require.NoError(t, h.run(head)) require.Len(t, h.messages, 1) - got, err := entity.BatchIDFromBytes(h.messages[0].Payload) + got, err := sqmq.UnmarshalBatchID(sqmq.TopicKeySpeculate, h.messages[0].Payload) require.NoError(t, err) assert.Equal(t, head, got.ID) assert.Equal(t, "q", got.Queue, "the payload must name the real queue, not the partition key") diff --git a/submitqueue/orchestrator/controller/speculate/speculate.go b/submitqueue/orchestrator/controller/speculate/speculate.go index 93bdcd2a3..fb6ed0fef 100644 --- a/submitqueue/orchestrator/controller/speculate/speculate.go +++ b/submitqueue/orchestrator/controller/speculate/speculate.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -95,7 +96,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - bid, err := entity.BatchIDFromBytes(msg.Payload) + bid, err := sqmq.UnmarshalBatchID(c.topicKey, msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) return fmt.Errorf("failed to deserialize batch ID: %w", err) @@ -235,7 +236,7 @@ func (c *Controller) publishBatchID(ctx context.Context, key consumer.TopicKey, // attached to the delivery (nil for none). Used to carry a failed batch's reason // to conclude without persisting it as batch state. func (c *Controller) publishBatchIDWithMetadata(ctx context.Context, key consumer.TopicKey, msgID, batchID, queue, partitionKey string, metadata map[string]string) error { - payload, err := entity.BatchID{ID: batchID, Queue: queue}.ToBytes() + payload, err := sqmq.MarshalID(key, batchID, queue) if err != nil { return fmt.Errorf("failed to serialize batch ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/speculate/speculate_test.go b/submitqueue/orchestrator/controller/speculate/speculate_test.go index 06b9e14f8..020e82628 100644 --- a/submitqueue/orchestrator/controller/speculate/speculate_test.go +++ b/submitqueue/orchestrator/controller/speculate/speculate_test.go @@ -26,6 +26,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" consumermock "github.com/uber/submitqueue/platform/consumer/mock" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/speculation/speculator" @@ -84,7 +85,7 @@ func (h *procHarness) listsInFlight(batches ...entity.Batch) { func batchIDPayload(t *testing.T, id string) []byte { t.Helper() - payload, err := entity.BatchID{ID: id, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, id, "test-queue") require.NoError(t, err) return payload } @@ -141,7 +142,7 @@ func newProcHarness(t *testing.T, ctrl *gomock.Controller, publishErr error) *pr return publishErr } if topic == "log" { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) h.logs = append(h.logs, entry) return nil diff --git a/submitqueue/orchestrator/controller/start/BUILD.bazel b/submitqueue/orchestrator/controller/start/BUILD.bazel index dbd2ced1c..3038163b2 100644 --- a/submitqueue/orchestrator/controller/start/BUILD.bazel +++ b/submitqueue/orchestrator/controller/start/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", @@ -31,6 +32,7 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/submitqueue/orchestrator/controller/start/start.go b/submitqueue/orchestrator/controller/start/start.go index d6b54deee..8d3a1f7dc 100644 --- a/submitqueue/orchestrator/controller/start/start.go +++ b/submitqueue/orchestrator/controller/start/start.go @@ -24,6 +24,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" @@ -76,7 +77,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er msg := delivery.Message() - landRequest, err := entity.LandRequestFromBytes(msg.Payload) + landRequest, err := sqmq.UnmarshalLandRequest(msg.Payload) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "deserialize_errors", 1) // Non-retryable: malformed messages will never succeed regardless of retry count @@ -147,8 +148,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // The request ID is the message ID with no cause: a request is handed to the // next stage once, so a redelivery that re-hands it is meant to dedup away. func (c *Controller) publish(ctx context.Context, key consumer.TopicKey, requestID string, queue string) error { - rid := entity.RequestID{ID: requestID, Queue: queue} - payload, err := rid.ToBytes() + payload, err := sqmq.MarshalID(key, requestID, queue) if err != nil { return fmt.Errorf("failed to serialize request ID: %w", err) } diff --git a/submitqueue/orchestrator/controller/start/start_test.go b/submitqueue/orchestrator/controller/start/start_test.go index 6b47f9363..196f377cb 100644 --- a/submitqueue/orchestrator/controller/start/start_test.go +++ b/submitqueue/orchestrator/controller/start/start_test.go @@ -29,6 +29,7 @@ import ( consumermock "github.com/uber/submitqueue/platform/consumer/mock" "github.com/uber/submitqueue/platform/errs" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -86,7 +87,7 @@ func newMockStorage(ctrl *gomock.Controller) *storagemock.MockStorage { // makeDelivery builds a MockDelivery wrapping a serialized LandRequest. func makeDelivery(t *testing.T, ctrl *gomock.Controller, lr entity.LandRequest) *consumermock.MockDelivery { - payload, err := lr.ToBytes() + payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(lr)) require.NoError(t, err) msg := entityqueue.NewMessage(lr.ID, payload, lr.Queue, nil) @@ -125,7 +126,7 @@ func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) controller := newTestController(t, ctrl, newMockStorage(ctrl), nil) request := entity.LandRequest{ID: "test-queue/123", Queue: "test-queue"} - payload, err := request.ToBytes() + payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(request)) require.NoError(t, err) msg := entityqueue.NewMessage(request.ID, payload, request.Queue, nil) msg.Tenant = "other-queue" diff --git a/submitqueue/orchestrator/controller/validate/BUILD.bazel b/submitqueue/orchestrator/controller/validate/BUILD.bazel index 11ba47349..c07f3074f 100644 --- a/submitqueue/orchestrator/controller/validate/BUILD.bazel +++ b/submitqueue/orchestrator/controller/validate/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//platform/consumer:go_default_library", "//platform/metrics:go_default_library", "//platform/publish:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/request:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/changeprovider:go_default_library", @@ -38,6 +39,7 @@ go_test( "//platform/consumer/mock:go_default_library", "//platform/errs:go_default_library", "//platform/extension/messagequeue/mock:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/changeprovider/mock:go_default_library", diff --git a/submitqueue/orchestrator/controller/validate/validate.go b/submitqueue/orchestrator/controller/validate/validate.go index 161b434b9..d8f3313f1 100644 --- a/submitqueue/orchestrator/controller/validate/validate.go +++ b/submitqueue/orchestrator/controller/validate/validate.go @@ -29,6 +29,7 @@ import ( "github.com/uber/submitqueue/platform/consumer" coremetrics "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/platform/publish" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" corerequest "github.com/uber/submitqueue/submitqueue/core/request" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/changeprovider" @@ -92,8 +93,7 @@ func NewController( func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) error { msg := delivery.Message() - // Deserialize request ID from payload - rid, err := entity.RequestIDFromBytes(msg.Payload) + rid, err := sqmq.UnmarshalRequestID(c.topicKey, msg.Payload) if err != nil { coremetrics.NamedCounter(c.metricsScope, "process", "deserialize_errors", 1) return fmt.Errorf("failed to deserialize request ID: %w", err) diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index f009207c7..3fb384578 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -31,6 +31,7 @@ import ( consumermock "github.com/uber/submitqueue/platform/consumer/mock" "github.com/uber/submitqueue/platform/errs" queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" changeprovidermock "github.com/uber/submitqueue/submitqueue/extension/changeprovider/mock" @@ -75,7 +76,7 @@ func requestWithState(request entity.Request, state entity.RequestState) entity. // requestIDPayload serializes a RequestID to JSON bytes for test message payloads. func requestIDPayload(t *testing.T, id string) []byte { - payload, err := entity.RequestID{ID: id, Queue: "test-queue"}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, id, "test-queue") require.NoError(t, err) return payload } @@ -710,7 +711,7 @@ func TestController_Process_CustomValidatorFails(t *testing.T) { mockPub := queuemock.NewMockPublisher(ctrl) mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - log, err := entity.RequestLogFromBytes(msg.Payload) + log, err := sqmq.UnmarshalRequestLog(msg.Payload) require.NoError(t, err) gotLog = log return nil @@ -774,7 +775,7 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te // before validation ever ran and this test would stop covering termination. mockPub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( func(_ context.Context, _ string, msg entityqueue.Message) error { - entry, err := entity.RequestLogFromBytes(msg.Payload) + entry, err := sqmq.UnmarshalRequestLog(msg.Payload) if err == nil && entry.Status == entity.RequestStatusError { return fmt.Errorf("publish boom") } diff --git a/submitqueue/orchestrator/pipeline.go b/submitqueue/orchestrator/pipeline.go index b85defbaf..23d6630f7 100644 --- a/submitqueue/orchestrator/pipeline.go +++ b/submitqueue/orchestrator/pipeline.go @@ -131,7 +131,7 @@ var Stages = []pipeline.Stage[Deps]{ return validate.NewController(d.Logger, d.Scope, d.Storage, sc.Registry, d.ChangeProvider, d.Validator, runwaymq.TopicKeyMergeConflictCheck, sc.TopicKey, sc.ConsumerGroup), nil }, DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return dlq.NewDLQRequestController(d.Logger, d.Scope, d.Storage, sc.Registry, dlq.DecodeRequestID, sc.TopicKey, sc.ConsumerGroup), nil + return dlq.NewDLQRequestController(d.Logger, d.Scope, d.Storage, sc.Registry, dlq.DecodeRequestID(topickey.TopicKeyValidate), sc.TopicKey, sc.ConsumerGroup), nil }, }, { @@ -153,7 +153,7 @@ var Stages = []pipeline.Stage[Deps]{ return batch.NewController(d.Logger, d.Scope, sc.Registry, d.Counter, d.Storage, sc.TopicKey, sc.ConsumerGroup), nil }, DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { - return dlq.NewDLQRequestController(d.Logger, d.Scope, d.Storage, sc.Registry, dlq.DecodeRequestID, sc.TopicKey, sc.ConsumerGroup), nil + return dlq.NewDLQRequestController(d.Logger, d.Scope, d.Storage, sc.Registry, dlq.DecodeRequestID(topickey.TopicKeyBatch), sc.TopicKey, sc.ConsumerGroup), nil }, }, { diff --git a/test/e2e/submitqueue/BUILD.bazel b/test/e2e/submitqueue/BUILD.bazel index fb03d6cad..502f9f46f 100644 --- a/test/e2e/submitqueue/BUILD.bazel +++ b/test/e2e/submitqueue/BUILD.bazel @@ -47,6 +47,7 @@ go_test( "//platform/git/exectest:go_default_library", "//platform/publish:go_default_library", "//submitqueue/core/batch:go_default_library", + "//submitqueue/core/messagequeue:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/entity:go_default_library", "//submitqueue/extension/storage:go_default_library", diff --git a/test/e2e/submitqueue/harness_test.go b/test/e2e/submitqueue/harness_test.go index 68427ef0d..9288e1f4e 100644 --- a/test/e2e/submitqueue/harness_test.go +++ b/test/e2e/submitqueue/harness_test.go @@ -41,6 +41,7 @@ import ( queuemysql "github.com/uber/submitqueue/platform/extension/messagequeue/mysql" "github.com/uber/submitqueue/platform/publish" corebatch "github.com/uber/submitqueue/submitqueue/core/batch" + sqmq "github.com/uber/submitqueue/submitqueue/core/messagequeue" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/entity" "github.com/uber/submitqueue/submitqueue/extension/storage" @@ -297,7 +298,7 @@ func (s *E2EIntegrationSuite) redeliverBatchMessage(req request) { }) require.NoError(t, err) - payload, err := entity.RequestID{ID: req.sqid, Queue: req.queue}.ToBytes() + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, req.sqid, req.queue) require.NoError(t, err) require.NoError(t, publish.Message(entityqueue.WithQueueName(s.ctx, req.queue), registry, topickey.TopicKeyBatch, publish.MessageParams{ diff --git a/tool/proto/BUILD.bazel b/tool/proto/BUILD.bazel index 16bc485cf..8c6037e11 100644 --- a/tool/proto/BUILD.bazel +++ b/tool/proto/BUILD.bazel @@ -73,6 +73,31 @@ go_proto_generated_files( out_dir = "api_stovepipe", ) +# SubmitQueue internal queue contract (message-only, no RPC service). +go_proto_generated_files( + name = "submitqueue_core_messagequeue", + srcs = [ + "//submitqueue/core/messagequeue/proto:batch.proto", + "//submitqueue/core/messagequeue/proto:cancel.proto", + "//submitqueue/core/messagequeue/proto:conclude.proto", + "//submitqueue/core/messagequeue/proto:dependencyanalysis.proto", + "//submitqueue/core/messagequeue/proto:log.proto", + "//submitqueue/core/messagequeue/proto:speculate.proto", + "//submitqueue/core/messagequeue/proto:start.proto", + "//submitqueue/core/messagequeue/proto:submitqueuebuild.proto", + "//submitqueue/core/messagequeue/proto:submitqueuebuildsignal.proto", + "//submitqueue/core/messagequeue/proto:submitqueuemerge.proto", + "//submitqueue/core/messagequeue/proto:validate.proto", + ], + gen_services = False, + imports = [ + "//api/base/change/proto:change.proto", + "//api/base/mergestrategy/proto:mergestrategy.proto", + "//api/base/messagequeue/proto:messagequeue.proto", + ], + out_dir = "submitqueue_core_messagequeue", +) + # Stovepipe internal queue contract (message-only, no RPC service). go_proto_generated_files( name = "stovepipe_core_messagequeue", @@ -102,5 +127,6 @@ filegroup( ":api_submitqueue_gateway", ":api_submitqueue_orchestrator", ":stovepipe_core_messagequeue", + ":submitqueue_core_messagequeue", ], )