feat(messagequeue): level the queue's own logs separately

## Summary

### Why?

The services run at debug, and the queue logs a line per message published, fetched, leased and acked. Reading a service's logs means scrolling past hundreds of `fetched messages {"count": 0}` to find the one line that matters — following a request through the pipeline is impractical at the level everything else is useful at.

Turning the whole service down to info loses the output that is actually being read. The queue's chatter is the part worth silencing, not the service's.

### What?

`Params.LogLevel` sets the minimum level for the queue's own logs, applied with `zap.IncreaseLevel` so the rest of the service keeps the level it was built with. Empty selects info, which is what stops the chatter by default.

The four servers pass `QUEUE_LOG_LEVEL` through, and the compose files forward it, so the queue can be turned back up for a run that is chasing a message that never arrived:

```
QUEUE_LOG_LEVEL=debug make local-submitqueue-start
```

The level is a raw string on `Params`, parsed in the package, which keeps the environment lookup in the wiring layer where the other knobs live. `IncreaseLevel` can only raise the level, never lower it, so this cannot be used to make a quiet service verbose — an unparseable value is rejected at construction rather than silently ignored.

## Test Plan

✅ `bazel test //platform/extension/messagequeue/mysql:go_default_test` — the level is accepted, and a value that is not a level fails construction.

✅ `make test` — 98 targets.

✅ Passthrough verified rather than assumed, since compose forwards only what it declares: `QUEUE_LOG_LEVEL=debug docker compose -f service/submitqueue/docker-compose.yml config` shows the variable set on all three services.

Author: behinddwalls

doc/howto/PROVIDER-E2E.md

@@ -134,6 +134,14 @@ Runway logs each merge and each head-branch move:
 moved change head branch to its landed commit  {"change": "you/repo#1", "branch": "refs/heads/feature-a", ...}
 ```
 
+The message queue logs a line per message published, fetched, leased and acked, which at debug level buries everything else a service says. It is levelled separately from the rest of the service, at info by default. To follow the queue itself — chasing a message that never arrived, or a partition that never got leased — turn it back up for the services you care about:
+
+```bash
+QUEUE_LOG_LEVEL=debug make local-submitqueue-start
+```
+
+`QUEUE_LOG_LEVEL` takes any zap level name. It can only raise the queue's level above the one the service logger was built with, never lower it, so it cannot be used to make a quiet service verbose.
+
 ## When it does not work
 
 **The push is rejected on the first try.** Branch protection on `main` — required status checks, or a linear-history or no-force-push rule — applies to the merger like anyone else. Either relax it on the scratch repo or add the token's identity to the bypass list.

platform/extension/messagequeue/mysql/BUILD.bazel

@@ -26,6 +26,7 @@ go_library(
         "@com_github_uber_go_tally//:go_default_library",
         "@org_uber_go_mock//gomock:go_default_library",
         "@org_uber_go_zap//:go_default_library",
+        "@org_uber_go_zap//zapcore:go_default_library",
     ],
 )
 

platform/extension/messagequeue/mysql/sql.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/uber-go/tally"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 
 	extqueue "github.com/uber/submitqueue/platform/extension/messagequeue"
 )
@@ -40,6 +41,16 @@ type Params struct {
 	// Logger for debugging and observability (required)
 	Logger *zap.Logger
 
+	// LogLevel is the minimum level for the queue's own logs, as a zap level
+	// name ("debug", "info", ...). Empty selects info.
+	//
+	// The queue logs a line per message published, fetched, leased and acked,
+	// which at debug buries everything else a service says. Levelling it here
+	// rather than at the service logger keeps the rest of that service's debug
+	// output intact. The level can only be raised above the one the supplied
+	// logger was built with, never lowered.
+	LogLevel string
+
 	// MetricsScope for metrics collection (required)
 	MetricsScope tally.Scope
 
@@ -55,8 +66,17 @@ func NewQueue(params Params) (extqueue.Queue, error) {
 		return nil, fmt.Errorf("failed to ping database: %w", err)
 	}
 
-	logger := params.Logger.Sugar().Named("queue_mysql")
-	logger.Infow("created SQL queue")
+	level := zapcore.InfoLevel
+	if params.LogLevel != "" {
+		parsed, err := zapcore.ParseLevel(params.LogLevel)
+		if err != nil {
+			return nil, fmt.Errorf("invalid queue log level %q: %w", params.LogLevel, err)
+		}
+		level = parsed
+	}
+
+	logger := params.Logger.WithOptions(zap.IncreaseLevel(level)).Sugar().Named("queue_mysql")
+	logger.Infow("created SQL queue", "log_level", level.String())
 
 	// Create stores
 	messageStore := newMessageStore(params.DB, logger, params.MetricsScope)

platform/extension/messagequeue/mysql/sql_test.go

@@ -70,6 +70,44 @@ func TestNewQueue(t *testing.T) {
 
 		require.NoError(t, mock.ExpectationsWereMet())
 	})
+	t.Run("accepts a log level", func(t *testing.T) {
+		db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true))
+		require.NoError(t, err)
+		defer db.Close()
+
+		mock.ExpectPing()
+
+		q, err := NewQueue(Params{
+			DB:           db,
+			Logger:       zaptest.NewLogger(t),
+			LogLevel:     "debug",
+			MetricsScope: tally.NewTestScope("test", nil),
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, q)
+		assert.NoError(t, q.Close())
+
+		require.NoError(t, mock.ExpectationsWereMet())
+	})
+
+	t.Run("error when the log level is not a level", func(t *testing.T) {
+		db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true))
+		require.NoError(t, err)
+		defer db.Close()
+
+		mock.ExpectPing()
+
+		q, err := NewQueue(Params{
+			DB:           db,
+			Logger:       zaptest.NewLogger(t),
+			LogLevel:     "loud",
+			MetricsScope: tally.NewTestScope("test", nil),
+		})
+
+		require.Error(t, err)
+		assert.Nil(t, q)
+	})
 }
 
 func TestQueue_Publisher(t *testing.T) {

service/runway/server/docker-compose.yml

@@ -49,6 +49,9 @@ services:
       - MERGER=${SQ_RUNWAY_MERGER:-}
       # Queue infrastructure connection
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       - HOSTNAME=runway-dev
     depends_on:
       mysql-queue:

service/runway/server/main.go

@@ -138,6 +138,7 @@ func run() error {
 	mysqlQueue, err := queueMySQL.NewQueue(queueMySQL.Params{
 		DB:           queueDB,
 		Logger:       logger,
+		LogLevel:     os.Getenv("QUEUE_LOG_LEVEL"),
 		MetricsScope: scope.SubScope("queue"),
 	})
 	if err != nil {

service/stovepipe/docker-compose.yml

@@ -67,6 +67,9 @@ services:
       - PORT=:8080
       - STORAGE_MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       - HOSTNAME=stovepipe-dev
     depends_on:
       mysql-app:

service/stovepipe/server/main.go

@@ -237,6 +237,7 @@ func run() error {
 	mysqlQueue, err := queueMySQL.NewQueue(queueMySQL.Params{
 		DB:           queueDB,
 		Logger:       logger,
+		LogLevel:     os.Getenv("QUEUE_LOG_LEVEL"),
 		MetricsScope: scope.SubScope("queue"),
 	})
 	if err != nil {

service/submitqueue/docker-compose.yml

@@ -73,6 +73,9 @@ services:
       - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       # Queue infrastructure connection (separate database)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       # Path to YAML queue configuration baked into the image
       - QUEUE_CONFIG_PATH=/app/queues.yaml
       # Stable subscriber name for the request-log consumer
@@ -101,6 +104,9 @@ services:
       - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       # Queue infrastructure connection (separate database)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       - HOSTNAME=orchestrator-dev
       # Consumer-gate state shared with the host (see header comment)
       - CONSUMER_GATE_DIR=/var/submitqueue/consumergate
@@ -129,6 +135,9 @@ services:
       - PORT=:8080
       # Queue infrastructure connection (shared with the orchestrator)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       - HOSTNAME=runway-dev
       # Consumer-gate state shared with the host (see header comment)
       - CONSUMER_GATE_DIR=/var/submitqueue/consumergate

service/submitqueue/gateway/server/docker-compose.yml

@@ -62,6 +62,9 @@ services:
       - MYSQL_DSN=root:root@tcp(mysql-app:3306)/submitqueue?parseTime=true
       # Queue infrastructure connection (separate database)
       - QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
+      # Level for the queue's own logs; info by default so its per-message
+      # chatter does not bury the rest of the service at debug.
+      - QUEUE_LOG_LEVEL=${QUEUE_LOG_LEVEL:-}
       # Path to YAML queue configuration baked into the image
       - QUEUE_CONFIG_PATH=/app/queues.yaml
       # Stable subscriber name for the request-log consumer