From 281ba9aec9d02a533c05d77d5715de199fe0ebed Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 5 Aug 2026 20:30:53 -0500 Subject: [PATCH 1/6] [History Server] Fix cold-load CPU starvation and per-event logging Loading a dead session is CPU-bound, but the sample manifest sets only `limits.cpu: "500m"`, so Kubernetes pins requests there too and the load saturates the quota for its entire duration. Measured on kind across 13 runs, the container sat at 0.42-0.50 cores every time. Raising the limit to 4 (requests stay at 500m, so scheduling cost is unchanged): | tasks in session | 500m | 4 cores | |---|---|---| | 50,000 | 97.9s | 30.5s | | 100,000 | 907.3s | 62.7s | This also removes what looked like superlinear degradation past 50k tasks: at 500m the per-task cost went from 1.96ms to 9.07ms between 50k and 100k, while at 4 cores it is flat (0.61ms and 0.63ms). The load uses ~1.2 cores on average and peaks at ~2.2, because Go's GC runs concurrently and needs cores of its own. The second change drops the per-event log line in storeEvent to Debug. It runs once per event, so a 100k-task session writes ~436,000 INFO lines per cold load, and the binary never calls logrus.SetLevel, so there is no way to turn it off. Worth ~13% of load time at 50k tasks (97.9s -> 85.3s). --- historyserver/config/historyserver.yaml | 7 ++++++- historyserver/pkg/eventserver/eventserver.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index f07d45d34ee..46000d59e33 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -64,5 +64,10 @@ spec: ports: - containerPort: 8080 resources: - limits: + requests: cpu: "500m" + limits: + # Loading a dead session is CPU-bound: it uses ~1.2 cores with bursts + # to ~2.2. A 500m cap is saturated for the whole load, which made a + # 100k-task session take 907s instead of 63s. + cpu: "4" diff --git a/historyserver/pkg/eventserver/eventserver.go b/historyserver/pkg/eventserver/eventserver.go index 1b1f81ddfa6..920bc25d385 100644 --- a/historyserver/pkg/eventserver/eventserver.go +++ b/historyserver/pkg/eventserver/eventserver.go @@ -133,7 +133,7 @@ func (h *EventHandler) storeEvent(clusterSessionKey string, eventMap map[string] } eventType := types.EventType(eventTypeStr) - logrus.Infof("current eventType: %v", eventType) + logrus.Debugf("current eventType: %v", eventType) switch eventType { case types.TASK_DEFINITION_EVENT: return h.handleTaskDefinitionEvent(eventMap, clusterSessionKey, false) From e4a26d1ae0fff5158b72ec3eea1570b6042b4d34 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Sun, 9 Aug 2026 12:06:00 -0500 Subject: [PATCH 2/6] historyserver: leave CPU limit configurable Signed-off-by: Future-Outlier --- historyserver/config/historyserver.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index 46000d59e33..80a87c360fc 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -66,8 +66,3 @@ spec: resources: requests: cpu: "500m" - limits: - # Loading a dead session is CPU-bound: it uses ~1.2 cores with bursts - # to ~2.2. A 500m cap is saturated for the whole load, which made a - # 100k-task session take 907s instead of 63s. - cpu: "4" From 7b4833e5c1f8702e134eef201391b6acefed2c04 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 10 Aug 2026 12:58:53 -0500 Subject: [PATCH 3/6] historyserver: size sample memory for session cache Signed-off-by: Future-Outlier --- historyserver/config/historyserver.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index 80a87c360fc..6c781c8d164 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -66,3 +66,7 @@ spec: resources: requests: cpu: "500m" + memory: "2Gi" + limits: + # Tune with --session-cache-max-bytes and concurrent session reads. + memory: "4Gi" From 6e533d6d44be82bc0f5548728b6edeee88ba2fc8 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 10 Aug 2026 14:16:27 -0500 Subject: [PATCH 4/6] historyserver: repair tests after cache changes Signed-off-by: Future-Outlier --- .../pkg/historyserver/byte_cache_integration_test.go | 11 ++++++----- historyserver/pkg/historyserver/enter_cluster_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/historyserver/pkg/historyserver/byte_cache_integration_test.go b/historyserver/pkg/historyserver/byte_cache_integration_test.go index f2af993ce13..18f1c01b059 100644 --- a/historyserver/pkg/historyserver/byte_cache_integration_test.go +++ b/historyserver/pkg/historyserver/byte_cache_integration_test.go @@ -29,9 +29,13 @@ func TestByteCache_ReadEndpointsRoundTripIsLossless(t *testing.T) { session = "session_2026-04-22_10-00-00_000000_1" ) + cluster := utils.ClusterInfo{ + Namespace: ns, Name: name, SessionName: session, + OwnerKind: "RayJob", OwnerName: "job-bc", + } handler := &ServerHandler{ maxClusters: 100, - clustersMap: make(map[utils.ClusterKey][]utils.ClusterInfo), + reader: &mockStorageReader{clusters: []utils.ClusterInfo{cluster}}, } fp := &fakeProcessor{ fn: func(_ context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) { @@ -40,9 +44,6 @@ func TestByteCache_ReadEndpointsRoundTripIsLossless(t *testing.T) { }, } handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) - handler.clustersMap[utils.ClusterKey{Namespace: ns, Name: name}] = []utils.ClusterInfo{ - {Namespace: ns, Name: name, SessionName: session, OwnerKind: "RayJob", OwnerName: "job-bc"}, - } routerRayClusterSet(handler) routerNodes(handler) @@ -51,7 +52,7 @@ func TestByteCache_ReadEndpointsRoundTripIsLossless(t *testing.T) { routerLogical(handler) container := restful.DefaultContainer - enterURL := "/enter_cluster/" + ns + "/" + name + "/" + session + enterURL := "/enter_cluster/" + ns + "/raycluster/" + name + "/" + session enterReq := httptest.NewRequest(http.MethodGet, enterURL, nil) enterResp := httptest.NewRecorder() container.ServeHTTP(enterResp, enterReq) diff --git a/historyserver/pkg/historyserver/enter_cluster_test.go b/historyserver/pkg/historyserver/enter_cluster_test.go index 215e7b58579..84a8f1afe26 100644 --- a/historyserver/pkg/historyserver/enter_cluster_test.go +++ b/historyserver/pkg/historyserver/enter_cluster_test.go @@ -275,7 +275,7 @@ func TestEnterClusterLatestFromStorage(t *testing.T) { return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil }, } - handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL) + handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) routerRayClusterSet(handler) container := restful.DefaultContainer @@ -340,7 +340,7 @@ func TestEnterClusterLatestPrioritizesLive(t *testing.T) { return SessionStatusLive, nil, nil }, } - handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL) + handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) routerRayClusterSet(handler) container := restful.DefaultContainer @@ -392,7 +392,7 @@ func TestEnterClusterReturnsNotFoundWhenRemovedFromStorage(t *testing.T) { return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil }, } - handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL) + handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) routerRayClusterSet(handler) container := restful.DefaultContainer @@ -450,7 +450,7 @@ func TestEnterClusterReturnsErrorOnTransientK8sError(t *testing.T) { return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil }, } - handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL) + handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) routerRayClusterSet(handler) container := restful.DefaultContainer @@ -508,7 +508,7 @@ func TestEnterClusterRayJobAndRayService(t *testing.T) { return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil }, } - handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL) + handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheMaxBytes, DefaultSessionCacheTTL) routerRayClusterSet(handler) container := restful.DefaultContainer From 0e1b497dc0465680dfb7c0ddf6edd5df5433d038 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 10 Aug 2026 14:33:34 -0500 Subject: [PATCH 5/6] historyserver: size example cache memory Signed-off-by: Future-Outlier --- historyserver/config/historyserver-azureblob.yaml | 8 +++++++- historyserver/config/historyserver-gcs.yaml | 8 +++++++- historyserver/config/historyserver.yaml | 6 ++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/historyserver/config/historyserver-azureblob.yaml b/historyserver/config/historyserver-azureblob.yaml index 2c1a72d4771..8cd5c6e04d0 100644 --- a/historyserver/config/historyserver-azureblob.yaml +++ b/historyserver/config/historyserver-azureblob.yaml @@ -46,6 +46,12 @@ spec: - --ray-root-dir=log ports: - containerPort: 8080 + # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + args: + - --session-cache-max-bytes=8589934592 resources: - limits: + requests: cpu: "500m" + memory: "2Gi" + limits: + memory: "12Gi" diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index a678356a2aa..c74a3f711aa 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -44,6 +44,12 @@ spec: imagePullPolicy: IfNotPresent ports: - containerPort: 8080 + # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + args: + - --session-cache-max-bytes=8589934592 resources: - limits: + requests: cpu: "500m" + memory: "2Gi" + limits: + memory: "12Gi" diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index 02fe164c3e7..f9cbe2c2ad9 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -65,10 +65,12 @@ spec: # - --use-auth-token-mode=true ports: - containerPort: 8080 + # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + args: + - --session-cache-max-bytes=8589934592 resources: requests: cpu: "500m" memory: "2Gi" limits: - # Tune with --session-cache-max-bytes and concurrent session reads. - memory: "4Gi" + memory: "12Gi" From b4e934fc4a5f912a56f632e0dfb52ab724185ae1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 10 Aug 2026 14:53:48 -0500 Subject: [PATCH 6/6] historyserver: document example memory sizing Signed-off-by: Future-Outlier --- historyserver/config/historyserver-azureblob.yaml | 3 ++- historyserver/config/historyserver-gcs.yaml | 3 ++- historyserver/config/historyserver.yaml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/historyserver/config/historyserver-azureblob.yaml b/historyserver/config/historyserver-azureblob.yaml index 8cd5c6e04d0..8b33160269e 100644 --- a/historyserver/config/historyserver-azureblob.yaml +++ b/historyserver/config/historyserver-azureblob.yaml @@ -46,7 +46,8 @@ spec: - --ray-root-dir=log ports: - containerPort: 8080 - # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + # Cache is soft-bounded at 8 GiB; 12 GiB leaves decode headroom. + # The 2 GiB request keeps examples schedulable; size production requests to residency. args: - --session-cache-max-bytes=8589934592 resources: diff --git a/historyserver/config/historyserver-gcs.yaml b/historyserver/config/historyserver-gcs.yaml index c74a3f711aa..8bb4a7dea64 100644 --- a/historyserver/config/historyserver-gcs.yaml +++ b/historyserver/config/historyserver-gcs.yaml @@ -44,7 +44,8 @@ spec: imagePullPolicy: IfNotPresent ports: - containerPort: 8080 - # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + # Cache is soft-bounded at 8 GiB; 12 GiB leaves decode headroom. + # The 2 GiB request keeps examples schedulable; size production requests to residency. args: - --session-cache-max-bytes=8589934592 resources: diff --git a/historyserver/config/historyserver.yaml b/historyserver/config/historyserver.yaml index f9cbe2c2ad9..0928bc53b9a 100644 --- a/historyserver/config/historyserver.yaml +++ b/historyserver/config/historyserver.yaml @@ -65,7 +65,8 @@ spec: # - --use-auth-token-mode=true ports: - containerPort: 8080 - # Soft-bound encoded snapshots at 8 GiB; leave pod memory for decoding. + # Cache is soft-bounded at 8 GiB; 12 GiB leaves decode headroom. + # The 2 GiB request keeps examples schedulable; size production requests to residency. args: - --session-cache-max-bytes=8589934592 resources: