Add custom choom helper to lower crocochrome's OOM score

Dockerfile

@@ -9,19 +9,25 @@ ARG TARGETARCH
 # Build with CGO_ENABLED=0 as grafana-build-tools is debian-based.
 RUN --mount=type=cache,target=/root/.cache/go-build \
   --mount=type=cache,target=/root/go/pkg \
-  CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/crocochrome ./cmd
+  CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/crocochrome ./cmd/crocochrome/ &&\
+  CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/choom ./cmd/choom/
 
 # For setting caps, use the same image than the final layer is using to avoid pulling two distinct ones.
 FROM ghcr.io/grafana/chromium-swiftshader-alpine:131.0.6778.139-r1-3.21.0@sha256:d3071cfe8721cee56fecf8e5d0bf77031d531bc1091b04b05bccf5f50a32365b AS setcapper
 
 RUN apk --no-cache add libcap
 
 COPY --from=buildtools /usr/local/bin/crocochrome /usr/local/bin/crocochrome
+COPY --from=buildtools /usr/local/bin/choom /usr/local/bin/choom
 
 # The following capabilities are used by sm-k6-runner to sandbox the k6 binary. More details about what each cap is used
 # for can be found in /sandbox/sandbox.go.
 # WARNING: The container MUST be also granted all of the following capabilities too, or the CRI will refuse to start it.
 RUN setcap cap_setuid,cap_setgid,cap_kill,cap_chown,cap_dac_override,cap_fowner+ep /usr/local/bin/crocochrome
+# Grant sys_resource capability to custom choom binary, so it can lower OOM scores, and dac_override so it can do it for
+# other processes.
+# Same warnign as above applies.
+RUN setcap cap_sys_resource,cap_dac_override+ep /usr/local/bin/choom
 
 FROM ghcr.io/grafana/chromium-swiftshader-alpine:131.0.6778.139-r1-3.21.0@sha256:d3071cfe8721cee56fecf8e5d0bf77031d531bc1091b04b05bccf5f50a32365b
 
@@ -36,6 +42,7 @@ RUN find / -type f -perm -4000 -delete
 
 # The crocochrome binary has extra capabilities, so we make sure only the k6 user (and not nobody) can run it.
 COPY --from=setcapper --chown=k6:k6 --chmod=0500 /usr/local/bin/crocochrome /usr/local/bin/crocochrome
+COPY --from=setcapper --chown=k6:k6 --chmod=0500 /usr/local/bin/choom /usr/local/bin/choom
 
 USER k6
 

Makefile

@@ -11,7 +11,8 @@ buildtools = $(docker) run --rm -i \
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 go build -v -o build/sm-k6-archiver ./cmd
+	CGO_ENABLED=0 go build -v -o build/crocochrome ./cmd/crocochrome/
+	CGO_ENABLED=0 go build -v -o build/choom ./cmd/choom/
 
 image ?= test.local/sm-k6-archiver
 .PHONY: build-container

cmd/choom/README.md

@@ -0,0 +1,15 @@
+# `choom`
+
+`choom` is a portable, simplified, not compatible version of [`chroom(1)`](https://man7.org/linux/man-pages/man1/choom.1.html), which allows to change a process' `oom_score_adj` to make it more or less attractive to the kernel's OOM killer.
+
+This tiny program exists because binaries need a specific capability to lower OOM scores, `cap_sys_resource`. This capability, however, is not granted in the [default set docker uses](https://github.com/moby/moby/blob/master/oci/caps/defaults.go#L6-L19). Due to how linux capabilities work, binaries with a given capability added will fail to start if the container does not have that capability added as well. In practice, what this means is that if we granted `cap_sys_resource` to the main binary of the container, and attempted to run the container naively with `docker run crocochrome:latest`, the container would cryptically fail to start.
+
+This is a UX problem, where users unfamiliar with the codebase would need to troubleshoot a cryptic error message, but also a problem for testing, where developers would need to go out of the "standard route" to figure out how to add specific capabilities to local kubernetes clusters, or testcontainers, in order to perform tests to the container.
+
+Instead of that, using a tiny helper with the capability added, and calling this helper from the main binary, allows the oom score adjust process to fail gracefully. If the container does not have the required `sys_resource` capability, the OOM score will not be adjusted and a log will be errored:
+
+```
+{"time":"2025-01-30T12:50:37.771068928Z","level":"ERROR","msg":"Error changing OOM score. Assuming this is a test environment and continuing anyway.","err":"fork/exec /usr/local/bin/choom: operation not permitted"}
+```
+
+But execution will continue, and as OOM score adjusting is not critical to functionality whatsoever, tests will do their job just fine. In production, we grant the container the `sys_resource` capability to handle low-memory situations better.

cmd/choom/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+)
+
+func main() {
+	if len(os.Args) < 3 {
+		log.Fatalf("Usage: %s <pid> <oom_score_adj>", os.Args[0])
+	}
+
+	pid := os.Args[1]
+	newOOMScore := os.Args[2]
+
+	adjFile, err := os.OpenFile(filepath.Join("/", "proc", pid, "oom_score_adj"), os.O_WRONLY, os.FileMode(0o600))
+	if err != nil {
+		log.Fatalf("Opening adjust file: %v", err)
+	}
+
+	defer func() {
+		if err := adjFile.Close(); err != nil {
+			log.Fatalf("Closing file: %v", err)
+		}
+	}()
+
+	_, err = adjFile.Write([]byte(newOOMScore))
+	if err != nil {
+		log.Fatalf("Writing oom_score_adj: %v", err)
+	}
+}

cmd/main.go → cmd/crocochrome/main.go

@@ -5,6 +5,8 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"os/exec"
+	"strconv"
 
 	"github.com/grafana/crocochrome"
 	crocohttp "github.com/grafana/crocochrome/http"
@@ -18,6 +20,17 @@ func main() {
 		Level: slog.LevelDebug,
 	}))
 
+	const oomScore = -500
+	if out, err := choom(oomScore); err != nil {
+		logger.Error(
+			"Error changing OOM score, assuming this is not a production environment and continuing anyway",
+			"err", err,
+			"choomOutput", string(out),
+		)
+	} else {
+		logger.Info("Main process OOM score adjusted successfully", "oomScore", oomScore)
+	}
+
 	mux := http.NewServeMux()
 
 	registry := prometheus.NewRegistry()
@@ -54,5 +67,13 @@ func main() {
 	err = http.ListenAndServe(address, mux)
 	if err != nil {
 		logger.Error("Setting up HTTP listener", "err", err)
+		os.Exit(1)
+		return
 	}
 }
+
+// choom runs the choom helper (source included in this repo) to lower the current process OOM score.
+func choom(score int) ([]byte, error) {
+	choom := exec.Command("choom", strconv.Itoa(os.Getpid()), strconv.Itoa(score))
+	return choom.CombinedOutput()
+}