Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/reusable-code.yml
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ jobs:
REPO_FULL_NAME: ${{ inputs.source_repo }}
CODE_ALLOWED_TARGET_BRANCHES: ''
TARGET_BRANCH: main
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/reusable-dispatch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,7 @@ jobs:
env:
GITHUB_ISSUE_URL: ${{ fromJSON(needs.route.outputs.event_payload).issue.html_url }}
REPO_FULL_NAME: ${{ github.repository }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -738,6 +739,7 @@ jobs:
REPO_FULL_NAME: ${{ github.repository }}
CODE_ALLOWED_TARGET_BRANCHES: ''
TARGET_BRANCH: main
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -850,6 +852,7 @@ jobs:
PRIOR_REVIEW_SHA: ${{ steps.prior-review.outputs.prior_sha }}
PRIOR_REVIEW_FILE: ${{ steps.prior-review.outputs.prior_review_file }}
PRIOR_REVIEW_PROVENANCE: ${{ steps.prior-review.outputs.prior_review_provenance }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -1133,6 +1136,7 @@ jobs:
FIX_ITERATION: ${{ steps.context.outputs.iteration }}
REVIEW_BODY_FILE: ${{ steps.review-body.outputs.review_file }}
PRE_AGENT_HEAD: ${{ steps.pre-agent.outputs.head }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -1228,6 +1232,7 @@ jobs:
RETRO_COMMENT: ${{ fromJSON(needs.route.outputs.event_payload).comment.body || '' }}
REPO_FULL_NAME: ${{ github.repository }}
MINT_REPOS: ${{ steps.workspace.outputs.repo_name != '' && (inputs.install_mode == 'per-repo' && steps.workspace.outputs.repo_name || format('{0},.fullsend', steps.workspace.outputs.repo_name)) || '' }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -1308,6 +1313,7 @@ jobs:
env:
GITHUB_ISSUE_URL: ${{ fromJSON(needs.route.outputs.event_payload).issue.html_url }}
REPO_FULL_NAME: ${{ github.repository }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down Expand Up @@ -1615,6 +1621,7 @@ jobs:
env:
GITHUB_ISSUE_URL: ${{ steps.dispatch-env.outputs.issue_url }}
REPO_FULL_NAME: ${{ matrix.source_repo }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-fix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ jobs:
FIX_ITERATION: ${{ steps.context.outputs.iteration }}
REVIEW_BODY_FILE: ${{ steps.review-body.outputs.review_file }}
PRE_AGENT_HEAD: ${{ steps.pre-agent.outputs.head }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-prioritize.yml
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ jobs:
env:
GITHUB_ISSUE_URL: ${{ fromJSON(inputs.event_payload).issue.html_url }}
REPO_FULL_NAME: ${{ inputs.source_repo }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-retro.yml
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ jobs:
RETRO_COMMENT: ${{ fromJSON(inputs.event_payload).comment.body || '' }}
REPO_FULL_NAME: ${{ inputs.source_repo }}
MINT_REPOS: ${{ steps.repo-parts.outputs.name != '' && (inputs.install_mode == 'per-repo' && steps.repo-parts.outputs.name || format('{0},.fullsend', steps.repo-parts.outputs.name)) || '' }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ jobs:
PRIOR_REVIEW_SHA: ${{ steps.prior-review.outputs.prior_sha }}
PRIOR_REVIEW_FILE: ${{ steps.prior-review.outputs.prior_review_file }}
PRIOR_REVIEW_PROVENANCE: ${{ steps.prior-review.outputs.prior_review_provenance }}
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-triage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ jobs:
# ADR 0050 Level 2: orgs opt into OTLP trace export by defining the
# endpoint as an Actions variable (and, when the backend needs auth
# or routing headers, the headers secret). Unset = export inert.
OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}
OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}
OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }}
Expand Down
54 changes: 38 additions & 16 deletions docs/guides/infrastructure/distributed-tracing.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,8 @@ configuration required:
(sandbox creation, agent iterations, validation) with timestamps, durations,
trace IDs, and token/cost attributes.

This file is written on every run unless `OTEL_SDK_DISABLED=true`, which
suppresses all telemetry output including the local file. It contains
metadata only — no prompts, completions, or source code content.
This file is written on every run. It contains metadata only — no prompts,
completions, or source code content.

## Prerequisites

Expand All @@ -31,53 +30,73 @@ Level 1 requires nothing. To enable OTLP export (Level 2 and Level 3) you need:
and bring-your-own-workflow runs only — the managed workflows do not yet
pass a CA bundle through.

## Disabling telemetry

To disable all telemetry, including the local file exporter:

```bash
export OTEL_SDK_DISABLED=true # case-insensitive
```

To disable only the OTLP exporter:
Comment thread
rh-hemartin marked this conversation as resolved.

```bash
unset OTEL_EXPORTER_OTLP_ENDPOINT
unset OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
```

## Enabling OTLP export (Level 2)

To send metadata spans to an OpenTelemetry-compatible backend, set one of the
standard OTEL environment variables:

```bash
# Signal-specific (takes precedence, used as-isno /v1/traces appended)
# Signal-specific (used as-is, no path appended)
Comment thread
rh-hemartin marked this conversation as resolved.
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://your-backend:4318/v1/traces"

# Base URL (SDK appends /v1/traces automatically)
# Base URL (SDK appends /v1/traces)
export OTEL_EXPORTER_OTLP_ENDPOINT="https://your-backend:4318"
```

**Precedence:** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` > `OTEL_EXPORTER_OTLP_ENDPOINT`.
Headers follow the same pattern: `OTEL_EXPORTER_OTLP_TRACES_HEADERS` > `OTEL_EXPORTER_OTLP_HEADERS`.

The local file (`run-telemetry.jsonl`) is produced with no configuration
needed (Level 1), unless `OTEL_SDK_DISABLED=true`.
needed (Level 1).

Comment thread
rh-hemartin marked this conversation as resolved.
When an endpoint is configured, spans are exported via OTLP/HTTP. Any backend
that speaks OTLP works: Jaeger, Grafana Tempo, MLflow, Arize Phoenix,
Langfuse, SigNoz, Honeycomb, Datadog, etc.

If the endpoint is unreachable, the CLI continues normally local files are
If the endpoint is unreachable, the CLI continues normally; local files are
still produced and the run is not affected.

Operational details:
Comment thread
rh-hemartin marked this conversation as resolved.

- **Export timing:** spans are exported live via the OTel SDK's batch
processor as they complete. On shutdown, the provider flushes remaining
spans within a 5-second budget. A dead endpoint does not block the run.
- **Retry:** the exporter retries on transient failures (HTTP 503, etc.)
with an initial interval of 250 ms and a max interval of 2 s. The
5-second context deadline passed to `tp.Shutdown` bounds both retries
and in-flight requests, so a persistently failing or hanging endpoint
does not extend shutdown.
- **Crashed runs:** completed spans that were already flushed mid-run reach
the backend; spans still in the batch buffer are lost. The local
`run-telemetry.jsonl` (written synchronously per span) remains the
forensic record.
- **Sampling:** when the run continues an inbound `TRACEPARENT` whose W3C
sampled flag is unset (`-00`), the upstream sampling decision is respected:
nothing is exported. The local file is still written.
- **Protocol:** OTLP over `http/protobuf` only. Setting
`OTEL_EXPORTER_OTLP_PROTOCOL` (or the traces-specific variant) to anything
else — e.g. `grpc` — skips export with a warning rather than posting
protobuf at a gRPC endpoint.
- **Validation:** a malformed endpoint value skips export with a warning; it
is never silently replaced with the SDK's `localhost:4318` default.
- **Kill switches:** `OTEL_SDK_DISABLED=true` disables all telemetry output
(OTLP export *and* the local file). `OTEL_TRACES_EXPORTER=none` disables
only the OTLP export; the local file is still written.
- **Endpoint validation:** before creating the OTLP exporter, the CLI
Comment thread
rh-hemartin marked this conversation as resolved.
validates whichever endpoint the SDK will actually use
(`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` if set, else
`OTEL_EXPORTER_OTLP_ENDPOINT`). An endpoint is invalid if it cannot be
parsed as a URL, has no scheme, uses a scheme other than `http` or `https`,
or has no host (e.g. `localhost:4318` instead of
`http://localhost:4318`). When invalid, the CLI prints a warning to stderr
and skips OTLP export; the local file exporter is unaffected. A valid
signal-specific endpoint is not blocked by an invalid generic endpoint.
- **Private CAs:** point `OTEL_EXPORTER_OTLP_CERTIFICATE` at a PEM bundle for
backends with certificates outside the system trust store. There is no
skip-verify option.
Expand Down Expand Up @@ -205,6 +224,8 @@ that hosts the fullsend caller workflows:

1. Actions **variable** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — the backend's
full traces URL (e.g. `https://mlflow.example.com/v1/traces`).
Alternatively, set `OTEL_EXPORTER_OTLP_ENDPOINT` (the base URL without a
signal path); managed workflows forward both variants.
2. Actions **secret** `OTEL_EXPORTER_OTLP_TRACES_HEADERS` — the complete
header string, auth and routing included (e.g.
`Authorization=Bearer%20<token>,x-mlflow-experiment-id=42`).
Expand All @@ -223,6 +244,7 @@ Add the environment variables to any job that runs `fullsend run`:

```yaml
env:
OTEL_EXPORTER_OTLP_ENDPOINT: "${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}"
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "${{ vars.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT }}"
OTEL_EXPORTER_OTLP_TRACES_HEADERS: "${{ secrets.OTEL_EXPORTER_OTLP_TRACES_HEADERS }}"
```
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0
go.opentelemetry.io/otel/sdk v1.44.0
go.opentelemetry.io/otel/trace v1.44.0
go.opentelemetry.io/proto/otlp v1.10.0
go.uber.org/goleak v1.3.0
golang.org/x/crypto v0.54.0
golang.org/x/oauth2 v0.36.0
golang.org/x/term v0.45.0
Expand All @@ -38,7 +40,6 @@ require (
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.56.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
Expand Down Expand Up @@ -83,7 +84,7 @@ require (
golang.org/x/image v0.44.0 // indirect
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
google.golang.org/protobuf v1.36.11
k8s.io/klog/v2 v2.140.0 // indirect
)

Expand Down
2 changes: 1 addition & 1 deletion internal/cli/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep
}
rootSpan.End()

flushCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
flushCtx, cancel := context.WithTimeout(context.Background(), telemetry.FlushTimeout)
Comment thread
rh-hemartin marked this conversation as resolved.
defer cancel()
tracingCleanup(flushCtx)
}()
Expand Down
11 changes: 11 additions & 0 deletions internal/telemetry/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package telemetry

import (
"testing"

"go.uber.org/goleak"
)

func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
83 changes: 83 additions & 0 deletions internal/telemetry/otlpsink_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package telemetry

import (
"bytes"
"compress/gzip"
"io"
"net/http"
"net/http/httptest"
"sync"
"testing"

coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
"google.golang.org/protobuf/proto"
)

type otlpSink struct {
mu sync.Mutex
reqs []*coltracepb.ExportTraceServiceRequest
headers []http.Header
paths []string
srv *httptest.Server
}

func newOTLPSink(t *testing.T) *otlpSink {
t.Helper()
s := &otlpSink{}
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if r.Header.Get("Content-Encoding") == "gzip" {
zr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
raw, err = io.ReadAll(zr)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
var req coltracepb.ExportTraceServiceRequest
if err := proto.Unmarshal(raw, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
s.mu.Lock()
s.reqs = append(s.reqs, &req)
s.headers = append(s.headers, r.Header.Clone())
s.paths = append(s.paths, r.URL.Path)
s.mu.Unlock()
resp, _ := proto.Marshal(&coltracepb.ExportTraceServiceResponse{})
w.Header().Set("Content-Type", "application/x-protobuf")
w.Write(resp)
}))
t.Cleanup(s.srv.Close)
return s
}

func (s *otlpSink) spanNames() []string {
s.mu.Lock()
defer s.mu.Unlock()
var names []string
for _, req := range s.reqs {
for _, rs := range req.GetResourceSpans() {
for _, ss := range rs.GetScopeSpans() {
for _, sp := range ss.GetSpans() {
names = append(names, sp.GetName())
}
}
}
}
return names
}

func (s *otlpSink) requestCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.reqs)
}
Loading
Loading