Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion proto/cymbal/resolution/v1/resolution.proto
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ message Accepted {}
message Done {
// Serialized rust/cymbal Exception after frame resolution and exception
// remapping. It replaces only the submitted exception item, not the full
// event or exception list.
// event or exception list. Resolved frames may carry a `release` object
// (the release bound to the symbol set that resolved them); older servers
// omit it and callers must treat a missing key as "no release".
bytes resolved_exception_json = 1;
}

Expand Down
5 changes: 3 additions & 2 deletions rust/cymbal-proto/tests/contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ fn resolve_outcome_echoes_id_and_carries_done_error_or_retry() {
.collect();

assert!(matches!(
decoded[0].result,
Some(resolve_outcome::Result::Done(_))
&decoded[0].result,
Some(resolve_outcome::Result::Done(done))
if done.resolved_exception_json == br#"{"type":"ResolvedError"}"#
));
assert!(matches!(
decoded[1].result,
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions rust/cymbal/docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ No generated type changes are needed:
- Events use the remote pool and do not silently fall back to inline resolution on failures.
- Events are flattened into exception-level `ResolveItem`s. Items are grouped by their first symbol-set reference when available, with a per-team fallback, then submitted over per-endpoint bidirectional `Resolve` streams.
- Resolver-specific context is carried in `ResolveItem.metadata` as JSON bytes. The native symbolication convention uses a `debug_images_json` key.
- Resolved frames in `Done.resolved_exception_json` may carry a `release` object: the release bound to the symbol set that resolved the frame. The key is omitted when a frame has none (also what older servers send for every frame), and callers must treat a missing key as "no release". The processing side strips `Frame.release` before any clickhouse-bound serialization.
- Per-item `ResolveOutcome.Error.kind` is the control-flow surface. `ERROR_KIND_OVERLOADED` is result-only backpressure and triggers item reroute. Accepted items emit `ResolveOutcome.Accepted` before their terminal outcome; cymbal releases its routing permit on that acceptance signal. If `CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_MS` is non-zero, the overloaded endpoint is also temporarily excluded from new routing in that cymbal process. Repeated overloads double that cooldown up to `CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_MAX_MS`, and a quiet `CYMBAL_REMOTE_RESOLUTION_OVERLOAD_EJECTION_DECAY_MS` window resets it. `CYMBAL_REMOTE_RESOLUTION_ROUTING_JITTER` controls how much routing flattens across the load-adjusted rendezvous-ranked candidate list (`0.0` strict top load-adjusted candidate, `1.0` load-weighted across candidates). `LoadEvent` carries endpoint freshness/draining state plus `in_flight` / `max_in_flight` as a soft routing load signal.

This means Node request chunking limits protect cymbal's public HTTP body size, while cymbal's private gRPC path owns exception-level routing, reroute depth, and overload handling.
23 changes: 16 additions & 7 deletions rust/cymbal/src/core/symbolication/symbol/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use sqlx::PgPool;
use crate::{
core::config::ResolverConfig,
error::{JsResolveErr, ProguardError, ResolveError, UnhandledError},
frames::{Frame, RawFrame},
frames::{releases::ReleaseRecord, Frame, RawFrame},
langs::native::DebugImage,
metric_consts::{
FRAME_CACHE_HITS, FRAME_CACHE_MISSES, FRAME_DB_HITS, FRAME_DB_MISSES,
Expand Down Expand Up @@ -156,19 +156,28 @@ impl LocalSymbolResolver {

assert!(!resolved.is_empty()); // If this ever happens, we've got a data-dropping bug, and want to crash

let set = if let Some(set_ref) = frame.symbol_set_ref(debug_images) {
let mut set = SymbolSetRecord::load(&self.pool, raw_id.team_id, &set_ref).await?;
let (set, release) = if let Some(set_ref) = frame.symbol_set_ref(debug_images) {
let set_fut = SymbolSetRecord::load(&self.pool, raw_id.team_id, &set_ref);
let release_fut = async {
ReleaseRecord::for_symbol_set_ref(&self.pool, &set_ref, raw_id.team_id)
.await
.map_err(UnhandledError::from)
};
let (mut set, release) = tokio::try_join!(set_fut, release_fut)?;
if let Some(s) = &mut set {
s.set_last_used(&self.pool).await?;
}
set
(set, release)
} else {
None
(None, None)
};

let mut records = Vec::new();
for r_frame in &resolved {
// Save back to the DB
let mut resolved = resolved;
for r_frame in resolved.iter_mut() {
r_frame.release = release.clone(); // Enrich with release information

// And save back to the DB
Comment thread
ablaszkiewicz marked this conversation as resolved.
let record = ErrorTrackingStackFrame::new(
r_frame.frame_id.clone(),
set.as_ref().map(|s| s.id),
Expand Down
17 changes: 15 additions & 2 deletions rust/cymbal/src/core/symbolication/symbol/records.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use sqlx::Executor;
use uuid::Uuid;

use crate::error::UnhandledError;
use crate::frames::{Context, Frame};
use crate::frames::{releases::ReleaseRecord, Context, Frame};

const FRAME_TTL_JITTER_PERCENT: u32 = 10;

Expand Down Expand Up @@ -108,6 +108,12 @@ impl ErrorTrackingStackFrame {
} else {
None
};
// Stored contents never include the release: `load_all` re-joins it via the symbol set so
// a release (re)bind takes effect on the next load instead of going stale in cache rows.
let mut contents = serde_json::to_value(&self.contents)?;
if let Some(object) = contents.as_object_mut() {
object.remove("release");
}
sqlx::query!(
r#"
INSERT INTO posthog_errortrackingstackframe (raw_id, part, team_id, created_at, symbol_set_id, contents, resolved, id, context)
Expand All @@ -124,7 +130,7 @@ impl ErrorTrackingStackFrame {
self.id.team_id,
self.created_at,
self.symbol_set_id,
serde_json::to_value(&self.contents)?,
contents,
self.resolved,
Uuid::now_v7(),
context,
Expand Down Expand Up @@ -175,6 +181,11 @@ impl ErrorTrackingStackFrame {
return Ok(Vec::new());
}

let mut release = None;
if let Some(ss_id) = &res[0].symbol_set_id {
release = ReleaseRecord::for_symbol_set_id(e, *ss_id, id.team_id).await?;
}

for found in res {
// Frame ID's lose team_id when they're serialized, so we fix that up here when loading them
let frame_id = FrameId::new(found.raw_id, found.team_id, found.part);
Expand All @@ -192,6 +203,7 @@ impl ErrorTrackingStackFrame {
None
};

frame.release = release.clone();
frame.context = context.clone();

results.push(Self {
Expand Down Expand Up @@ -230,6 +242,7 @@ mod tests {
junk_drawer: None,
code_variables: None,
context: None,
release: None,
}
}

Expand Down
2 changes: 1 addition & 1 deletion rust/cymbal/src/core/symbolication/symbol_store/saving.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const NEGATIVE_CACHE_MAX_WEIGHT: u64 = 64 * 1024 * 1024;

// We truncate the reference to resolve an issue with the maximum size in a BTRee index on Postgres
// TODO: update model to use a hash of the reference instead
fn truncate_ref(s: &str) -> &str {
pub(crate) fn truncate_ref(s: &str) -> &str {
if s.len() <= MAX_REF_BYTES {
return s;
}
Expand Down
7 changes: 7 additions & 0 deletions rust/cymbal/src/core/types/frames/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::collections::HashMap;

use common_types::error_tracking::{FrameData, FrameId, RawFrameId};
use releases::ReleaseRecord;
use serde::{Deserialize, Serialize};
use serde_json::Value;

Expand Down Expand Up @@ -172,6 +173,12 @@ pub struct Frame {
// use in the frontend
#[serde(skip)]
pub context: Option<Context>,
// The release bound to the symbol set that resolved this frame. Serializable so it crosses the
// resolution-service wire inside the frame JSON, but it must not reach the clickhouse-bound
// event JSON — `into_resolved` strips it after `$exception_release` selection — and the PG
// frame cache re-joins it at load instead of trusting a stored copy (see records.rs).
#[serde(skip_serializing_if = "Option::is_none", default)]
pub release: Option<ReleaseRecord>,
}

#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
Expand Down
Loading
Loading