chore(suspect flags): Include filtered flag in output

src/sentry/issues/endpoints/organization_group_suspect_flags.py

@@ -21,6 +21,7 @@ class ResponseDataItem(TypedDict):
     score: float
     baseline_percent: float
     distribution: Distribution
+    is_filtered: bool
 
 
 class ResponseData(TypedDict):
@@ -78,6 +79,7 @@ def get(self, request: Request, group: Group) -> Response:
                         "flag": item["flag"],
                         "score": item["score"],
                         "issue_id": group.id,
+                        "is_filtered": item["is_filtered"],
                     },
                 )
 

src/sentry/issues/suspect_flags.py

@@ -5,7 +5,7 @@
 import sentry_sdk
 from snuba_sdk import Column, Condition, Entity, Function, Limit, Op, Query, Request
 
-from sentry.seer.workflows.compare import KeyedValueCount, keyed_rrf_score
+from sentry.seer.workflows.compare import KeyedValueCount, keyed_rrf_score_with_filter
 from sentry.utils.snuba import raw_snql_query
 
 
@@ -19,6 +19,7 @@ class Score(TypedDict):
     score: float
     baseline_percent: float
     distribution: Distribution
+    is_filtered: bool
 
 
 @sentry_sdk.trace
@@ -42,7 +43,7 @@ def get_suspect_flag_scores(
     outliers_count = query_error_counts(org_id, project_id, start, end, envs, group_id=group_id)
     baseline_count = query_error_counts(org_id, project_id, start, end, envs, group_id=None)
 
-    keyed_scores = keyed_rrf_score(
+    keyed_scores = keyed_rrf_score_with_filter(
         baseline,
         outliers,
         total_baseline=baseline_count,
@@ -67,8 +68,9 @@ def get_suspect_flag_scores(
             "score": score,
             "baseline_percent": baseline_percent_dict[key],
             "distribution": distributions[key],
+            "is_filtered": is_filtered,
         }
-        for key, score in keyed_scores
+        for key, score, is_filtered in keyed_scores
     ]
 
 

src/sentry/seer/math.py

@@ -86,3 +86,136 @@ def _rrf(kl_rank: int, entropy_rank: int) -> float:
 def rank_min(xs: list[float], ascending: bool = False):
     ranks = {x: rank for rank, x in enumerate(sorted(set(xs), reverse=not ascending), 1)}
     return [ranks[x] for x in xs]
+
+
+def boxcox_transform(
+    values: list[float], lambda_param: float | None = None
+) -> tuple[list[float], float]:
+    """
+    Apply BoxCox transformation to a list of values.
+
+    Parameters:
+        values: List of positive values to transform
+        lambda_param: BoxCox lambda parameter. If None, finds optimal lambda.
+
+    Returns:
+        Tuple of (transformed values, lambda parameter used)
+    """
+    min_value = min(values) if values else 0
+    if min_value <= 0:
+        shift_amount = -min_value + 1e-10
+        shifted_values = [v + shift_amount for v in values]
+    else:
+        shifted_values = values
+
+    # Get lambda parameter: use provided one or find optimal
+    lambda_param = _boxcox_normmax(shifted_values) if lambda_param is None else lambda_param
+
+    # Apply transformation
+    if lambda_param == 0.0:
+        transformed = [math.log(max(v, 1e-10)) for v in shifted_values]
+    else:
+        transformed = [
+            (pow(max(v, 1e-10), lambda_param) - 1) / lambda_param for v in shifted_values
+        ]
+
+    return transformed, lambda_param
+
+
+def _boxcox_llf(lambda_param: float, values: list[float]) -> float:
+    """
+    Compute the Box-Cox log-likelihood function.
+
+    Uses numerically stable log-space arithmetic following scipy's implementation.
+
+    Parameters:
+        lambda_param: BoxCox lambda parameter
+        values: List of positive values
+
+    Returns:
+        Log-likelihood value
+    """
+    n = len(values)
+    if n == 0:
+        return 0.0
+
+    log_values = [math.log(max(v, 1e-10)) for v in values]
+    log_sum = sum(log_values)
+
+    if lambda_param == 0.0:
+        log_mean = log_sum / n
+        log_var = sum((lv - log_mean) ** 2 for lv in log_values) / n
+        logvar = math.log(max(log_var, 1e-10))
+    else:
+        # For λ≠0: Use log-space arithmetic for numerical stability
+        # This avoids computing (x^λ - 1)/λ directly which can overflow
+        # Uses identity: var((x^λ - 1)/λ) = var(x^λ)/λ²
+        logx = [lambda_param * lv for lv in log_values]  # log(x^λ) = λ*log(x)
+        logx_mean = sum(logx) / n
+        logx_var = sum((lx - logx_mean) ** 2 for lx in logx) / n
+        # log(var(y)) = log(var(x^λ)) - 2*log(|λ|)
+        logvar = math.log(max(logx_var, 1e-10)) - 2 * math.log(abs(lambda_param))
+
+    # Box-Cox log-likelihood: (λ-1)*Σlog(x) - n/2*log(var(y))
+    return (lambda_param - 1) * log_sum - (n / 2) * logvar
+
+
+def _boxcox_normmax(values: list[float], max_iters: int = 100) -> float:
+    """
+    Calculate the approximate optimal lambda parameter for BoxCox transformation that maximizes the log-likelihood.
+
+    Uses MLE method with ternary search rather than Brent's method for efficient optimization.
+
+    Parameters:
+        values: List of positive values
+        max_iters: Maximum number of iterations to run for ternary search
+
+    Returns:
+        Approximate optimal lambda parameter
+    """
+    if not values:
+        return 0.0
+
+    left = -2.0
+    right = 2.0
+    tolerance = 1e-6
+    iters = 0
+
+    while right - left > tolerance and iters < max_iters:
+        m1 = left + (right - left) / 3
+        m2 = right - (right - left) / 3
+
+        llf_m1 = _boxcox_llf(m1, values)
+        llf_m2 = _boxcox_llf(m2, values)
+
+        if llf_m1 > llf_m2:
+            right = m2
+        else:
+            left = m1
+
+        iters += 1
+
+    return (left + right) / 2
+
+
+def calculate_z_scores(values: list[float]) -> list[float]:
+    """
+    Calculate z-scores for a list of values.
+
+    Parameters:
+        values: List of numerical values
+
+    Returns:
+        List of z-scores corresponding to input values
+    """
+    if not values:
+        return []
+
+    mean_val = sum(values) / len(values)
+    variance = sum((x - mean_val) ** 2 for x in values) / len(values)
+    std_dev = math.sqrt(variance)
+
+    if std_dev == 0:
+        return [0.0] * len(values)
+
+    return [(x - mean_val) / std_dev for x in values]

src/sentry/seer/workflows/compare.py

@@ -2,7 +2,14 @@
 from collections.abc import Callable, Generator, Mapping, Sequence
 from typing import TypeVar
 
-from sentry.seer.math import entropy, kl_divergence, laplace_smooth, rrf_score
+from sentry.seer.math import (
+    boxcox_transform,
+    calculate_z_scores,
+    entropy,
+    kl_divergence,
+    laplace_smooth,
+    rrf_score,
+)
 
 T = TypeVar("T")
 
@@ -186,3 +193,66 @@ def _ensure_symmetry(a: Distribution, b: Distribution) -> tuple[Distribution, Di
 
 def _smooth_distribution(dist: Distribution) -> Distribution:
     return dict(zip(dist.keys(), laplace_smooth(list(dist.values()))))
+
+
+def keyed_rrf_score_with_filter(
+    baseline: Sequence[KeyedValueCount],
+    outliers: Sequence[KeyedValueCount],
+    total_baseline: int,
+    total_outliers: int,
+    entropy_alpha: float = 0.2,
+    kl_alpha: float = 0.8,
+    offset: int = 60,
+    z_threshold: float = 1.5,
+) -> list[tuple[str, float, bool]]:
+    """
+    RRF score a multi-dimensional distribution of values. Returns a list of key, score pairs, and a mapping of if the key was filtered.
+    The filtered keys are those that have a normalized entropy and kl score less than the z_threshold.
+    Duplicates are not tolerated.
+
+    Sample distribution:
+        [("key", "true", 93), ("key", "false", 219), ("other", "true", 1)]
+
+    Sample output:
+        [("key", 0.5, True), ("key", 0.3, False), ("other", 0.1, False)]
+    """
+
+    def _scoring_fn(baseline: list[float], outliers: list[float]):
+        return (entropy(outliers), kl_divergence(baseline, outliers))
+
+    scored_keys = _score_each_key(
+        baseline,
+        outliers,
+        total_baseline,
+        total_outliers,
+        scoring_fn=_scoring_fn,
+    )
+
+    keys = []
+    entropy_scores = []
+    kl_scores = []
+
+    for key, (entropy_score, kl_score) in scored_keys:
+        keys.append(key)
+        entropy_scores.append(entropy_score)
+        kl_scores.append(kl_score)
+
+    normalized_entropy_scores, _ = boxcox_transform(entropy_scores)
+    normalized_kl_scores, _ = boxcox_transform(kl_scores)
+    entropy_z_scores = calculate_z_scores(normalized_entropy_scores)
+    kl_z_scores = calculate_z_scores(normalized_kl_scores)
+
+    filtered_keys = [
+        entropy_z_score <= z_threshold and kl_z_score <= z_threshold
+        for entropy_z_score, kl_z_score in zip(entropy_z_scores, kl_z_scores)
+    ]
+
+    return sorted(
+        zip(
+            keys,
+            rrf_score(entropy_scores, kl_scores, entropy_alpha, kl_alpha, offset),
+            filtered_keys,
+        ),
+        key=lambda k: k[1],
+        reverse=True,
+    )

tests/sentry/issues/endpoints/test_organization_group_suspect_flags.py

@@ -69,6 +69,7 @@ def test_get(self) -> None:
                             "true": 1,
                         },
                     },
+                    "is_filtered": True,
                 },
                 {
                     "flag": "other",
@@ -82,6 +83,7 @@ def test_get(self) -> None:
                             "false": 1,
                         },
                     },
+                    "is_filtered": True,
                 },
             ]
         }

tests/sentry/issues/test_suspect_flags.py

@@ -132,11 +132,13 @@ def test_get_suspect_flag_scores(self) -> None:
                     "baseline": {"false": 1, "true": 1},
                     "outliers": {"true": 1},
                 },
+                "is_filtered": True,
             },
             {
                 "flag": "other",
                 "score": 0.016181914331041776,
                 "baseline_percent": 0,
                 "distribution": {"baseline": {"false": 2}, "outliers": {"false": 1}},
+                "is_filtered": True,
             },
         ]

tests/sentry/seer/test_math.py

@@ -1,6 +1,8 @@
 import math
 
 from sentry.seer.math import (
+    boxcox_transform,
+    calculate_z_scores,
     entropy,
     kl_divergence,
     laplace_smooth,
@@ -90,3 +92,64 @@ def test_rrf_score():
 def test_rank_min():
     assert rank_min(xs=[1, 2, 2, 2, 3], ascending=False) == [3, 2, 2, 2, 1]
     assert rank_min(xs=[1, 2, 2, 2, 3], ascending=True) == [1, 2, 2, 2, 3]
+
+
+def test_boxcox_transform():
+    # Test with lambda = 0 (log transformation)
+    values = [1.0, 2.0, 4.0, 8.0]
+    transformed, lambda_used = boxcox_transform(values, lambda_param=0.0)
+    expected = [math.log(v) for v in values]
+    assert lambda_used == 0.0
+    for t, e in zip(transformed, expected):
+        assert math.isclose(t, e, rel_tol=1e-9)
+
+    # Test with lambda = 1 (no transformation, just (x-1)/1 = x-1)
+    transformed, lambda_used = boxcox_transform(values, lambda_param=1.0)
+    expected = [v - 1.0 for v in values]
+    assert lambda_used == 1.0
+    for t, e in zip(transformed, expected):
+        assert math.isclose(t, e, rel_tol=1e-9)
+
+    # Test with lambda = 0.5 (square root transformation)
+    transformed, lambda_used = boxcox_transform(values, lambda_param=0.5)
+    expected = [(math.sqrt(v) - 1.0) / 0.5 for v in values]
+    assert lambda_used == 0.5
+    for t, e in zip(transformed, expected):
+        assert math.isclose(t, e, rel_tol=1e-9)
+
+    # Test auto lambda detection
+    transformed, lambda_used = boxcox_transform(values, lambda_param=None)
+    assert isinstance(lambda_used, float)
+    assert len(transformed) == len(values)
+
+    # Test empty input
+    transformed, lambda_used = boxcox_transform([], lambda_param=0.0)
+    assert transformed == []
+    assert lambda_used == 0.0
+
+
+def test_calculate_z_scores():
+    values = [1.0, 2.0, 3.0, 4.0, 5.0]
+    z_scores = calculate_z_scores(values)
+
+    expected_mean = 3.0
+    expected_std = math.sqrt(2.0)
+    expected = [(v - expected_mean) / expected_std for v in values]
+
+    assert len(z_scores) == len(values)
+    for z, e in zip(z_scores, expected):
+        assert math.isclose(z, e, rel_tol=1e-9)
+
+    same_values = [5.0, 5.0, 5.0, 5.0]
+    z_scores = calculate_z_scores(same_values)
+    assert all(z == 0.0 for z in z_scores)
+
+    assert calculate_z_scores([]) == []
+
+    single_z = calculate_z_scores([42.0])
+    assert single_z == [0.0]
+
+    simple_values = [0.0, 10.0]
+    z_scores = calculate_z_scores(simple_values)
+    assert math.isclose(z_scores[0], -1.0, rel_tol=1e-9)
+    assert math.isclose(z_scores[1], 1.0, rel_tol=1e-9)