From cbdde3a0098a58a2743926f9d2ff063e214802d2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:42:25 -0800 Subject: [PATCH 01/27] feat(seer): Gate agent chat messages on Seer credits Return a 402 before sending a chat message when a token-billed org does not have enough Seer credits. Dashboard generation runs are not gated. can_run_seer_operation now passes orgs that are not token-billed. Refs CW-3061 --- src/sentry/seer/credits.py | 14 +++- .../endpoints/organization_seer_agent_chat.py | 18 +++- .../test_organization_seer_agent_chat.py | 84 +++++++++++++++++++ .../test_organization_seer_credit_check.py | 4 +- tests/sentry/seer/test_credits.py | 4 +- 5 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 6908b5ba4d85..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,8 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization has enough Seer credits to run the operation.""" +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -75,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index ae787654d804..269705ba3b88 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -28,6 +28,7 @@ has_seer_agent_access_with_detail, snapshot_to_markdown, ) +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import ResolvedSeerRun, resolve_seer_run from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun @@ -356,10 +357,21 @@ def post( return result resolved = result - if not has_access and not ( - resolved is not None and _is_dashboard_generate_run(organization, resolved) - ): + is_dashboard_generate_run = resolved is not None and _is_dashboard_generate_run( + organization, resolved + ) + if not has_access and not is_dashboard_generate_run: raise PermissionDenied(error) + # Do not bill for dashboard generation. + if not is_dashboard_generate_run and not can_run_seer_operation( + organization, SeerOperation.AGENT_CHAT_MESSAGE + ): + return Response( + { + "detail": "Your organization does not have enough credits left for you to message Seer." + }, + status=402, + ) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index f9cf4b9eadf8..1b33d749ef45 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.seer.agent.client_models import ( MemoryBlock, @@ -493,6 +496,87 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=555, user_id=self.user.id + ) + + response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 199a106443b3..90aaf513233d 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -19,7 +19,7 @@ def setUp(self) -> None: super().setUp() self.login_as(user=self.user) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), @@ -39,7 +39,7 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index ba587cd18881..afd37295c106 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -27,7 +27,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -45,7 +45,7 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), From c2473282edb707762f2ba74673fc5f8c36b06b98 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:43:12 -0800 Subject: [PATCH 02/27] feat(seer): Check Seer credits before Autofix steps and Open PR Token-billed orgs are checked for Seer credits before each Autofix step and before opening a PR, and get SeerOutOfCreditsException when they can't pay. Other orgs keep the seat-based quota check. Agentic triage runs are not gated. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 50 ++++--- src/sentry/seer/autofix/exceptions.py | 4 + src/sentry/seer/autofix/feature/dispatch.py | 90 ++++++++++-- src/sentry/seer/autofix/steps.py | 14 ++ .../seer/autofix/feature/test_dispatch.py | 86 ++++++++++- .../sentry/seer/autofix/test_autofix_agent.py | 136 +++++++++++++++++- 6 files changed, 342 insertions(+), 38 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 6515cef5ab52..7ce38f75c09a 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -9,7 +9,7 @@ from scm.types import GetBranchProtocol, GetRepositoryProtocol from sentry_sdk import traces -from sentry import features, quotas +from sentry import features from sentry.analytics.events.autofix_events import ( AiAutofixAgentHandoffEvent, AiAutofixCodeChangesCompletedEvent, @@ -23,7 +23,7 @@ AiAutofixSolutionCompletedEvent, AiAutofixSolutionStartedEvent, ) -from sentry.constants import ENABLE_SEER_CODING_DEFAULT, DataCategory +from sentry.constants import ENABLE_SEER_CODING_DEFAULT from sentry.integrations.services.integration import integration_service from sentry.models.group import Group from sentry.seer.agent.client import SeerAgentClient @@ -31,10 +31,11 @@ from sentry.seer.autofix.analytics import record_autofix_event from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import ( AutofixFeatureArgs, autofix_run_targets_other_issue, + check_autofix_billing, require_autofix_run_for_group, trigger_autofix_feature, ) @@ -62,6 +63,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) +from sentry.seer.credits import SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -460,19 +462,13 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - if run_id is None: - skip_quota_check = allow_free_cohort and is_free_cohort_org(group.organization) - if not skip_quota_check: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - raise NoSeerQuotaException() + check_autofix_billing( + group, + step.seer_operation, + referrer, + is_new_run=run_id is None, + skip_quota_for_free_cohort=allow_free_cohort, + ) if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) @@ -547,7 +543,12 @@ def trigger_autofix_agent( actor_user_id=actor_user_id, ) return feature_run - except (NoSeerQuotaException, SeerPermissionError, PrIterationNoPullRequestException): + except ( + NoSeerQuotaException, + SeerOutOfCreditsException, + SeerPermissionError, + PrIterationNoPullRequestException, + ): outcome = "declined" raise except BaseException: @@ -834,6 +835,17 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Open PR always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_OPEN_PR, + seer_run.referrer if seer_run else None, + is_new_run=False, + ) + client = get_autofix_agent_client(group, user=user) if state is None: @@ -851,10 +863,6 @@ def trigger_push_changes( ) ) - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/exceptions.py b/src/sentry/seer/autofix/exceptions.py index 74332026f4dc..7ee853708745 100644 --- a/src/sentry/seer/autofix/exceptions.py +++ b/src/sentry/seer/autofix/exceptions.py @@ -2,6 +2,10 @@ class NoSeerQuotaException(Exception): """Raised when an Autofix run cannot consume Seer quota.""" +class SeerOutOfCreditsException(Exception): + """Raised when the organization doesn't have enough Seer credits for an Autofix operation.""" + + class IssueSummaryUnavailable(Exception): """Raised when an issue summary cannot be generated for an expected reason.""" diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 5ec273daaaec..02b7f3f5c49b 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -22,7 +22,7 @@ ) from sentry.seer.agent.on_completion_hook import extract_hook_definition from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import ( FEATURE_ID, LEGACY_FEATURE_ID, @@ -41,6 +41,7 @@ get_automation_handoff, is_free_cohort_org, ) +from sentry.seer.credits import SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -140,18 +141,42 @@ def _single_run_pr_internal_settings( ) -def trigger_autofix_feature( - group: Group, - args: AutofixFeatureArgs, -) -> SeerRun: - # Avoid a circular import through the legacy Autofix dispatcher. - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook +def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: + """ + Raise if the organization can't afford the operation under token-based + credits. Agentic triage runs stay free. + """ + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + return + if not has_enough_seer_credits(group.organization, operation): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() - is_new_run = args.existing_run_id is None + +def _check_autofix_quota( + group: Group, + referrer: str | None, + *, + is_new_run: bool, + allow_free_cohort: bool, +) -> bool: + """ + Raise if the organization has no seat-based Seer quota left. Returns whether + the step uses quota, which is recorded once the run starts. + """ # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and args.allow_free_cohort and is_free_cohort_org(group.organization) - if is_new_run and not skip_quota: + skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) + uses_quota = is_new_run and not skip_quota + if uses_quota: has_budget: bool = quotas.backend.check_seer_quota( org_id=group.organization.id, data_category=DataCategory.SEER_AUTOFIX, @@ -162,11 +187,50 @@ def trigger_autofix_feature( extra={ "group_id": group.id, "organization_id": group.organization.id, - "referrer": args.referrer.value, + "referrer": referrer, }, ) raise NoSeerQuotaException() + return uses_quota + + +def check_autofix_billing( + group: Group, + operation: SeerOperation, + referrer: str | None, + *, + is_new_run: bool, + skip_quota_for_free_cohort: bool = False, +) -> bool: + """ + Raise if the organization can't pay for the operation. Token-billed orgs are + checked for credits, everyone else for seat-based quota. Returns whether the + operation uses quota. + """ + if features.has("organizations:token-billed-seer-enabled", group.organization): + _check_autofix_credits(group, operation, referrer) + return False + return _check_autofix_quota( + group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + ) + + +def trigger_autofix_feature( + group: Group, + args: AutofixFeatureArgs, +) -> SeerRun: + # Avoid a circular import through the legacy Autofix dispatcher. + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=args.existing_run_id is None, + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + single_run = should_use_single_run( group, args.step, @@ -259,7 +323,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if is_new_run: + if args.existing_run_id is None: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), @@ -285,7 +349,7 @@ def trigger_autofix_feature( else: raise Exception("Unhandled run_id branch, this should never happen") - if is_new_run and not skip_quota: + if uses_quota: quotas.backend.record_seer_run( group.organization.id, group.project.id, DataCategory.SEER_AUTOFIX ) diff --git a/src/sentry/seer/autofix/steps.py b/src/sentry/seer/autofix/steps.py index b35a0d86ebea..8ec531cfde9e 100644 --- a/src/sentry/seer/autofix/steps.py +++ b/src/sentry/seer/autofix/steps.py @@ -3,6 +3,7 @@ from enum import StrEnum from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerOperation class AutofixStep(StrEnum): @@ -13,6 +14,19 @@ class AutofixStep(StrEnum): CODE_CHANGES = "code_changes" PR_ITERATION = "pr_iteration" + @property + def seer_operation(self) -> SeerOperation: + """The Seer credit check operation corresponding to this step.""" + match self: + case AutofixStep.ROOT_CAUSE: + return SeerOperation.AUTOFIX_ROOT_CAUSE + case AutofixStep.SOLUTION: + return SeerOperation.AUTOFIX_SOLUTION + case AutofixStep.CODE_CHANGES: + return SeerOperation.AUTOFIX_CODE_CHANGES + case AutofixStep.PR_ITERATION: + return SeerOperation.AUTOFIX_PR_ITERATION + @staticmethod def from_autofix_stopping_point( autofix_stopping_point: AutofixStoppingPoint, diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 06ddaf9ca71b..9827183d33c6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -2,10 +2,13 @@ from unittest.mock import MagicMock, patch import pytest +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.constants import DataCategory from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature from sentry.seer.autofix.feature.models import ( FEATURE_ID, @@ -21,6 +24,7 @@ from sentry.seer.autofix.utils import AutofixStoppingPoint from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.testutils.pytest.fixtures import django_db_all @@ -275,6 +279,86 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + MockClient.return_value.start_feature_run.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits_for_a_continued_step( + self, mock_check_seer_credits: MagicMock + ) -> None: + """Unlike quota, credits are checked for every step, not only new runs.""" + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.SOLUTION, + existing_run_id=123, + step_args=SolutionStepArgs(), + ), + ) + + mock_quotas.backend.check_seer_quota.assert_not_called() + MockClient.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 477b8bf89ffa..dd598ac0a01e 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings from rest_framework.exceptions import PermissionDenied +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent from sentry.constants import DataCategory @@ -27,7 +30,7 @@ ) from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer, SeerAutomationSource -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.pr_iteration.feedback import Feedback @@ -38,6 +41,7 @@ from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.types.activity import ActivityType from sentry.utils import json @@ -720,6 +724,30 @@ def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMoc ) mock_feature.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + def test_when_out_of_credits( + self, + mock_check_quota: MagicMock, + mock_feature: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.UNKNOWN, + run_id=None, + ) + mock_check_seer_credits.assert_called_once() + mock_check_quota.assert_not_called() + mock_feature.assert_not_called() + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -732,7 +760,7 @@ def test_free_cohort_skips_quota_check( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=True), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=True), ): trigger_autofix_agent( group=self.group, @@ -756,7 +784,7 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=False), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), ): trigger_autofix_agent( group=self.group, @@ -767,6 +795,35 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( mock_check_quota.assert_called_once() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + def test_agentic_triage_skips_credit_check( + self, + mock_feature: MagicMock, + mock_broadcast: MagicMock, + mock_check_quota: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + mock_feature.return_value = self.create_seer_run( + organization=self.group.organization, seer_run_state_id=12345 + ) + + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.AGENTIC_TRIAGE, + ) + + mock_check_seer_credits.assert_not_called() + mock_check_quota.assert_not_called() + mock_feature.assert_called_once() + @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -975,6 +1032,35 @@ def test_counts_declined_when_no_quota( } ] + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_counts_declined_when_out_of_credits( + self, + mock_check_seer_credits: MagicMock, + mock_client_class: MagicMock, + mock_incr: MagicMock, + mock_check_quota: MagicMock, + mock_record_run: MagicMock, + mock_broadcast: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + self.group, + AutofixStep.ROOT_CAUSE, + AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + ) + + assert _trigger_attempts(mock_incr) == [ + { + "referrer": "issue_summary.post_process_fixability", + "step": "root_cause", + "outcome": "declined", + } + ] + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") def test_counts_error_when_seer_rejects_run( self, @@ -1674,6 +1760,50 @@ def test_raises_permission_denied_when_coding_disabled(self) -> None: referrer=AutofixReferrer.UNKNOWN, ) + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_raises_when_out_of_credits( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_push_changes( + group=self.group, + run_id=123, + referrer=AutofixReferrer.UNKNOWN, + ) + + mock_post.assert_not_called() + + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_agentic_triage_run_skips_credit_check( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + # The push is called with another referrer; the run's own referrer decides. + self.create_seer_run( + organization=self.organization, + seer_run_state_id=123, + referrer=AutofixReferrer.AGENTIC_TRIAGE.value, + ) + + self._push( + mock_post, + features={ + "organizations:seer-credit-checks": True, + "organizations:token-billed-seer-enabled": True, + }, + ) + + mock_check_seer_credits.assert_not_called() + mock_post.assert_called_once() + @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock From e2855ba1be5c9a28d483661f1ea99b9c1bc7cbef Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:48:24 -0800 Subject: [PATCH 03/27] ref(seer): Keep the token-billed check inside can_run_seer_operation --- src/sentry/seer/credits.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..17ae7eb017a6 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,11 +60,10 @@ class SeerOperationConfig: } -def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: - """ - Checks whether the organization has enough Seer credits to run the operation - for callers that already checked the org is token-billed. - """ +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + if not features.has("organizations:token-billed-seer-enabled", organization): + return True if not features.has("organizations:seer-credit-checks", organization): return True @@ -78,10 +77,3 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation ) ) return response.has_enough_credits - - -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - return not features.has( - "organizations:token-billed-seer-enabled", organization - ) or has_enough_seer_credits(organization, operation) From d70d6cbddd51750df5c0dd1aac152f630622a332 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:49:40 -0800 Subject: [PATCH 04/27] ref(seer): Split has_enough_seer_credits out of can_run_seer_operation Autofix billing checks the token-billed flag itself, so it only needs the credit check. --- src/sentry/seer/credits.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,10 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -77,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) From 58a1b631b942cf732754ee84b6e5037efbc25268 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:54:24 -0800 Subject: [PATCH 05/27] ref(seer): Combine the credit check flag conditions --- src/sentry/seer/credits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..ebf4c3fc1411 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True - if not features.has("organizations:seer-credit-checks", organization): + if not features.has( + "organizations:token-billed-seer-enabled", organization + ) or not features.has("organizations:seer-credit-checks", organization): return True config = SEER_OPERATIONS[operation] From 0ab3b109a98367d1f3cee0159b7a89d71a3656bc Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:21:40 -0800 Subject: [PATCH 06/27] feat(seer): Check Seer credits before replying to an Autofix single run Chat replies to single runs continue the run through continue_single_run, which skipped billing. The run's own referrer decides, so agentic triage runs stay free. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 1e9e5496062b..46d533585a58 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..a4915a46067e 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From a3dae0d0298e80374d49e598e1a4e129766c4a4d Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:38:52 -0800 Subject: [PATCH 07/27] ref(seer): Check the credit check flag first --- src/sentry/seer/credits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index ebf4c3fc1411..ac40567af5cd 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has( + if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization - ) or not features.has("organizations:seer-credit-checks", organization): + ): return True config = SEER_OPERATIONS[operation] From 94b979c0fadb6da8ca292f4165cf2e0dc5b2fe89 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:54:15 -0800 Subject: [PATCH 08/27] test(seer): Align credit check tests with their files Cover orgs that aren't token-billed, group the chat credit tests after the access tests, and match the neighboring tests' names and setup. --- .../test_organization_seer_agent_chat.py | 165 +++++++++--------- .../test_organization_seer_credit_check.py | 13 ++ tests/sentry/seer/test_credits.py | 12 ++ 3 files changed, 109 insertions(+), 81 deletions(-) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 46b6c73afedf..26bd0eb24494 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -647,87 +647,6 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_new_conversation_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=555, user_id=self.user.id - ) - - response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_dashboard_run_without_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=789, user_id=self.user.id - ) - self.create_seer_agent_run(run=run, source="dashboard_generate") - mock_client = MagicMock() - mock_client.continue_run.return_value = 789 - mock_client_class.return_value = mock_client - - response = self.client.post( - f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" - ) - - assert response.status_code == 200 - assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=True), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_with_enough_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - mock_client = MagicMock() - mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) - mock_client_class.return_value = mock_client - - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 200 - mock_client.start_run.assert_called_once() - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock @@ -1007,6 +926,90 @@ def test_get_denied_without_seer_access(self) -> None: assert response.status_code == 403 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + mock_check_seer_credits.assert_not_called() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_json_on_page_context_converted_to_markdown( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 90aaf513233d..9a7303e5b8d3 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -57,6 +57,19 @@ def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> Non assert response.data == {"hasEnoughCredits": True} + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_get_returns_has_enough_credits_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + response = self.get_success_response(self.organization.slug, operation="autofix_solution") + + assert response.data == {"hasEnoughCredits": True} + mock_check_seer_credits.assert_not_called() + def test_get_returns_400_when_operation_missing(self) -> None: self.get_error_response(self.organization.slug, status_code=400) diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index afd37295c106..0b730e6d390a 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -17,6 +17,7 @@ def test_every_operation_has_a_credit_estimate(self) -> None: for operation in SeerOperation: assert SEER_OPERATIONS[operation].estimate() > 0 + @with_feature("organizations:token-billed-seer-enabled") @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -27,6 +28,17 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_can_run_seer_operation_skips_billing_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + mock_check_seer_credits.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", From 6f7cb6f9492f6d1332b90b4d05a7fc34213af13a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:16:01 -0800 Subject: [PATCH 09/27] ref(seer): Leave single run replies out of credit gating Single runs are behind organizations:autofix-single-run and need their own estimate, so they're out of scope here. --- src/sentry/seer/autofix/feature/dispatch.py | 8 ----- src/sentry/seer/credits.py | 4 --- .../seer/autofix/feature/test_dispatch.py | 36 ------------------- 3 files changed, 48 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..408b573b403e 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,14 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..5a894919c0a3 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,7 +19,6 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" - AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -58,9 +57,6 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..7264c3e2dbc1 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,39 +692,3 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, - pytest.raises(SeerOutOfCreditsException), - ): - continue_single_run( - self.agent_run, - self.group, - "make a code fix", - user=self.user, - can_publish=True, - referrer=AutofixReferrer.SEER_EXPLORER, - ) - - mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() From 99eb2acddfff3a71f38b84b997d3d98c300fbf0b Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:43:19 -0800 Subject: [PATCH 10/27] fix(seer): Don't let API callers claim the agentic triage referrer Agentic triage runs skip the Seer credit check, and the Autofix endpoint accepted any referrer from the request body. Treat a client-supplied agentic triage referrer as unknown. --- src/sentry/seer/endpoints/group_ai_autofix.py | 7 ++++++- .../seer/endpoints/test_group_ai_autofix.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4767dde92f13..93010d501f0e 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,10 +165,15 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - return AutofixReferrer(raw) + referrer = AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN + # Agentic triage runs skip the credit check, so callers can't claim to be it. + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) + return AutofixReferrer.UNKNOWN + return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 10c804a02408..f4ab9f832b8c 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,6 +1198,25 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_agentic_triage_referrer_is_unknown( + self, mock_trigger_explorer: MagicMock + ) -> None: + """Agentic triage runs skip the credit check, so the API can't claim to be one.""" + group = self.create_group() + run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + mock_trigger_explorer.return_value = run + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, + format="json", + ) + + assert response.status_code == 202, response.data + assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause""" From e8f415e822486ba13a99ce8f40d9a8a0def89f58 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:58:04 -0800 Subject: [PATCH 11/27] ref(seer): Clarify why the endpoint ignores the agentic triage referrer --- src/sentry/seer/endpoints/group_ai_autofix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 93010d501f0e..4b6bfb155757 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,7 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage runs skip the credit check, so callers can't claim to be it. + # Real agentic triage runs never come through this endpoint, so don't trust a + # request claiming to be one. Agentic triage runs skip the Seer credit check. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 396779d7c000946eb2fba43bd4360e90f3824eb2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:59:06 -0800 Subject: [PATCH 12/27] feat(seer): Check Seer credits before replying to an Autofix single run Restores the single run reply check, using the placeholder estimate for now. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 408b573b403e..82b9a0b31dc8 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5a894919c0a3..b945dfb0ea83 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From 90c5db5f8f301e460de7cd339f3e3358bf6c2ba6 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:40:39 -0800 Subject: [PATCH 13/27] cleaning up diff --- src/sentry/seer/autofix/feature/dispatch.py | 5 +++-- src/sentry/seer/credits.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..326785f2f95c 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -295,11 +295,12 @@ def trigger_autofix_feature( # Avoid a circular import through the legacy Autofix dispatcher. from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + is_new_run = args.existing_run_id is None uses_quota = check_autofix_billing( group, args.step.seer_operation, args.referrer, - is_new_run=args.existing_run_id is None, + is_new_run=is_new_run, skip_quota_for_free_cohort=args.allow_free_cohort, ) @@ -391,7 +392,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if args.existing_run_id is None: + if is_new_run: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..d6cff3e7f32d 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -66,7 +66,7 @@ class SeerOperationConfig: def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: """ - Checks whether the organization has enough Seer credits to run the operation + Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. """ config = SEER_OPERATIONS[operation] @@ -82,7 +82,7 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): From ff4faaef48b3f2ac633a425603f81f04d85996bd Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:05:09 -0800 Subject: [PATCH 14/27] ref(seer): Restore the billing comment in trigger_autofix_agent --- src/sentry/seer/autofix/autofix_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 588cec9a4705..275a475ee4d5 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,6 +463,11 @@ def trigger_autofix_agent( """ outcome = "success" try: + # check billing quota for triggering a new autofix run + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). The API endpoint never sets this flag, + # so manual triggers still require quota. + # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( group, step.seer_operation, From b86335d8d96a012ea9b3d57d971a624168cabad9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:18:02 -0800 Subject: [PATCH 15/27] cleaning up --- src/sentry/seer/autofix/autofix_agent.py | 3 +-- src/sentry/seer/autofix/feature/dispatch.py | 1 - src/sentry/seer/endpoints/group_ai_autofix.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 275a475ee4d5..f7a2f66c5f2b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,7 +463,7 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run + # Seat-based orgs are checked for billing quota when triggering a new autofix run. # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). The API endpoint never sets this flag, # so manual triggers still require quota. @@ -848,7 +848,6 @@ def trigger_push_changes( seer_run = SeerRun.objects.filter( organization_id=group.organization.id, seer_run_state_id=run_id ).first() - # Open PR always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_OPEN_PR, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 326785f2f95c..e2b7fda54543 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,7 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4b6bfb155757..1d8af3cdc4cf 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,8 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Real agentic triage runs never come through this endpoint, so don't trust a - # request claiming to be one. Agentic triage runs skip the Seer credit check. + # Agentic triage is not exposed via the API. Don't trust such a referrer since + # it may have token-based billing implications. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 245aca91af2f299185fcdf5ee22924c413b5457a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:19:02 -0800 Subject: [PATCH 16/27] fix ordering --- src/sentry/seer/autofix/feature/dispatch.py | 120 ++++++++++---------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e2b7fda54543..112efc9ade08 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,66 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def continue_single_run( - agent_run: SeerAgentRun, - group: Group, - reply: str, - *, - user: User | RpcUser, - can_publish: bool, - referrer: AutofixReferrer, -) -> SeerRun: - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - - enable_coding = _org_allows_coding(group) - open_pr = enable_coding and can_publish - payload = AutofixSingleRunFeaturePayload( - group_id=group.id, - project_id=group.project_id, - short_id=group.qualified_short_id or str(group.id), - title=group.title or "Unknown error", - culprit=group.culprit or "unknown", - on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), - should_run_repo_checks=True, - existing_run_id=agent_run.run.seer_run_state_id, - reply=reply, - pr_internal_settings=( - _single_run_pr_internal_settings(group, referrer) if open_pr else None - ), - ) - agent_run_options = AgentRunOptions( - is_context_engine_enabled=False, - enable_frontend_code_search=False, - enable_coding=enable_coding, - enable_pr_context_tools=False, - enable_write_pull_request=open_pr, - ) - - client = SeerAgentClient( - organization=group.organization, - project=group.project, - group=group, - user=user, - enable_coding=enable_coding, - enable_bash_mode=True, - ) - return client.continue_feature_run( - existing_agent_run=agent_run, - payload=payload.dict(), - referrer=referrer.value, - user_org_context=collect_user_org_context(user, group.organization), - proxy_headers=get_proxy_headers(), - agent_run_options=agent_run_options, - ) - - def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: """ Raise if the organization can't afford the operation under token-based @@ -287,6 +227,66 @@ def check_autofix_billing( ) +def continue_single_run( + agent_run: SeerAgentRun, + group: Group, + reply: str, + *, + user: User | RpcUser, + can_publish: bool, + referrer: AutofixReferrer, +) -> SeerRun: + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + + enable_coding = _org_allows_coding(group) + open_pr = enable_coding and can_publish + payload = AutofixSingleRunFeaturePayload( + group_id=group.id, + project_id=group.project_id, + short_id=group.qualified_short_id or str(group.id), + title=group.title or "Unknown error", + culprit=group.culprit or "unknown", + on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), + should_run_repo_checks=True, + existing_run_id=agent_run.run.seer_run_state_id, + reply=reply, + pr_internal_settings=( + _single_run_pr_internal_settings(group, referrer) if open_pr else None + ), + ) + agent_run_options = AgentRunOptions( + is_context_engine_enabled=False, + enable_frontend_code_search=False, + enable_coding=enable_coding, + enable_pr_context_tools=False, + enable_write_pull_request=open_pr, + ) + + client = SeerAgentClient( + organization=group.organization, + project=group.project, + group=group, + user=user, + enable_coding=enable_coding, + enable_bash_mode=True, + ) + return client.continue_feature_run( + existing_agent_run=agent_run, + payload=payload.dict(), + referrer=referrer.value, + user_org_context=collect_user_org_context(user, group.organization), + proxy_headers=get_proxy_headers(), + agent_run_options=agent_run_options, + ) + + def trigger_autofix_feature( group: Group, args: AutofixFeatureArgs, From 80bba2393003f19a88955e554663a45d757ebc2a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:21:07 -0800 Subject: [PATCH 17/27] Restore comment --- src/sentry/seer/autofix/feature/dispatch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 112efc9ade08..419de6744f38 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -300,6 +300,8 @@ def trigger_autofix_feature( args.step.seer_operation, args.referrer, is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. skip_quota_for_free_cohort=args.allow_free_cohort, ) From 914a60cd07bc47d7d50df3f05db2a6f47a1ffda2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:24:39 -0800 Subject: [PATCH 18/27] ref(seer): Keep all billing skip rules in check_autofix_billing --- src/sentry/seer/autofix/feature/dispatch.py | 89 +++++++++------------ 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 419de6744f38..ad74466886e0 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,61 +149,22 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: - """ - Raise if the organization can't afford the operation under token-based - credits. Agentic triage runs stay free. - """ - if ( - not features.has("organizations:seer-credit-checks", group.organization) - or referrer == AutofixReferrer.AGENTIC_TRIAGE - ): - return - if not has_enough_seer_credits(group.organization, operation): - logger.info( - "autofix_feature.dispatch.out_of_credits", +def _check_autofix_quota(group: Group, referrer: str | None) -> None: + """Raise if the organization has no seat-based Seer quota left.""" + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", extra={ "group_id": group.id, "organization_id": group.organization.id, - "operation": operation.value, "referrer": referrer, }, ) - raise SeerOutOfCreditsException() - - -def _check_autofix_quota( - group: Group, - referrer: str | None, - *, - is_new_run: bool, - allow_free_cohort: bool, -) -> bool: - """ - Raise if the organization has no seat-based Seer quota left. Returns whether - the step uses quota, which is recorded once the run starts. - """ - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) - uses_quota = is_new_run and not skip_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - return uses_quota + raise NoSeerQuotaException() def check_autofix_billing( @@ -219,12 +180,34 @@ def check_autofix_billing( checked for credits, everyone else for seat-based quota. Returns whether the operation uses quota. """ - if features.has("organizations:token-billed-seer-enabled", group.organization): - _check_autofix_credits(group, operation, referrer) + organization = group.organization + if features.has("organizations:token-billed-seer-enabled", organization): + if ( + features.has("organizations:seer-credit-checks", organization) + and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. + and not has_enough_seer_credits(organization, operation) + ): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() return False - return _check_autofix_quota( - group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + + # Quota counts runs, so only new runs use it. + # Free cohort orgs bypass quota only when called from agentic triage + # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. + uses_quota = is_new_run and not ( + skip_quota_for_free_cohort and is_free_cohort_org(organization) ) + if uses_quota: + _check_autofix_quota(group, referrer) + return uses_quota def continue_single_run( From 28e59907cf76b2a9b874161de47b5bfaf9825327 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:33:09 -0800 Subject: [PATCH 19/27] refactor billing helper --- src/sentry/seer/autofix/feature/dispatch.py | 50 ++++++++++----------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index ad74466886e0..4c650cca6ad9 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,24 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_quota(group: Group, referrer: str | None) -> None: - """Raise if the organization has no seat-based Seer quota left.""" - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - def check_autofix_billing( group: Group, operation: SeerOperation, @@ -177,10 +159,12 @@ def check_autofix_billing( ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are - checked for credits, everyone else for seat-based quota. Returns whether the - operation uses quota. + checked for credits, and everyone else is checked for quota once per run. + Returns whether the operation uses quota. """ organization = group.organization + + # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( features.has("organizations:seer-credit-checks", organization) @@ -199,14 +183,26 @@ def check_autofix_billing( raise SeerOutOfCreditsException() return False - # Quota counts runs, so only new runs use it. - # Free cohort orgs bypass quota only when called from agentic triage - # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. - uses_quota = is_new_run and not ( - skip_quota_for_free_cohort and is_free_cohort_org(organization) - ) + # Check quota for non-token-billed orgs once per run. + # Free cohort orgs skip quota when the caller passes skip_quota_for_free_cohort=True. + # Only agentic triage does, and it is not exposed via API. + skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) + uses_quota = is_new_run and not skips_quota if uses_quota: - _check_autofix_quota(group, referrer) + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": referrer, + }, + ) + raise NoSeerQuotaException() return uses_quota From 766bc10bc13f1811b71349e821d3726b331b7815 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:39:40 -0800 Subject: [PATCH 20/27] move logs out of helper --- src/sentry/seer/autofix/feature/dispatch.py | 69 +++++++++++---------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 4c650cca6ad9..f636c20c0b88 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -171,15 +171,6 @@ def check_autofix_billing( and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. and not has_enough_seer_credits(organization, operation) ): - logger.info( - "autofix_feature.dispatch.out_of_credits", - extra={ - "group_id": group.id, - "organization_id": organization.id, - "operation": operation.value, - "referrer": referrer, - }, - ) raise SeerOutOfCreditsException() return False @@ -188,21 +179,11 @@ def check_autofix_billing( # Only agentic triage does, and it is not exposed via API. skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) uses_quota = is_new_run and not skips_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() + if uses_quota and not quotas.backend.check_seer_quota( + org_id=organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ): + raise NoSeerQuotaException() return uses_quota @@ -274,15 +255,37 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None - uses_quota = check_autofix_billing( - group, - args.step.seer_operation, - args.referrer, - is_new_run=is_new_run, - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota_for_free_cohort=args.allow_free_cohort, - ) + try: + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + except NoSeerQuotaException: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": args.referrer.value, + }, + ) + raise + except SeerOutOfCreditsException: + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": args.step.seer_operation.value, + "referrer": args.referrer.value, + }, + ) + raise single_run = should_use_single_run( group, From 6c0043dddb83fe8e502df025c1719775a275a7be Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:45:45 -0800 Subject: [PATCH 21/27] small test fix: --- tests/sentry/seer/autofix/feature/test_dispatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..2bf923774c12 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -348,7 +348,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, ): - mock_quotas.backend.check_seer_quota.return_value = True MockClient.return_value.start_feature_run.return_value = fake_run run = trigger_autofix_feature( @@ -362,6 +361,8 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") From 1c7c1906056cd11fc5818b36256c423ed39a38eb Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:49:40 -0800 Subject: [PATCH 22/27] cleaning up tests --- tests/sentry/seer/autofix/feature/test_dispatch.py | 7 ++----- tests/sentry/seer/autofix/test_autofix_agent.py | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 2bf923774c12..751f07dd2818 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -320,7 +320,7 @@ def test_raises_when_out_of_credits_for_a_continued_step( """Unlike quota, credits are checked for every step, not only new runs.""" with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): with pytest.raises(SeerOutOfCreditsException): trigger_autofix_feature( @@ -333,7 +333,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( ), ) - mock_quotas.backend.check_seer_quota.assert_not_called() MockClient.return_value.continue_feature_run.assert_not_called() @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @@ -346,7 +345,7 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): MockClient.return_value.start_feature_run.return_value = fake_run @@ -361,8 +360,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() - mock_quotas.backend.check_seer_quota.assert_not_called() - mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index dd598ac0a01e..af9eb7ef1951 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -821,8 +821,6 @@ def test_agentic_triage_skips_credit_check( ) mock_check_seer_credits.assert_not_called() - mock_check_quota.assert_not_called() - mock_feature.assert_called_once() @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") From 451f3bda8485afed7a78612d1894e2aeb4f5cc05 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:28:49 +0000 Subject: [PATCH 23/27] ref(seer-billing): Pass organization to check_autofix_billing Co-Authored-By: Sofia Rest --- src/sentry/seer/autofix/autofix_agent.py | 4 ++-- src/sentry/seer/autofix/feature/dispatch.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index f7a2f66c5f2b..9c138164c6f1 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -469,7 +469,7 @@ def trigger_autofix_agent( # so manual triggers still require quota. # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( - group, + group.organization, step.seer_operation, referrer, is_new_run=run_id is None, @@ -849,7 +849,7 @@ def trigger_push_changes( organization_id=group.organization.id, seer_run_state_id=run_id ).first() check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_OPEN_PR, seer_run.referrer if seer_run else None, is_new_run=False, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index f636c20c0b88..e5fc4c3ca0af 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -13,6 +13,7 @@ DataCategory, ) from sentry.models.group import Group +from sentry.models.organization import Organization from sentry.models.project import Project from sentry.seer.agent.client import SeerAgentClient from sentry.seer.agent.client_utils import ( @@ -150,7 +151,7 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( - group: Group, + organization: Organization, operation: SeerOperation, referrer: str | None, *, @@ -162,8 +163,6 @@ def check_autofix_billing( checked for credits, and everyone else is checked for quota once per run. Returns whether the operation uses quota. """ - organization = group.organization - # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( @@ -199,7 +198,7 @@ def continue_single_run( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, agent_run.run.referrer, is_new_run=False, @@ -257,7 +256,7 @@ def trigger_autofix_feature( is_new_run = args.existing_run_id is None try: uses_quota = check_autofix_billing( - group, + group.organization, args.step.seer_operation, args.referrer, is_new_run=is_new_run, From ab0d0aa2223dd6df6afab58d1c8809a448508ae9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 13:08:32 -0800 Subject: [PATCH 24/27] ref(seer-billing): Check Autofix billing only in dispatch trigger_autofix_agent checked billing, then called trigger_autofix_feature, which checked it again with the same arguments. Keep only the dispatch check, and test a token-billed org with credit checks off. --- src/sentry/seer/autofix/autofix_agent.py | 13 --- .../seer/autofix/feature/test_dispatch.py | 52 +++++++++++ .../sentry/seer/autofix/test_autofix_agent.py | 91 ------------------- 3 files changed, 52 insertions(+), 104 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 9c138164c6f1..1b8a2a6d609b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,19 +463,6 @@ def trigger_autofix_agent( """ outcome = "success" try: - # Seat-based orgs are checked for billing quota when triggering a new autofix run. - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - # Token-billed orgs are checked for credits on every step instead. - check_autofix_billing( - group.organization, - step.seer_operation, - referrer, - is_new_run=run_id is None, - skip_quota_for_free_cohort=allow_free_cohort, - ) - if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 751f07dd2818..809188a99de6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -283,6 +283,58 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + def test_non_free_cohort_org_checks_quota_and_records_usage(self) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + allow_free_cohort=True, + ), + ) + + assert run is fake_run + mock_quotas.backend.check_seer_quota.assert_called_once() + mock_quotas.backend.record_seer_run.assert_called_once() + + @with_feature("organizations:token-billed-seer-enabled") + @patch("sentry.seer_credit.backend.check_seer_credits") + def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index af9eb7ef1951..95e41640240d 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -9,7 +9,6 @@ ) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent -from sentry.constants import DataCategory from sentry.models.activity import Activity from sentry.models.group import Group from sentry.seer.agent.client_models import ( @@ -708,46 +707,6 @@ def test_pr_iteration_requires_existing_pr( mock_feature.assert_not_called() - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) - def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMock) -> None: - with pytest.raises(NoSeerQuotaException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_quota.assert_called_once_with( - org_id=self.group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - mock_feature.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - def test_when_out_of_credits( - self, - mock_check_quota: MagicMock, - mock_feature: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - with pytest.raises(SeerOutOfCreditsException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_seer_credits.assert_called_once() - mock_check_quota.assert_not_called() - mock_feature.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -772,56 +731,6 @@ def test_free_cohort_skips_quota_check( mock_check_quota.assert_not_called() assert mock_feature.call_args.args[1].allow_free_cohort is True - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( - self, mock_feature: MagicMock, mock_broadcast: MagicMock, mock_check_quota: MagicMock - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - with ( - patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), - ): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - allow_free_cohort=True, - ) - - mock_check_quota.assert_called_once() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_agentic_triage_skips_credit_check( - self, - mock_feature: MagicMock, - mock_broadcast: MagicMock, - mock_check_quota: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - ) - - mock_check_seer_credits.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") From 89bdfdcc00ade7ac8fa9ff52c0b5b42c78ce0e5f Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 15:25:38 -0800 Subject: [PATCH 25/27] ref(seer-billing): Drop the agentic triage referrer exemption Credit checks no longer skip runs by referrer. Agentic triage runs are checked like any other until a follow-up marks them by feature instead. Credit checks are behind seer-credit-checks, so nothing changes in production. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 14 +++---- src/sentry/seer/autofix/feature/dispatch.py | 11 ++--- src/sentry/seer/endpoints/group_ai_autofix.py | 8 +--- .../seer/autofix/feature/test_dispatch.py | 41 ------------------- .../sentry/seer/autofix/test_autofix_agent.py | 26 ------------ .../seer/endpoints/test_group_ai_autofix.py | 19 --------- 6 files changed, 9 insertions(+), 110 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 1b8a2a6d609b..af99e4c3e05f 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -832,15 +832,7 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - check_autofix_billing( - group.organization, - SeerOperation.AUTOFIX_OPEN_PR, - seer_run.referrer if seer_run else None, - is_new_run=False, - ) + check_autofix_billing(group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False) client = get_autofix_agent_client(group, user=user) @@ -859,6 +851,10 @@ def trigger_push_changes( ) ) + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e5fc4c3ca0af..be25d1b1c476 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -153,7 +153,6 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( organization: Organization, operation: SeerOperation, - referrer: str | None, *, is_new_run: bool, skip_quota_for_free_cohort: bool = False, @@ -165,11 +164,9 @@ def check_autofix_billing( """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if ( - features.has("organizations:seer-credit-checks", organization) - and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. - and not has_enough_seer_credits(organization, operation) - ): + if features.has( + "organizations:seer-credit-checks", organization + ) and not has_enough_seer_credits(organization, operation): raise SeerOutOfCreditsException() return False @@ -200,7 +197,6 @@ def continue_single_run( check_autofix_billing( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, is_new_run=False, ) @@ -258,7 +254,6 @@ def trigger_autofix_feature( uses_quota = check_autofix_billing( group.organization, args.step.seer_operation, - args.referrer, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 1d8af3cdc4cf..4767dde92f13 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,16 +165,10 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - referrer = AutofixReferrer(raw) + return AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage is not exposed via the API. Don't trust such a referrer since - # it may have token-based billing implications. - if referrer == AutofixReferrer.AGENTIC_TRIAGE: - logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) - return AutofixReferrer.UNKNOWN - return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 809188a99de6..62f8d3c9e03c 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -387,32 +387,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( MockClient.return_value.continue_feature_run.assert_not_called() - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: - fake_run = self.create_seer_run(organization=self.organization, type="feature_run") - - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas"), - ): - MockClient.return_value.start_feature_run.return_value = fake_run - - run = trigger_autofix_feature( - self.group, - AutofixFeatureArgs( - referrer=AutofixReferrer.AGENTIC_TRIAGE, - step=AutofixStep.ROOT_CAUSE, - step_args=RCAStepArgs(), - ), - ) - - assert run is fake_run - mock_check_seer_credits.assert_not_called() - def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -763,18 +737,3 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ) mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 95e41640240d..555ef5924e20 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -1685,32 +1685,6 @@ def test_raises_when_out_of_credits( mock_post.assert_not_called() - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.agent.client.make_agent_update_request") - def test_agentic_triage_run_skips_credit_check( - self, mock_post: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - # The push is called with another referrer; the run's own referrer decides. - self.create_seer_run( - organization=self.organization, - seer_run_state_id=123, - referrer=AutofixReferrer.AGENTIC_TRIAGE.value, - ) - - self._push( - mock_post, - features={ - "organizations:seer-credit-checks": True, - "organizations:token-billed-seer-enabled": True, - }, - ) - - mock_check_seer_credits.assert_not_called() - mock_post.assert_called_once() - @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index f4ab9f832b8c..10c804a02408 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,25 +1198,6 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") - def test_post_agentic_triage_referrer_is_unknown( - self, mock_trigger_explorer: MagicMock - ) -> None: - """Agentic triage runs skip the credit check, so the API can't claim to be one.""" - group = self.create_group() - run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) - mock_trigger_explorer.return_value = run - - self.login_as(user=self.user) - response = self.client.post( - self._get_url(group.id), - data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, - format="json", - ) - - assert response.status_code == 202, response.data - assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause""" From 5e7ccfca3eaf68761cf27000f3c4ff6ba8c8bd68 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 10:57:44 -0800 Subject: [PATCH 26/27] ref(seer-billing): Pass the Seer feature to credit checks An operation only sets the credit estimate, since it can run under more than one feature, so callers pass the feature explicitly. The credit check endpoint now takes a feature param. --- src/sentry/seer/autofix/feature/dispatch.py | 4 +- src/sentry/seer/credits.py | 54 +++++++------------ .../endpoints/organization_seer_agent_chat.py | 4 +- .../organization_seer_credit_check.py | 8 ++- .../test_organization_seer_credit_check.py | 39 +++++++++++--- tests/sentry/seer/test_credits.py | 27 +++++++--- 6 files changed, 80 insertions(+), 56 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index be25d1b1c476..165bd37e86b9 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -42,7 +42,7 @@ get_automation_handoff, is_free_cohort_org, ) -from sentry.seer.credits import SeerOperation, has_enough_seer_credits +from sentry.seer.credits import SeerFeature, SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -166,7 +166,7 @@ def check_autofix_billing( if features.has("organizations:token-billed-seer-enabled", organization): if features.has( "organizations:seer-credit-checks", organization - ) and not has_enough_seer_credits(organization, operation): + ) and not has_enough_seer_credits(organization, SeerFeature.AUTOFIX, operation): raise SeerOutOfCreditsException() return False diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index d6cff3e7f32d..6debf2e2f7f0 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass from enum import StrEnum from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( @@ -29,62 +28,47 @@ class SeerFeature(StrEnum): EXPLORER = "explorer" -@dataclass(frozen=True) -class SeerOperationConfig: - feature: SeerFeature - estimate: Callable[[], int] - - # TODO(CW-2314): Fake value until we have real estimates. PLACEHOLDER_CREDIT_ESTIMATE = 100 -SEER_OPERATIONS: dict[SeerOperation, SeerOperationConfig] = { - SeerOperation.AGENT_CHAT_MESSAGE: SeerOperationConfig( - SeerFeature.EXPLORER, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_ROOT_CAUSE: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_SOLUTION: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_CODE_CHANGES: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_OPEN_PR: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), +# The operation sets the estimate. Callers pass the feature, since an operation can run under +# more than one feature. +SEER_OPERATION_ESTIMATES: dict[SeerOperation, Callable[[], int]] = { + SeerOperation.AGENT_CHAT_MESSAGE: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_ROOT_CAUSE: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_SOLUTION: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_CODE_CHANGES: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_OPEN_PR: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_PR_ITERATION: lambda: PLACEHOLDER_CREDIT_ESTIMATE, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: lambda: PLACEHOLDER_CREDIT_ESTIMATE, } -def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: +def has_enough_seer_credits( + organization: Organization, feature: SeerFeature, operation: SeerOperation +) -> bool: """ Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. """ - config = SEER_OPERATIONS[operation] response = seer_credit.backend.check_seer_credits( CheckSeerCreditsRequest( organization_id=organization.id, - feature_id=config.feature.value, - predicted_credit_usage=config.estimate(), + feature_id=feature.value, + predicted_credit_usage=SEER_OPERATION_ESTIMATES[operation](), take_lease=False, ) ) return response.has_enough_credits -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: +def can_run_seer_operation( + organization: Organization, feature: SeerFeature, operation: SeerOperation +) -> bool: """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): return True - return has_enough_seer_credits(organization, operation) + return has_enough_seer_credits(organization, feature, operation) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 081e624fdcb2..15f40820ec08 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -37,7 +37,7 @@ from sentry.seer.autofix.feature.dispatch import continue_single_run from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run -from sentry.seer.credits import SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation from sentry.seer.endpoints.group_ai_autofix import RUN_IN_FLIGHT_CODE from sentry.seer.endpoints.utils import ResolvedSeerRun, resolve_seer_run from sentry.seer.models import SeerApiError, SeerPermissionError @@ -439,7 +439,7 @@ def post( raise PermissionDenied(error) # Do not bill for dashboard generation. if not is_dashboard_generate_run and not can_run_seer_operation( - organization, SeerOperation.AGENT_CHAT_MESSAGE + organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE ): return Response( { diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index 56bef39f973e..c204bbce49d1 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -10,11 +10,12 @@ from sentry.api.bases.organization import OrganizationEndpoint from sentry.models.organization import Organization from sentry.ratelimits.config import RateLimitConfig -from sentry.seer.credits import SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation from sentry.types.ratelimit import RateLimit, RateLimitCategory class SeerCreditCheckSerializer(serializers.Serializer[dict[str, str]]): + feature = serializers.ChoiceField(choices=[feature.value for feature in SeerFeature]) operation = serializers.ChoiceField(choices=[operation.value for operation in SeerOperation]) @@ -45,5 +46,8 @@ def get(self, request: Request, organization: Organization) -> Response: if not serializer.is_valid(): return Response(serializer.errors, status=400) + feature = SeerFeature(serializer.validated_data["feature"]) operation = SeerOperation(serializer.validated_data["operation"]) - return Response({"hasEnoughCredits": can_run_seer_operation(organization, operation)}) + return Response( + {"hasEnoughCredits": can_run_seer_operation(organization, feature, operation)} + ) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 9a7303e5b8d3..0f65db2c5b18 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -7,7 +7,7 @@ CheckSeerCreditsResponse, ) -from sentry.seer.credits import SEER_OPERATIONS, SeerOperation +from sentry.seer.credits import SEER_OPERATION_ESTIMATES, SeerOperation from sentry.testutils.cases import APITestCase from sentry.testutils.helpers.features import with_feature @@ -27,14 +27,16 @@ def setUp(self) -> None: def test_get_returns_has_enough_credits_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response(self.organization.slug, operation="autofix_root_cause") + response = self.get_success_response( + self.organization.slug, feature="autofix", operation="autofix_root_cause" + ) assert response.data == {"hasEnoughCredits": True} mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, feature_id="autofix", - predicted_credit_usage=SEER_OPERATIONS[SeerOperation.AUTOFIX_ROOT_CAUSE].estimate(), + predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) ) @@ -47,13 +49,17 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( def test_get_returns_not_enough_credits_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response(self.organization.slug, operation="agent_chat_message") + response = self.get_success_response( + self.organization.slug, feature="explorer", operation="agent_chat_message" + ) assert response.data == {"hasEnoughCredits": False} assert mock_check_seer_credits.call_args.args[0].feature_id == "explorer" def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> None: - response = self.get_success_response(self.organization.slug, operation="autofix_solution") + response = self.get_success_response( + self.organization.slug, feature="autofix", operation="autofix_solution" + ) assert response.data == {"hasEnoughCredits": True} @@ -65,13 +71,30 @@ def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> Non def test_get_returns_has_enough_credits_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response(self.organization.slug, operation="autofix_solution") + response = self.get_success_response( + self.organization.slug, feature="autofix", operation="autofix_solution" + ) assert response.data == {"hasEnoughCredits": True} mock_check_seer_credits.assert_not_called() def test_get_returns_400_when_operation_missing(self) -> None: - self.get_error_response(self.organization.slug, status_code=400) + self.get_error_response(self.organization.slug, feature="autofix", status_code=400) def test_get_returns_400_when_operation_invalid(self) -> None: - self.get_error_response(self.organization.slug, operation="nightshift", status_code=400) + self.get_error_response( + self.organization.slug, feature="autofix", operation="nightshift", status_code=400 + ) + + def test_get_returns_400_when_feature_missing(self) -> None: + self.get_error_response( + self.organization.slug, operation="autofix_root_cause", status_code=400 + ) + + def test_get_returns_400_when_feature_invalid(self) -> None: + self.get_error_response( + self.organization.slug, + feature="nightshift", + operation="autofix_root_cause", + status_code=400, + ) diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index 0b730e6d390a..ab40c1306bec 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -7,7 +7,12 @@ CheckSeerCreditsResponse, ) -from sentry.seer.credits import SEER_OPERATIONS, SeerOperation, can_run_seer_operation +from sentry.seer.credits import ( + SEER_OPERATION_ESTIMATES, + SeerFeature, + SeerOperation, + can_run_seer_operation, +) from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -15,7 +20,7 @@ class SeerCreditsTest(TestCase): def test_every_operation_has_a_credit_estimate(self) -> None: for operation in SeerOperation: - assert SEER_OPERATIONS[operation].estimate() > 0 + assert SEER_OPERATION_ESTIMATES[operation]() > 0 @with_feature("organizations:token-billed-seer-enabled") @patch( @@ -25,7 +30,9 @@ def test_every_operation_has_a_credit_estimate(self) -> None: def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( self, mock_check_seer_credits: MagicMock ) -> None: - assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + assert can_run_seer_operation( + self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + ) mock_check_seer_credits.assert_not_called() @with_feature("organizations:seer-credit-checks") @@ -36,7 +43,9 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( def test_can_run_seer_operation_skips_billing_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: - assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + assert can_run_seer_operation( + self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + ) mock_check_seer_credits.assert_not_called() @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @@ -47,12 +56,14 @@ def test_can_run_seer_operation_skips_billing_when_not_token_billed( def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: - assert not can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + assert not can_run_seer_operation( + self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + ) mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, feature_id="autofix", - predicted_credit_usage=SEER_OPERATIONS[SeerOperation.AUTOFIX_ROOT_CAUSE].estimate(), + predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) ) @@ -65,4 +76,6 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( def test_can_run_seer_operation_returns_true_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: - assert can_run_seer_operation(self.organization, SeerOperation.AGENT_CHAT_MESSAGE) + assert can_run_seer_operation( + self.organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + ) From 1a78b12aa113bbb79a023a9d1f1e6ad7e2b74bc5 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 11:21:55 -0800 Subject: [PATCH 27/27] ref(seer-billing): Pick the credit check feature from the operation The UI can't tell runs agentic triage started apart yet, so the credit check endpoint picks the feature itself instead of requiring the UI to send it. --- .../organization_seer_credit_check.py | 8 +++-- .../test_organization_seer_credit_check.py | 35 ++++--------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index c204bbce49d1..16d580ddedfc 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -15,7 +15,6 @@ class SeerCreditCheckSerializer(serializers.Serializer[dict[str, str]]): - feature = serializers.ChoiceField(choices=[feature.value for feature in SeerFeature]) operation = serializers.ChoiceField(choices=[operation.value for operation in SeerOperation]) @@ -46,8 +45,13 @@ def get(self, request: Request, organization: Organization) -> Response: if not serializer.is_valid(): return Response(serializer.errors, status=400) - feature = SeerFeature(serializer.validated_data["feature"]) operation = SeerOperation(serializer.validated_data["operation"]) + # TODO: Let the UI pass the feature once it can tell agentic triage runs apart. + feature = ( + SeerFeature.EXPLORER + if operation == SeerOperation.AGENT_CHAT_MESSAGE + else SeerFeature.AUTOFIX + ) return Response( {"hasEnoughCredits": can_run_seer_operation(organization, feature, operation)} ) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 0f65db2c5b18..92246f1f6a68 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -27,9 +27,7 @@ def setUp(self) -> None: def test_get_returns_has_enough_credits_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response( - self.organization.slug, feature="autofix", operation="autofix_root_cause" - ) + response = self.get_success_response(self.organization.slug, operation="autofix_root_cause") assert response.data == {"hasEnoughCredits": True} mock_check_seer_credits.assert_called_once_with( @@ -49,17 +47,13 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( def test_get_returns_not_enough_credits_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response( - self.organization.slug, feature="explorer", operation="agent_chat_message" - ) + response = self.get_success_response(self.organization.slug, operation="agent_chat_message") assert response.data == {"hasEnoughCredits": False} assert mock_check_seer_credits.call_args.args[0].feature_id == "explorer" def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> None: - response = self.get_success_response( - self.organization.slug, feature="autofix", operation="autofix_solution" - ) + response = self.get_success_response(self.organization.slug, operation="autofix_solution") assert response.data == {"hasEnoughCredits": True} @@ -71,30 +65,13 @@ def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> Non def test_get_returns_has_enough_credits_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: - response = self.get_success_response( - self.organization.slug, feature="autofix", operation="autofix_solution" - ) + response = self.get_success_response(self.organization.slug, operation="autofix_solution") assert response.data == {"hasEnoughCredits": True} mock_check_seer_credits.assert_not_called() def test_get_returns_400_when_operation_missing(self) -> None: - self.get_error_response(self.organization.slug, feature="autofix", status_code=400) + self.get_error_response(self.organization.slug, status_code=400) def test_get_returns_400_when_operation_invalid(self) -> None: - self.get_error_response( - self.organization.slug, feature="autofix", operation="nightshift", status_code=400 - ) - - def test_get_returns_400_when_feature_missing(self) -> None: - self.get_error_response( - self.organization.slug, operation="autofix_root_cause", status_code=400 - ) - - def test_get_returns_400_when_feature_invalid(self) -> None: - self.get_error_response( - self.organization.slug, - feature="nightshift", - operation="autofix_root_cause", - status_code=400, - ) + self.get_error_response(self.organization.slug, operation="nightshift", status_code=400)