Merge pull request #1240 from paullizer/fix/tabular-analyze-truncation-schema-deferral

Fix tabular Analyze truncation and combined-run schema lock

Author: paullizer

application/single_app/background_tasks.py

@@ -705,7 +705,10 @@ def run_file_sync_scheduler_loop():
         try:
             lock_document = acquire_distributed_task_lock('file_sync_scheduler_scan', lease_seconds=300)
             if lock_document:
-                check_due_file_sync_sources_once()
+                due_sources = check_due_file_sync_sources_once()
+                debug_print(f"File Sync scheduler tick processed {len(due_sources or [])} source(s).")
+            else:
+                debug_print('Skipping File Sync scheduler tick because another worker holds the lease.')
         except Exception as exc:
             print(f"Error in File Sync scheduler check: {exc}")
             log_event(f"[FILE_SYNC] Error in scheduler check: {exc}", level=logging.ERROR)
@@ -723,7 +726,14 @@ def run_tabular_generated_output_scheduler_loop():
         try:
             lock_document = acquire_distributed_task_lock('tabular_generated_output_scheduler_scan', lease_seconds=120)
             if lock_document:
-                check_due_tabular_generated_output_runs_once()
+                processed_run_ids = check_due_tabular_generated_output_runs_once()
+                debug_print(
+                    f"Tabular generated-output scheduler tick processed {len(processed_run_ids or [])} run(s)."
+                )
+            else:
+                debug_print(
+                    'Skipping tabular generated-output scheduler tick because another worker holds the lease.'
+                )
         except Exception as exc:
             print(f"Error in tabular generated-output scheduler check: {exc}")
             log_event(f"[TABULAR_GENERATED_OUTPUT] Error in scheduler check: {exc}", level=logging.ERROR)
@@ -741,7 +751,10 @@ def run_data_management_scheduler_loop(app=None):
         try:
             lock_document = acquire_distributed_task_lock('data_management_scheduler_scan', lease_seconds=300)
             if lock_document:
-                check_due_data_management_jobs_once(app=app)
+                due_jobs = check_due_data_management_jobs_once(app=app)
+                debug_print(f"Data Management scheduler tick processed {len(due_jobs or [])} job(s).")
+            else:
+                debug_print('Skipping Data Management scheduler tick because another worker holds the lease.')
         except Exception as exc:
             print(f"Error in Data Management scheduler check: {exc}")
             log_event(f"[DATA_MANAGEMENT] Error in scheduler check: {exc}", level=logging.ERROR)

application/single_app/config.py

@@ -96,7 +96,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.250.185"
+VERSION = "0.250.191"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/functions_settings.py

@@ -18,6 +18,7 @@
 import app_settings_cache
 import inspect
 import copy
+import os
 import json
 import uuid
 from support_menu_config import (
@@ -1010,6 +1011,25 @@ def _update_cache(stage):
     _update_cache("after_version_bump")
 
 
+def _env_flag_enabled(name):
+    return str(os.environ.get(name, '')).strip().lower() in {'1', 'true', 'yes', 'on'}
+
+
+def _apply_tabular_parity_env_kill_switch(settings_payload):
+    """Force tabular durable-preflight parity off when the emergency env kill switch is set.
+
+    These parity controls ship active by default with no admin UI toggle; this
+    environment variable is the only rollback path for an operator incident.
+    """
+    if not isinstance(settings_payload, dict):
+        return settings_payload
+    if _env_flag_enabled('SIMPLECHAT_DISABLE_TABULAR_PARITY_DURABLE_PREFLIGHT'):
+        settings_payload['tabular_request_planner_mode'] = 'off'
+        settings_payload['enable_tabular_search_shared_preflight'] = False
+        settings_payload['enable_tabular_analyze_durable_preflight'] = False
+    return settings_payload
+
+
 def get_settings(use_cosmos=False, include_source=False):
     import secrets
     default_settings = {
@@ -1048,9 +1068,9 @@ def get_settings(use_cosmos=False, include_source=False):
         'tabular_analyze_parity_rollout_percent': 100,
         'tabular_analyze_parity_rollout_state': 'active',
         'tabular_background_handoff_mode': 'legacy',
-        'tabular_request_planner_mode': 'off',
-        'enable_tabular_search_shared_preflight': False,
-        'enable_tabular_analyze_durable_preflight': False,
+        'tabular_request_planner_mode': 'active',
+        'enable_tabular_search_shared_preflight': True,
+        'enable_tabular_analyze_durable_preflight': True,
         'enable_tabular_mixed_deferred_composition_planning': False,
         'enable_tabular_multifile_execution_unit_planning': False,
         'tabular_legacy_post_tool_fallback_mode': 'enabled',
@@ -1615,6 +1635,8 @@ def get_settings(use_cosmos=False, include_source=False):
     }
 
     def _format_result(settings_payload, source):
+        if isinstance(settings_payload, dict):
+            settings_payload = _apply_tabular_parity_env_kill_switch(settings_payload)
         if include_source:
             return settings_payload, source
         return settings_payload

application/single_app/functions_tabular_generated_exports.py

@@ -42,6 +42,8 @@
     ANALYSIS_ARTIFACT_ROLE_PRIMARY_ANALYSIS,
     ANALYSIS_ARTIFACT_ROLE_REQUESTED_OUTPUT,
     ANALYSIS_ARTIFACT_ROLE_SUPPORTING_OUTPUT,
+    ANALYSIS_DELIVERABLE_MAX_ARTIFACT_ID_LENGTH,
+    ANALYSIS_DELIVERABLE_MAX_ARTIFACTS,
     build_analysis_deliverable_contract,
     is_analysis_internal_lineage_field,
     project_structured_deliverable_row,
@@ -6855,6 +6857,17 @@ def _normalize_tabular_run_planner_metadata(planner_metadata):
             'action_mode': str(deliverable_contract.get('action_mode') or '').strip().lower()[:40],
             'analysis_required': bool(deliverable_contract.get('analysis_required')),
             'primary_artifact_role': str(deliverable_contract.get('primary_artifact_role') or '').strip().lower()[:80],
+            'requested_artifacts': [
+                {
+                    'artifact_id': str(artifact.get('artifact_id') or '').strip()[:ANALYSIS_DELIVERABLE_MAX_ARTIFACT_ID_LENGTH],
+                    'role': str(artifact.get('role') or '').strip().lower()[:40],
+                    'format': str(artifact.get('format') or '').strip().lower()[:20],
+                    'required': bool(artifact.get('required', True)),
+                    'request_order': _safe_int(artifact.get('request_order'), default=0, minimum=0),
+                }
+                for artifact in list(deliverable_contract.get('requested_artifacts') or [])[:ANALYSIS_DELIVERABLE_MAX_ARTIFACTS]
+                if isinstance(artifact, dict) and str(artifact.get('artifact_id') or '').strip()
+            ],
             'public_output_schema': [
                 str(field_name or '').strip()
                 for field_name in list(deliverable_contract.get('public_output_schema') or [])[:TABULAR_GENERATION_PLAN_MAX_FIELDS]
@@ -8689,6 +8702,33 @@ def _build_or_update_artifact_set_manifest(run):
         'rollback_state': str(existing_manifest.get('rollback_state') or '').strip().lower()[:40],
         'members': members,
     }
+    if (
+        str((run or {}).get('status') or '').strip().lower() == TABULAR_EXPORT_STATUS_COMPLETED
+        and lifecycle_state != TABULAR_ARTIFACT_SET_LIFECYCLE_COMPLETED
+    ):
+        log_event(
+            '[TABULAR_GENERATED_OUTPUT] Artifact set stuck below completed lifecycle on a completed run',
+            {
+                'run_id': run.get('id'),
+                'conversation_id': run.get('conversation_id'),
+                'task_type': manifest.get('task_type'),
+                'persisted_lifecycle_state': str(existing_manifest.get('lifecycle_state') or ''),
+                'recomputed_lifecycle_state': lifecycle_state,
+                'validation_state': manifest.get('validation_state'),
+                'validation_report': existing_manifest.get('validation_report'),
+                'members': [
+                    {
+                        'member_id': member.get('member_id'),
+                        'role': member.get('role'),
+                        'lifecycle_state': member.get('lifecycle_state'),
+                        'validation_state': member.get('validation_state'),
+                        'has_artifact_message_id': bool(member.get('artifact_message_id')),
+                    }
+                    for member in members
+                ],
+            },
+            level=logging.WARNING,
+        )
     return manifest
 
 
@@ -8738,13 +8778,33 @@ def _publish_artifact_set_members(run, published_member_ids):
     ]
     deliverable_contract = _get_tabular_run_deliverable_contract(run)
     artifact_set_valid = True
+    validation_report = None
     if deliverable_contract:
         validation_report = validate_analysis_artifact_set(deliverable_contract, validation_artifacts)
         artifact_set_valid = validation_report.valid
         manifest['validation_state'] = 'validated' if validation_report.valid else 'invalid'
         manifest['validation_report'] = validation_report.to_dict()
     else:
         manifest['validation_state'] = 'validated'
+    log_event(
+        '[TABULAR_GENERATED_OUTPUT] Artifact set publication validation',
+        {
+            'run_id': run.get('id'),
+            'conversation_id': run.get('conversation_id'),
+            'task_type': _normalize_tabular_run_task_type(run.get('task_type')),
+            'published_member_ids': sorted(published_ids),
+            'has_deliverable_contract': bool(deliverable_contract),
+            'artifact_set_valid': artifact_set_valid,
+            'reason_codes': list(validation_report.reason_codes) if validation_report else [],
+            'counts': dict(validation_report.counts) if validation_report else {},
+            'validation_artifacts': validation_artifacts,
+            'expected_artifact_ids': [
+                artifact.get('artifact_id')
+                for artifact in list((deliverable_contract or {}).get('requested_artifacts') or [])
+            ],
+        },
+        level=logging.INFO,
+    )
     manifest['lifecycle_state'] = (
         TABULAR_ARTIFACT_SET_LIFECYCLE_COMPLETED
         if artifact_set_valid
@@ -10946,7 +11006,9 @@ def queue_tabular_generated_output_run(
         'planner_started_at': None,
         'planner_completed_at': None,
         'processed_rows': 0,
-        'output_schema': contract_internal_checkpoint_schema or None,
+        # Only lock the schema up front when real output columns are already known; otherwise
+        # defer to batch-1 discovery instead of validating against a lineage-only placeholder.
+        'output_schema': contract_internal_checkpoint_schema if contract_public_output_schema else None,
         'public_output_schema': contract_public_output_schema,
         'internal_checkpoint_schema': contract_internal_checkpoint_schema,
         'lineage_schema': [
@@ -11103,17 +11165,16 @@ def check_due_tabular_generated_output_runs_once(limit=None):
                     'reason': f"{candidate.get('reason')}; claim or processing did not start",
                 })
 
-    if scanned_candidates or candidates:
-        log_event(
-            '[TABULAR_GENERATED_OUTPUT] Background scheduler scan result',
-            {
-                'scanned_count': len(scanned_candidates),
-                'candidate_count': len(candidates),
-                'status_counts': status_counts,
-                'processed_run_ids': processed,
-                'processed_count': len(processed),
-                'skipped': skipped[:10],
-            },
-            debug_only=True,
-        )
+    log_event(
+        '[TABULAR_GENERATED_OUTPUT] Background scheduler scan result',
+        {
+            'scanned_count': len(scanned_candidates),
+            'candidate_count': len(candidates),
+            'status_counts': status_counts,
+            'processed_run_ids': processed,
+            'processed_count': len(processed),
+            'skipped': skipped[:10],
+        },
+        debug_only=True,
+    )
     return processed

docs/explanation/features/TABULAR_ANALYZE_SEARCH_PARITY_ROLLOUT.md

@@ -37,13 +37,15 @@ Configuration options:
 
 All Phase 8 rollout controls are backend-only and are removed by `sanitize_settings_for_user()` before non-admin frontend settings are returned.
 
+> **Update (0.250.186):** `tabular_request_planner_mode` now defaults to `active` and both `enable_tabular_search_shared_preflight` and `enable_tabular_analyze_durable_preflight` default to `True`. Bounded foreground synthesis for exhaustive row-by-row tabular requests is no longer the default behavior. There is intentionally no admin UI toggle for this; use the `SIMPLECHAT_DISABLE_TABULAR_PARITY_DURABLE_PREFLIGHT` environment variable for emergency rollback (see below).
+
 ## Usage Instructions
 
 How to enable/configure:
-1. Keep the planner mode `off` for the default legacy behavior.
-2. Use `shadow` mode to compare planner decisions without queueing shared-planner durable work.
-3. Enable the relevant single-source Search or Analyze gate before switching that mode to active traffic.
-4. Adjust `tabular_analyze_parity_rollout_percent` for canary cohorts.
+1. The planner mode defaults to `active`; no configuration is required for normal operation.
+2. Use `shadow` mode only to compare planner decisions without queueing shared-planner durable work, for example while validating a new model or source type.
+3. Set the environment variable `SIMPLECHAT_DISABLE_TABULAR_PARITY_DURABLE_PREFLIGHT=true` to force `tabular_request_planner_mode` to `off` and both shared-preflight flags to `False` for every request during an incident; unset it to restore the active default.
+4. Adjust `tabular_analyze_parity_rollout_percent` for canary cohorts if a partial rollout is needed.
 5. Keep `tabular_legacy_post_tool_fallback_mode='enabled'` until operator telemetry shows no required legacy recovery traffic.
 
 Operator telemetry:

docs/explanation/fixes/TABULAR_ANALYZE_SEARCH_PARITY_DEFAULT_ACTIVATION_FIX.md

@@ -0,0 +1,59 @@
+# Tabular Analyze/Search Parity Default Activation Fix
+
+Fixed in version: **0.250.186**
+
+## Issue Description
+
+Customer testing showed Chat/Analyze still answering exhaustive row-by-row tabular requests (for example, "for each row, answer these eight questions") by producing detailed answers for roughly the first row and a half, then stating the remaining rows were unprocessed, truncated, or outside a bounded evidence handoff. This matched the exact failure mode the multi-phase tabular Analyze/Search parity roadmap (`feature/tabular-analyze-search-parity`, `feature/analyze-artifact-output-contract`) was built to eliminate.
+
+## Root Cause Analysis
+
+The durable-preflight parity path (`_maybe_execute_pure_tabular_analyze_preflight` in `functions_workflow_runner.py` for Analyze, and `maybe_queue_search_tabular_generated_output` in `route_backend_chats.py` for Search) is gated by three backend-only settings: `tabular_request_planner_mode`, `enable_tabular_search_shared_preflight`, and `enable_tabular_analyze_durable_preflight`. All three defaulted to `off`/`False` in `functions_settings.py`, and none had an admin UI toggle, so no deployed environment ever ran the durable, exhaustive-coverage path unless an operator manually edited the stored settings document directly (there was no supported way to do this from the Admin Settings UI). Every request instead fell back to the legacy bounded foreground path, which answers using only the tool-call rows that fit in one synthesis turn and explicitly reports the rest as missing/truncated evidence.
+
+## Version Implemented
+
+Fixed in version: **0.250.186**
+
+## Technical Details
+
+### Files Modified
+
+- `application/single_app/functions_settings.py`
+- `application/single_app/config.py`
+- `docs/reference/admin_configuration.md`
+- `docs/explanation/features/TABULAR_ANALYZE_SEARCH_PARITY_ROLLOUT.md`
+- `functional_tests/test_tabular_analyze_search_parity_default_activation.py`
+
+### Code Changes Summary
+
+- `tabular_request_planner_mode` now defaults to `active` (was `off`).
+- `enable_tabular_search_shared_preflight` and `enable_tabular_analyze_durable_preflight` now default to `True` (were `False`).
+- Added `_apply_tabular_parity_env_kill_switch()` in `functions_settings.py`, applied in `get_settings()`'s `_format_result()` choke point on every return path. When the environment variable `SIMPLECHAT_DISABLE_TABULAR_PARITY_DURABLE_PREFLIGHT` is truthy, it forces `tabular_request_planner_mode` back to `off` and both shared-preflight flags back to `False`, regardless of the stored settings document. This gives operators an emergency rollback path without requiring an admin UI toggle or a direct settings edit, consistent with treating always-on behavior as "on unless an operator opts out," not "off until an operator opts in."
+- `enable_tabular_mixed_deferred_composition_planning` and `enable_tabular_multifile_execution_unit_planning` remain `False` by default; per existing documentation these are planning-only metadata controls with no implemented durable execution behind them yet, so enabling them would not change runtime behavior.
+
+### Testing Approach
+
+- New functional test asserts the three defaults via AST-based extraction of `get_settings()`'s literal `default_settings` dict (avoids importing the full Flask app), and asserts the env kill switch forces them back off.
+- Re-ran the existing tabular parity suites (`test_tabular_shared_request_planner.py`, `test_tabular_analyze_shared_preflight_adapter.py`, `test_tabular_search_shared_preflight_adapter.py`, `test_analyze_artifact_phase7_rollout_rollback.py`, `test_tabular_phase8_ui_telemetry_rollout.py`, `test_tabular_execution_settings_sanitization.py`) to confirm no regressions; all pass unchanged because they construct explicit settings fixtures rather than relying on `get_settings()` defaults.
+- Compiled all changed Python files.
+
+## Impact Analysis
+
+Chat, Search, and Analyze now route exhaustive per-row/per-source tabular requests through the durable generated-output path by default, matching the behavior validated across the parity roadmap's Phases 1-9 and the analyze-artifact-output-contract Phases 1-7D. Operators who need to roll back during an incident set one environment variable instead of editing settings directly; no code deploy or Cosmos edit is required to disable, and none is required to re-enable.
+
+## Validation
+
+### Before
+
+- `tabular_request_planner_mode=off`, `enable_tabular_search_shared_preflight=False`, `enable_tabular_analyze_durable_preflight=False` in every environment by default.
+- Exhaustive row-by-row Chat/Analyze requests answered a small bounded subset of rows, then reported the remainder as unprocessed/truncated evidence, even though the durable parity infrastructure to answer exhaustively already existed and was fully merged.
+
+### After
+
+- New defaults route exhaustive tabular requests through the durable preflight/generated-output path automatically.
+- `functional_tests/test_tabular_analyze_search_parity_default_activation.py` passes 2/2.
+- Existing tabular parity regression suites remain green.
+
+## Related Version Updates
+
+- `application/single_app/config.py` was updated to version **0.250.186**.