diff --git a/application/single_app/config.py b/application/single_app/config.py index 6f51aaa99..cfaf7f499 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.260.023" +VERSION = "0.260.024" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/static/js/chat/chat-citation-tracking.js b/application/single_app/static/js/chat/chat-citation-tracking.js new file mode 100644 index 000000000..e849c3bee --- /dev/null +++ b/application/single_app/static/js/chat/chat-citation-tracking.js @@ -0,0 +1,50 @@ +// chat-citation-tracking.js +/** + * Browser mirror of functions_citation_tracking.py tracking detection. + * + * Assistant messages carry both the complete retrieved source arrays + * (hybrid_citations, web_search_citations) and the smaller subsets that the + * final response explicitly cited (cited_hybrid_citations, + * cited_web_search_citations). Surfaces that present media or references as + * supporting the answer must read the cited subsets, while the Sources + * disclosure keeps showing everything that was retrieved. + * + * Messages saved before citation tracking existed carry no cited arrays. Those + * fall back to the full source arrays rather than being parsed at read time, + * matching get_message_reference_citation_buckets() on the server. + */ + +const MIN_CITATION_TRACKING_VERSION = 1; + +function toCitationArray(value) { + return Array.isArray(value) ? value : []; +} + +export function messageHasCitationTracking(message) { + if (!message || typeof message !== "object") { + return false; + } + + const trackingVersion = Number(message.citation_tracking_version); + if (Number.isInteger(trackingVersion) && trackingVersion >= MIN_CITATION_TRACKING_VERSION) { + return true; + } + + return "cited_hybrid_citations" in message || "cited_web_search_citations" in message; +} + +export function getCitedHybridCitations(message, sourceCitations = []) { + if (!messageHasCitationTracking(message)) { + return toCitationArray(sourceCitations); + } + + return toCitationArray(message.cited_hybrid_citations); +} + +export function getCitedWebCitations(message, sourceCitations = []) { + if (!messageHasCitationTracking(message)) { + return toCitationArray(sourceCitations); + } + + return toCitationArray(message.cited_web_search_citations); +} diff --git a/application/single_app/static/js/chat/chat-inline-images.js b/application/single_app/static/js/chat/chat-inline-images.js index 107193a17..9eb6ec133 100644 --- a/application/single_app/static/js/chat/chat-inline-images.js +++ b/application/single_app/static/js/chat/chat-inline-images.js @@ -246,13 +246,13 @@ function normalizeWorkspaceCitationImageItem(rawCitation, index) { }; } -function extractWorkspaceCitationImageItems(hybridCitations = [], seenKeys = new Set()) { +function extractWorkspaceCitationImageItems(citedHybridCitations = [], seenKeys = new Set()) { const items = []; - if (!Array.isArray(hybridCitations) || hybridCitations.length === 0) { + if (!Array.isArray(citedHybridCitations) || citedHybridCitations.length === 0) { return items; } - hybridCitations.forEach((citation, index) => { + citedHybridCitations.forEach((citation, index) => { pushUniqueImageItem(items, seenKeys, normalizeWorkspaceCitationImageItem(citation, index)); }); @@ -287,13 +287,13 @@ function normalizeWebCitationImageItem(rawCitation, index) { }; } -function extractLinkedImageItems(webCitations = [], seenKeys = new Set()) { +function extractLinkedImageItems(citedWebCitations = [], seenKeys = new Set()) { const items = []; - if (!Array.isArray(webCitations) || webCitations.length === 0) { + if (!Array.isArray(citedWebCitations) || citedWebCitations.length === 0) { return items; } - webCitations.forEach((citation, index) => { + citedWebCitations.forEach((citation, index) => { pushUniqueImageItem(items, seenKeys, normalizeWebCitationImageItem(citation, index)); }); @@ -649,10 +649,17 @@ function createImageGalleryCard(result, messageId, index) { return { card }; } +/** + * Render inline image galleries for one assistant message. + * + * The workspace and linked galleries take the cited citation subsets, not the + * full retrieved sets, so images only appear when the response referenced them. + * Agent citations are executed tool results and are always rendered. + */ export async function renderInlineImageGalleries( messageElement, - hybridCitations = [], - webCitations = [], + citedHybridCitations = [], + citedWebCitations = [], agentCitations = [], messageId = "", conversationId = "" @@ -668,8 +675,8 @@ export async function renderInlineImageGalleries( container.querySelectorAll(".inline-image-gallery-card").forEach((card) => card.remove()); - const hasHybridCitations = Array.isArray(hybridCitations) && hybridCitations.length > 0; - const hasWebCitations = Array.isArray(webCitations) && webCitations.length > 0; + const hasHybridCitations = Array.isArray(citedHybridCitations) && citedHybridCitations.length > 0; + const hasWebCitations = Array.isArray(citedWebCitations) && citedWebCitations.length > 0; const hasAgentCitations = Array.isArray(agentCitations) && agentCitations.length > 0; if (!hasHybridCitations && !hasWebCitations && !hasAgentCitations) { container.classList.toggle("d-none", container.children.length === 0); @@ -680,7 +687,7 @@ export async function renderInlineImageGalleries( let galleryIndex = 0; const seenImageKeys = new Set(); - const workspaceItems = extractWorkspaceCitationImageItems(hybridCitations, seenImageKeys); + const workspaceItems = extractWorkspaceCitationImageItems(citedHybridCitations, seenImageKeys); if (workspaceItems.length > 0 && remainingSlots > 0) { const workspaceGallery = buildImageGalleryResult( "Workspace images", @@ -697,11 +704,11 @@ export async function renderInlineImageGalleries( } } - const linkedItems = extractLinkedImageItems(webCitations, seenImageKeys); + const linkedItems = extractLinkedImageItems(citedWebCitations, seenImageKeys); if (linkedItems.length > 0 && remainingSlots > 0) { const linkedGallery = buildImageGalleryResult( "Linked images", - "Direct image links returned with this response.", + "Image links cited in this response.", linkedItems.slice(0, remainingSlots), "Linked sources", linkedItems.length diff --git a/application/single_app/static/js/chat/chat-inline-videos.js b/application/single_app/static/js/chat/chat-inline-videos.js index be1a2ca5f..0a5b1aab7 100644 --- a/application/single_app/static/js/chat/chat-inline-videos.js +++ b/application/single_app/static/js/chat/chat-inline-videos.js @@ -280,13 +280,13 @@ function normalizeWorkspaceCitationVideoItem(rawCitation, index = 0) { }; } -function extractWorkspaceCitationVideoItems(hybridCitations = [], seenKeys = new Set()) { +function extractWorkspaceCitationVideoItems(citedHybridCitations = [], seenKeys = new Set()) { const items = []; - if (!Array.isArray(hybridCitations) || hybridCitations.length === 0) { + if (!Array.isArray(citedHybridCitations) || citedHybridCitations.length === 0) { return items; } - hybridCitations.forEach((citation, index) => { + citedHybridCitations.forEach((citation, index) => { pushUniqueVideoItem(items, seenKeys, normalizeWorkspaceCitationVideoItem(citation, index)); }); @@ -343,13 +343,13 @@ function normalizeWebCitationVideoItem(rawCitation, index = 0) { }; } -function extractLinkedVideoItems(webCitations = [], seenKeys = new Set()) { +function extractLinkedVideoItems(citedWebCitations = [], seenKeys = new Set()) { const items = []; - if (!Array.isArray(webCitations) || webCitations.length === 0) { + if (!Array.isArray(citedWebCitations) || citedWebCitations.length === 0) { return items; } - webCitations.forEach((citation, index) => { + citedWebCitations.forEach((citation, index) => { pushUniqueVideoItem(items, seenKeys, normalizeWebCitationVideoItem(citation, index)); }); @@ -720,10 +720,17 @@ function createVideoGalleryCard(result) { return { card }; } +/** + * Render inline video galleries for one assistant message. + * + * The workspace and linked galleries take the cited citation subsets, not the + * full retrieved sets, so videos only appear when the response referenced them. + * Agent citations are executed tool results and are always rendered. + */ export async function renderInlineVideoGalleries( messageElement, - hybridCitations = [], - webCitations = [], + citedHybridCitations = [], + citedWebCitations = [], agentCitations = [], conversationId = "" ) { @@ -738,8 +745,8 @@ export async function renderInlineVideoGalleries( container.querySelectorAll(".inline-video-gallery-card").forEach((card) => card.remove()); - const hasHybridCitations = Array.isArray(hybridCitations) && hybridCitations.length > 0; - const hasWebCitations = Array.isArray(webCitations) && webCitations.length > 0; + const hasHybridCitations = Array.isArray(citedHybridCitations) && citedHybridCitations.length > 0; + const hasWebCitations = Array.isArray(citedWebCitations) && citedWebCitations.length > 0; const hasAgentCitations = Array.isArray(agentCitations) && agentCitations.length > 0; if (!hasHybridCitations && !hasWebCitations && !hasAgentCitations) { container.classList.toggle("d-none", container.children.length === 0); @@ -749,7 +756,7 @@ export async function renderInlineVideoGalleries( let remainingSlots = MAX_INLINE_VIDEO_ITEMS; const seenVideoKeys = new Set(); - const workspaceItems = extractWorkspaceCitationVideoItems(hybridCitations, seenVideoKeys); + const workspaceItems = extractWorkspaceCitationVideoItems(citedHybridCitations, seenVideoKeys); if (workspaceItems.length > 0 && remainingSlots > 0) { const workspaceGallery = buildVideoGalleryResult( "Workspace videos", @@ -765,11 +772,11 @@ export async function renderInlineVideoGalleries( } } - const linkedItems = extractLinkedVideoItems(webCitations, seenVideoKeys); + const linkedItems = extractLinkedVideoItems(citedWebCitations, seenVideoKeys); if (linkedItems.length > 0 && remainingSlots > 0) { const linkedGallery = buildVideoGalleryResult( "Linked videos", - "Direct video links returned with this response.", + "Video links cited in this response.", linkedItems.slice(0, remainingSlots), "Linked sources", linkedItems.length diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index 7faec1a20..ffd0c8504 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -32,6 +32,7 @@ import { attachGeneratedImageProposalResults, extractInlineImageProposalBlocks, import { renderInlineVideoGalleries } from './chat-inline-videos.js'; import { renderInlineImageGalleries } from './chat-inline-images.js'; import { renderInlineAzureMaps } from './chat-inline-maps.js'; +import { getCitedHybridCitations, getCitedWebCitations } from './chat-citation-tracking.js'; // Conditionally import TTS if enabled let ttsModule = null; @@ -6010,17 +6011,23 @@ export function appendMessage( } void (async () => { + // Inline galleries present media as supporting the answer, so they render + // only what the response cited. The Sources disclosure keeps the complete + // retrieved set. + const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations); + const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations); + await renderInlineVideoGalleries( messageDiv, - hybridCitations || [], - webCitations || [], + citedHybridCitations, + citedWebCitations, agentCitations || [], messageConversationId ); await renderInlineImageGalleries( messageDiv, - hybridCitations || [], - webCitations || [], + citedHybridCitations, + citedWebCitations, agentCitations || [], messageId, messageConversationId diff --git a/docs/explanation/fixes/INLINE_MEDIA_CITED_ONLY_GATING_FIX.md b/docs/explanation/fixes/INLINE_MEDIA_CITED_ONLY_GATING_FIX.md new file mode 100644 index 000000000..9d3df981f --- /dev/null +++ b/docs/explanation/fixes/INLINE_MEDIA_CITED_ONLY_GATING_FIX.md @@ -0,0 +1,67 @@ +# Inline Media Cited-Only Gating Fix + +Fixed/Implemented in version: **0.260.024** + +GitHub issue: [#1329](https://github.com/microsoft/simplechat/issues/1329) (follow-up to [#1249](https://github.com/microsoft/simplechat/issues/1249)) + +## Issue Description + +Assistant messages rendered inline image and video galleries for every media document returned by retrieval, not just the media the response actually cited. A workspace search that surfaced five image files produced five inline gallery tiles even when the answer referenced only one of them, or none at all. + +Because the galleries sit directly inside the message bubble, unrelated media was presented as though it supported the answer. The five-item gallery cap could also be consumed entirely by retrieval noise, pushing genuinely cited media out of view, and each unreferenced workspace file triggered an additional enhanced-citation fetch. + +## Root Cause Analysis + +Issue #1249 introduced the separation between retrieved sources and exact references. Every assistant message now persists `cited_hybrid_citations` and `cited_web_search_citations` alongside the complete `hybrid_citations` and `web_search_citations` arrays, and those cited subsets already reach the browser on all four delivery paths: history load, the streaming terminal event, the legacy non-streaming bridge, and collaboration message serialization. + +The inline gallery renderers were never switched over. `appendMessage` passed the full retrieved arrays into `renderInlineImageGalleries` and `renderInlineVideoGalleries`, and those renderers select workspace media purely by file extension through `extractWorkspaceCitationImageItems` and `extractWorkspaceCitationVideoItems`. Any retrieved `.png` or `.mp4` therefore became a gallery tile regardless of whether it was cited. No frontend module read the cited subsets at all. + +## Technical Details + +Files modified: + +- `application/single_app/static/js/chat/chat-citation-tracking.js` (new) +- `application/single_app/static/js/chat/chat-messages.js` +- `application/single_app/static/js/chat/chat-inline-images.js` +- `application/single_app/static/js/chat/chat-inline-videos.js` +- `application/single_app/config.py` +- `functional_tests/test_inline_media_cited_only_gating.py` (new) +- `functional_tests/test_inline_image_gallery_visualization.py` +- `functional_tests/test_inline_video_gallery_visualization.py` +- `functional_tests/test_chat_cited_source_tracking.py` +- `ui_tests/test_chat_inline_image_gallery_rendering.py` +- `ui_tests/test_chat_inline_video_gallery_rendering.py` + +Code changes summary: + +- Added `chat-citation-tracking.js`, the browser mirror of `functions_citation_tracking._message_has_citation_tracking()`. It exports `messageHasCitationTracking`, `getCitedHybridCitations`, and `getCitedWebCitations`. A message counts as tracked when `citation_tracking_version` is at least `1` or either `cited_*` key is present, and each getter normalizes non-array values to an empty list. +- `appendMessage` now derives `citedHybridCitations` and `citedWebCitations` from the assistant message object and passes those to both gallery renderers. The Sources disclosure, its count badges, and the metadata drawer continue to receive the complete retrieved arrays. +- Renamed the gallery entry-point and extraction helper parameters to `citedHybridCitations` and `citedWebCitations` so the narrowed contract is explicit at the call boundary. +- Corrected the "Linked images" and "Linked videos" card summaries, which described the links as merely returned with the response rather than cited by it. + +No backend change was required. `functions_citation_tracking.build_cited_source_subsets()` already produces the subsets, and conversation and per-message exports already select them through `get_message_reference_citation_buckets()`. + +### Scope boundaries + +- **Agent and tool galleries still render.** An action that returns an image or video gallery is an executed tool result, not an unused retrieval candidate. This matches how #1249 treats agent records. +- **Legacy messages keep prior behavior.** Messages saved before citation tracking existed carry no cited arrays and fall back to the full retrieved set, matching the no-migration legacy fallback in `get_message_reference_citation_buckets()` and #1249's decision to avoid read-time history parsing. +- **A tracked response that cited nothing renders no workspace or linked gallery.** `renderInlineImageGalleries` and `renderInlineVideoGalleries` already collapse `.inline-visualizations-container` with `d-none` when it has no children, so an empty result leaves no visual gap. + +## Validation + +Test coverage added or updated: + +- `functional_tests/test_inline_media_cited_only_gating.py` executes the real `chat-citation-tracking.js` helper under Node across tracked, untracked, empty-cited, key-presence-only, missing-message, and malformed-value inputs, then asserts the wiring in `chat-messages.js`, that the Sources panel still receives full arrays, and that both renderers declare cited inputs. +- `functional_tests/test_chat_cited_source_tracking.py` gained `test_inline_media_galleries_render_cited_subsets_only`, keeping the inline galleries inside the citation-tracking contract suite (17/17 passing). +- `functional_tests/test_inline_image_gallery_visualization.py` and `functional_tests/test_inline_video_gallery_visualization.py` had their wiring assertions updated from the retrieved-set call shape to the cited-subset call shape. +- `ui_tests/test_chat_inline_image_gallery_rendering.py` and `ui_tests/test_chat_inline_video_gallery_rendering.py` gained a gating regression test covering three messages in one page: a tracked response that cited one of two retrieved media files, a tracked response that cited nothing but ran a media action, and a legacy untracked response. + +Before and after: + +| Scenario | Before | After | +| --- | --- | --- | +| Tracked response, 1 of 5 retrieved images cited | 5 inline tiles | 1 inline tile | +| Tracked response, no documents cited | Every retrieved image tiled | No workspace gallery | +| Tracked response, no documents cited, media action ran | Retrieved images plus action gallery | Action gallery only | +| Legacy untracked response | Every retrieved image tiled | Unchanged | +| Sources disclosure | Complete retrieved set | Unchanged | diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index dac690603..c8335252f 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -38,3 +38,4 @@ category: Version History - [Collaboration Mention Tab Autocomplete Fix](COLLABORATION_MENTION_TAB_AUTOCOMPLETE_FIX.md) - [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md) - [Admin Settings Pane Variable Scope Fix](ADMIN_SETTINGS_PANE_VARIABLE_SCOPE_FIX.md) +- [Inline Media Cited-Only Gating Fix](INLINE_MEDIA_CITED_ONLY_GATING_FIX.md) diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index ecdf05aeb..fe7ebbf19 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,8 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.260.024 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.260.023 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.021 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.020 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.019 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -30,8 +32,8 @@ This page includes the latest release notes inline. Older release sections are s | v0.260.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.013 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.012 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.260.011 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.260.010 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.260.011 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | +| v0.260.010 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.009 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.008 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.007 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | @@ -65,6 +67,28 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.260.024)** + +#### Bug Fixes + +* **Inline Images And Videos Now Show Only Cited Media** + * Assistant messages rendered an inline image or video gallery for every media file that retrieval returned, so a search that surfaced five workspace images produced five inline tiles even when the answer referenced only one of them, or none at all. Media that had nothing to do with the answer was presented inside the message bubble as though it supported the answer. + * Inline galleries now render only the media the response actually cited. The five-item gallery cap therefore goes to genuinely cited media instead of retrieval noise, and unreferenced workspace files no longer trigger enhanced-citation fetches. + * Galleries produced by an action or tool the assistant actually ran are unaffected, since those are executed results rather than unused search candidates. Conversations created before cited-source tracking existed also keep their previous behavior. + * The **Sources** disclosure is unchanged and still lists every retrieved document and web result, so nothing becomes harder to find. + * (Ref: `chat-citation-tracking.js`, `chat-inline-images.js`, `chat-inline-videos.js`, `chat-messages.js`, `cited_hybrid_citations`, [#1329](https://github.com/microsoft/simplechat/issues/1329)) + +### **(v0.260.023)** + +#### Bug Fixes + +* **Running Simple Chat Directly No Longer Fails To Start When An Agent Has Actions** + * Starting Simple Chat with `python app.py` (including via `uv run`) aborted with `RuntimeError: Working outside of request context` whenever any agent had an action assigned. The app started normally until the first action was saved, which made the failure look intermittent. + * Semantic Kernel initialization runs before any request exists on that path, but agent plugin loading read the signed-in user from the Flask session. It now resolves the user only when a request is actually in progress and otherwise loads with no user identity, matching how global plugin loading already behaved. + * Container and App Service deployments were never affected, because they start through gunicorn and initialize during the first request. Their behavior is unchanged. + * Three further identity lookups used for group scope and personal model endpoints had the same latent problem and were corrected at the same time. + * (Ref: `semantic_kernel_loader.py`, `functions_authentication.py`, `get_current_user_id_or_none`, issue #1327) + ### **(v0.260.021)** #### Bug Fixes @@ -307,61 +331,3 @@ This page includes the latest release notes inline. Older release sections are s * Cards were relocated between tabs without renaming a single field, so every saved value is preserved and the form submits exactly the payload it did before. * Sidebar search still finds a setting by group, tab or card name, so you can reach anything without knowing where it now lives. * (Ref: admin settings field contract, `admin_settings_nav.py`) - -### **(v0.260.011)** - -#### User Interface Enhancements - -* **Governance And Scale Split Into Focused Tabs** - * Governance held five cards covering three different jobs. It is now **Feature Governance** (which features are governed), **Policies** (the policies themselves), and **MCP Governance**. - * Scale mixed cache configuration with Cosmos capacity, and is now **Redis & Caching** and **Cosmos**. - * **Azure Front Door** moved out of Scale into Security, under a new **Network** tab. It configures authentication and redirect flows rather than throughput, so it never belonged with capacity settings. - * Existing links and bookmarks to `#governance` and `#scale` still work and land on the first tab of each group. - * No settings changed. Every option keeps its name and its saved value. - * (Ref: navigation map, `feature-governance`, `governance-policies`, `mcp-governance`, `redis-caching`, `cosmos`, `network`) - -#### Bug Fixes - -* **Governance Status Messages No Longer Get Stuck On One Tab** - * The inline governance status message lived inside the Governance pane, so a message raised while working in one area could end up rendered on a tab you were not looking at. - * It now sits outside the tabs and is visible wherever you are in Governance. - * (Ref: `governance-status`, `admin_governance.js`) - -#### Bug Fixes - -* **Reliable File Generation From Agent Action Results** - * Asking an agent for a downloadable file built from action results now produces the complete dataset in the requested format. Previously these requests could fail outright, publish a three-row sample of a large result, overwrite the assistant's written answer, or return nothing at all. Delivered across v0.260.004 through v0.260.011. - * **Files no longer fail to generate.** A CSV built from several actions in one turn could stop with `Generated output schema mismatch at row 2`, because each action returned a different set of columns. The export now pins a union of every column before the run starts and pads the missing cells, so mixed-shape results serialize instead of failing. - * **The written answer is no longer replaced by the file card.** CSV replies were suppressed alongside JSON and XML, but only JSON and XML withhold their payload from the response. CSV, DOCX, and PDF now keep the assistant's answer and append the file card beneath it. - * **Files contain the retrieved data, not a sample of it.** When the assistant pasted a few example rows above its answer, that excerpt outranked the real result set, producing a 3-row file from a 900-row query. Pasted rows are now used only when they are not an excerpt of the data actually retrieved. - * **Discovery calls no longer dilute the dataset.** A turn that lists instances, lists parameters, then retrieves history used to blend all three into one file. Rows are grouped by the action that produced them, and the action holding the substantive dataset wins. - * **Follow-up requests reuse data already gathered.** Asking "now make that a CSV" after the data was retrieved in an earlier turn no longer returns an empty result. The export reaches back through stored conversation citations, bounded by the **conversation history limit** in Admin Settings, and reuses the rows already collected instead of re-querying the source. - * **Answering a clarifying question now delivers the file.** When the assistant asks which rows and columns to include, replying "yes, all columns" now publishes the file that was originally requested. The clarification turn itself no longer publishes a placeholder file built from the question text. - * **The assistant no longer claims it cannot create files.** Every format now states the publication contract to the model, including on the turn that only answers a clarification, so replies stop saying "I cannot create or attach a file in this interface" and then producing one anyway. - * **Overlapping result pages no longer double the row count.** Agents frequently re-request a range from the same start time rather than paging forward, which produced a 1,000-row file for a window holding roughly 500 distinct records. Rows an earlier page of the same action already returned are dropped, while genuinely repeated records inside a single response are preserved. - * **Partial data is now labeled.** When an action reports that it truncated its own results, the file carries a **Partial** badge and a note explaining that it covers only the rows the action returned. Agents are also instructed to request the remainder starting after the last row they already hold, rather than repeating the original range. - * **CSV, DOCX, PDF, JSON, and XML now behave identically.** All five formats resolve rows the same way, reach back to earlier turns, decline to publish on a clarification turn, and report truncation. - * (Ref: `functions_generated_file_exports.py`, `functions_tabular_generated_exports.py`, `route_backend_chats.py`, `chat-messages.js`, [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](https://github.com/microsoft/simplechat/blob/main/docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md), Refs #1071) - -### **(v0.260.010)** - -#### New Features - -* **Admin Settings Navigation Is Now Grouped** - * Admin Settings presented 18 tabs in one flat list. Related tabs are now collected under 12 groups such as Appearance, Knowledge, Security and Operations, so the list is scannable and has room to grow. - * In the sidebar, groups are collapsible and remember whether you left them open. In the tab layout, a row of group pills filters the tab strip to one group at a time. - * Opening a tab always reveals its group first, so a deep link or a cross-reference can never land you on a pane whose tab is hidden. - * Sidebar search now matches group names as well as tab and setting names, and expands whatever it needs to show a result. - * No settings moved in this release. Every tab keeps its contents; only the navigation around them changed. - * (Ref: `admin_settings_nav.py`, `_sidebar_nav.html`, `admin_settings.html`, `admin_sidebar_nav.js`) - -#### Bug Fixes - -* **Shared Conversation File Approvals Is Reachable From The Sidebar** - * The Shared Conversation File Approvals card had no navigation entry, so it could only be found by scrolling the AI Models tab. It is now listed like every other setting. - * (Ref: `shared-conversation-file-approvals-section`, navigation map) - -* **Navigation Labels And Order Can No Longer Drift** - * The tab strip and the sidebar each maintained the same structure by hand and had diverged: tab order differed between them, and Agents, Custom Pages and Search and Extract each showed a different name depending on which navigation you used. - * Both now render from one definition, so a change is made once and appears in both. - * (Ref: `admin_settings_nav.py`, `test_admin_settings_nav_map.py`) diff --git a/docs/explanation/release-notes/v0.260.md b/docs/explanation/release-notes/v0.260.md index 36bd30157..d9ac58f69 100644 --- a/docs/explanation/release-notes/v0.260.md +++ b/docs/explanation/release-notes/v0.260.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.260 series" -description: "SimpleChat release notes for 0.260.009 – 0.260.001." +description: "SimpleChat release notes for 0.260.011 – 0.260.001." section: "Reference" layout: page --- @@ -11,6 +11,64 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.260.011)** + +#### User Interface Enhancements + +* **Governance And Scale Split Into Focused Tabs** + * Governance held five cards covering three different jobs. It is now **Feature Governance** (which features are governed), **Policies** (the policies themselves), and **MCP Governance**. + * Scale mixed cache configuration with Cosmos capacity, and is now **Redis & Caching** and **Cosmos**. + * **Azure Front Door** moved out of Scale into Security, under a new **Network** tab. It configures authentication and redirect flows rather than throughput, so it never belonged with capacity settings. + * Existing links and bookmarks to `#governance` and `#scale` still work and land on the first tab of each group. + * No settings changed. Every option keeps its name and its saved value. + * (Ref: navigation map, `feature-governance`, `governance-policies`, `mcp-governance`, `redis-caching`, `cosmos`, `network`) + +#### Bug Fixes + +* **Governance Status Messages No Longer Get Stuck On One Tab** + * The inline governance status message lived inside the Governance pane, so a message raised while working in one area could end up rendered on a tab you were not looking at. + * It now sits outside the tabs and is visible wherever you are in Governance. + * (Ref: `governance-status`, `admin_governance.js`) + +#### Bug Fixes + +* **Reliable File Generation From Agent Action Results** + * Asking an agent for a downloadable file built from action results now produces the complete dataset in the requested format. Previously these requests could fail outright, publish a three-row sample of a large result, overwrite the assistant's written answer, or return nothing at all. Delivered across v0.260.004 through v0.260.011. + * **Files no longer fail to generate.** A CSV built from several actions in one turn could stop with `Generated output schema mismatch at row 2`, because each action returned a different set of columns. The export now pins a union of every column before the run starts and pads the missing cells, so mixed-shape results serialize instead of failing. + * **The written answer is no longer replaced by the file card.** CSV replies were suppressed alongside JSON and XML, but only JSON and XML withhold their payload from the response. CSV, DOCX, and PDF now keep the assistant's answer and append the file card beneath it. + * **Files contain the retrieved data, not a sample of it.** When the assistant pasted a few example rows above its answer, that excerpt outranked the real result set, producing a 3-row file from a 900-row query. Pasted rows are now used only when they are not an excerpt of the data actually retrieved. + * **Discovery calls no longer dilute the dataset.** A turn that lists instances, lists parameters, then retrieves history used to blend all three into one file. Rows are grouped by the action that produced them, and the action holding the substantive dataset wins. + * **Follow-up requests reuse data already gathered.** Asking "now make that a CSV" after the data was retrieved in an earlier turn no longer returns an empty result. The export reaches back through stored conversation citations, bounded by the **conversation history limit** in Admin Settings, and reuses the rows already collected instead of re-querying the source. + * **Answering a clarifying question now delivers the file.** When the assistant asks which rows and columns to include, replying "yes, all columns" now publishes the file that was originally requested. The clarification turn itself no longer publishes a placeholder file built from the question text. + * **The assistant no longer claims it cannot create files.** Every format now states the publication contract to the model, including on the turn that only answers a clarification, so replies stop saying "I cannot create or attach a file in this interface" and then producing one anyway. + * **Overlapping result pages no longer double the row count.** Agents frequently re-request a range from the same start time rather than paging forward, which produced a 1,000-row file for a window holding roughly 500 distinct records. Rows an earlier page of the same action already returned are dropped, while genuinely repeated records inside a single response are preserved. + * **Partial data is now labeled.** When an action reports that it truncated its own results, the file carries a **Partial** badge and a note explaining that it covers only the rows the action returned. Agents are also instructed to request the remainder starting after the last row they already hold, rather than repeating the original range. + * **CSV, DOCX, PDF, JSON, and XML now behave identically.** All five formats resolve rows the same way, reach back to earlier turns, decline to publish on a clarification turn, and report truncation. + * (Ref: `functions_generated_file_exports.py`, `functions_tabular_generated_exports.py`, `route_backend_chats.py`, `chat-messages.js`, [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](https://github.com/microsoft/simplechat/blob/main/docs/explanation/fixes/GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md), Refs #1071) + +### **(v0.260.010)** + +#### New Features + +* **Admin Settings Navigation Is Now Grouped** + * Admin Settings presented 18 tabs in one flat list. Related tabs are now collected under 12 groups such as Appearance, Knowledge, Security and Operations, so the list is scannable and has room to grow. + * In the sidebar, groups are collapsible and remember whether you left them open. In the tab layout, a row of group pills filters the tab strip to one group at a time. + * Opening a tab always reveals its group first, so a deep link or a cross-reference can never land you on a pane whose tab is hidden. + * Sidebar search now matches group names as well as tab and setting names, and expands whatever it needs to show a result. + * No settings moved in this release. Every tab keeps its contents; only the navigation around them changed. + * (Ref: `admin_settings_nav.py`, `_sidebar_nav.html`, `admin_settings.html`, `admin_sidebar_nav.js`) + +#### Bug Fixes + +* **Shared Conversation File Approvals Is Reachable From The Sidebar** + * The Shared Conversation File Approvals card had no navigation entry, so it could only be found by scrolling the AI Models tab. It is now listed like every other setting. + * (Ref: `shared-conversation-file-approvals-section`, navigation map) + +* **Navigation Labels And Order Can No Longer Drift** + * The tab strip and the sidebar each maintained the same structure by hand and had diverged: tab order differed between them, and Agents, Custom Pages and Search and Extract each showed a different name depending on which navigation you used. + * Both now render from one definition, so a change is made once and appears in both. + * (Ref: `admin_settings_nav.py`, `test_admin_settings_nav_map.py`) + ### **(v0.260.009)** #### New Features diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index e3cc04304..5e349e020 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.260.024)** + +#### Bug Fixes + +* **Inline Images And Videos Now Show Only Cited Media** + * Assistant messages rendered an inline image or video gallery for every media file that retrieval returned, so a search that surfaced five workspace images produced five inline tiles even when the answer referenced only one of them, or none at all. Media that had nothing to do with the answer was presented inside the message bubble as though it supported the answer. + * Inline galleries now render only the media the response actually cited. The five-item gallery cap therefore goes to genuinely cited media instead of retrieval noise, and unreferenced workspace files no longer trigger enhanced-citation fetches. + * Galleries produced by an action or tool the assistant actually ran are unaffected, since those are executed results rather than unused search candidates. Conversations created before cited-source tracking existed also keep their previous behavior. + * The **Sources** disclosure is unchanged and still lists every retrieved document and web result, so nothing becomes harder to find. + * (Ref: `chat-citation-tracking.js`, `chat-inline-images.js`, `chat-inline-videos.js`, `chat-messages.js`, `cited_hybrid_citations`, [#1329](https://github.com/microsoft/simplechat/issues/1329)) + ### **(v0.260.023)** #### Bug Fixes diff --git a/functional_tests/test_chat_cited_source_tracking.py b/functional_tests/test_chat_cited_source_tracking.py index 3b8ba79c7..5e5377bbe 100644 --- a/functional_tests/test_chat_cited_source_tracking.py +++ b/functional_tests/test_chat_cited_source_tracking.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test for cited-source tracking. -Version: 0.250.215 +Version: 0.260.024 Implemented in: 0.250.215 This test ensures returned sources remain complete while exact document and @@ -37,6 +37,11 @@ CHAT_MESSAGES_JS = APP_ROOT / "static" / "js" / "chat" / "chat-messages.js" CHAT_RETRY_JS = APP_ROOT / "static" / "js" / "chat" / "chat-retry.js" CHAT_EDIT_JS = APP_ROOT / "static" / "js" / "chat" / "chat-edit.js" +CHAT_CITATION_TRACKING_JS = ( + APP_ROOT / "static" / "js" / "chat" / "chat-citation-tracking.js" +) +INLINE_IMAGES_JS = APP_ROOT / "static" / "js" / "chat" / "chat-inline-images.js" +INLINE_VIDEOS_JS = APP_ROOT / "static" / "js" / "chat" / "chat-inline-videos.js" SIMPLECHAT_OPERATIONS = APP_ROOT / "functions_simplechat_operations.py" WORKFLOW_RUNNER = APP_ROOT / "functions_workflow_runner.py" if str(APP_ROOT) not in sys.path: @@ -630,6 +635,46 @@ def test_ui_and_exports_select_cited_subsets_without_losing_sources(): assert "Web References" in export_source +def test_inline_media_galleries_render_cited_subsets_only(): + """Verify inline image and video galleries consume cited subsets, not sources.""" + tracking_source = CHAT_CITATION_TRACKING_JS.read_text(encoding="utf-8") + messages_source = CHAT_MESSAGES_JS.read_text(encoding="utf-8") + images_source = INLINE_IMAGES_JS.read_text(encoding="utf-8") + videos_source = INLINE_VIDEOS_JS.read_text(encoding="utf-8") + + assert "export function messageHasCitationTracking(message)" in tracking_source + assert "export function getCitedHybridCitations(message, sourceCitations = [])" in tracking_source + assert "export function getCitedWebCitations(message, sourceCitations = [])" in tracking_source + assert '"cited_hybrid_citations" in message || "cited_web_search_citations" in message' in tracking_source + + assert ( + "import { getCitedHybridCitations, getCitedWebCitations } from './chat-citation-tracking.js';" + in messages_source + ) + assert ( + "const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations);" + in messages_source + ) + assert ( + "const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations);" + in messages_source + ) + + # Sources keeps the complete retrieved inventory; only the galleries narrow. + citations_call = messages_source.split( + "const citationsButtonsHtml = createCitationsHtml(" + )[1].split(");")[0] + assert "hybridCitations," in citations_call + assert "webCitations," in citations_call + assert "citedHybridCitations" not in citations_call + assert "citedWebCitations" not in citations_call + + for gallery_source in (images_source, videos_source): + assert "citedHybridCitations = []," in gallery_source + assert "citedWebCitations = []," in gallery_source + assert "agentCitations = []," in gallery_source + + def test_version_is_available(): """Verify the application includes the cited-source tracking version.""" assert_app_version_at_least("0.250.215") @@ -652,6 +697,7 @@ def test_version_is_available(): test_lifecycle_mutations_and_forks_rebuild_exact_usage, test_collaboration_and_workflow_propagate_tracking_contract, test_ui_and_exports_select_cited_subsets_without_losing_sources, + test_inline_media_galleries_render_cited_subsets_only, test_version_is_available, ] for test in tests: diff --git a/functional_tests/test_inline_image_gallery_visualization.py b/functional_tests/test_inline_image_gallery_visualization.py index 02ffe93ec..aab1ecefa 100644 --- a/functional_tests/test_inline_image_gallery_visualization.py +++ b/functional_tests/test_inline_image_gallery_visualization.py @@ -2,7 +2,7 @@ # test_inline_image_gallery_visualization.py """ Functional test for inline image gallery visualization support. -Version: 0.241.066 +Version: 0.260.024 Implemented in: 0.241.057 This test ensures assistant agent citations can expose inline image galleries, @@ -105,8 +105,18 @@ def test_chat_renderer_wires_inline_image_galleries(): assert ".inline-image-modal-meta-row" in chats_css assert "max-height: 400px;" in chats_css assert "object-fit: contain;" in chats_css - assert "hybridCitations || []" in messages_js - assert "webCitations || []" in messages_js + assert "hybridCitations," in messages_js + assert "webCitations," in messages_js + assert ( + "const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations);" + in messages_js + ) + assert ( + "const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations);" + in messages_js + ) + assert "function extractWorkspaceCitationImageItems(citedHybridCitations = []" in images_js + assert "function extractLinkedImageItems(citedWebCitations = []" in images_js def test_workflow_created_conversations_keep_summary_citations(): diff --git a/functional_tests/test_inline_media_cited_only_gating.py b/functional_tests/test_inline_media_cited_only_gating.py new file mode 100644 index 000000000..5f838f537 --- /dev/null +++ b/functional_tests/test_inline_media_cited_only_gating.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +# test_inline_media_cited_only_gating.py +""" +Functional test for inline image and video galleries showing cited media only. +Version: 0.260.024 +Implemented in: 0.260.024 + +Inline galleries used to render every workspace and web media result returned by +retrieval, so a search that surfaced five images produced five inline tiles even +when the response referenced none of them. The renderers now consume the cited +citation subsets that issue #1249 already persists on each assistant message, +while the Sources disclosure keeps the complete retrieved set. + +This test ensures the browser helper resolves cited subsets with the same rules +as functions_citation_tracking.py, that chat-messages.js feeds those subsets to +both gallery renderers, and that the Sources panel still receives every +retrieved citation. + +Refs microsoft/simplechat#1329 +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from test_support.versioning import assert_app_version_at_least + + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CHAT_JS_DIR = os.path.join( + REPO_ROOT, "application", "single_app", "static", "js", "chat" +) +CITATION_TRACKING_JS = os.path.join(CHAT_JS_DIR, "chat-citation-tracking.js") +CHAT_MESSAGES_JS = os.path.join(CHAT_JS_DIR, "chat-messages.js") +INLINE_IMAGES_JS = os.path.join(CHAT_JS_DIR, "chat-inline-images.js") +INLINE_VIDEOS_JS = os.path.join(CHAT_JS_DIR, "chat-inline-videos.js") + +HYBRID_SOURCES = [ + {"citation_id": "doc-a_1", "file_name": "cited-photo.png"}, + {"citation_id": "doc-b_1", "file_name": "unreferenced-photo.png"}, +] +WEB_SOURCES = [ + {"url": "https://example.com/cited.png", "title": "Cited"}, + {"url": "https://example.com/unreferenced.png", "title": "Unreferenced"}, +] +CITED_HYBRID = [HYBRID_SOURCES[0]] +CITED_WEB = [WEB_SOURCES[0]] + +NODE_DRIVER_SOURCE = """ +import { + messageHasCitationTracking, + getCitedHybridCitations, + getCitedWebCitations, +} from "./chat-citation-tracking.mjs"; + +const scenarios = JSON.parse(process.argv[2]); +const results = scenarios.map((scenario) => ({ + name: scenario.name, + tracked: messageHasCitationTracking(scenario.message), + hybrid: getCitedHybridCitations(scenario.message, scenario.hybridSources), + web: getCitedWebCitations(scenario.message, scenario.webSources), +})); + +process.stdout.write(JSON.stringify(results)); +""" + + +def read_text(absolute_path): + with open(absolute_path, "r", encoding="utf-8") as handle: + return handle.read() + + +def run_citation_tracking_helper(scenarios): + """Execute the real browser helper under Node and return its results.""" + node_executable = shutil.which("node") + if not node_executable: + return None + + with tempfile.TemporaryDirectory() as work_dir: + # The helper is an ES module in a directory without a package.json, so + # Node needs the .mjs extension to load the source unmodified. + module_copy = os.path.join(work_dir, "chat-citation-tracking.mjs") + with open(module_copy, "w", encoding="utf-8") as handle: + handle.write(read_text(CITATION_TRACKING_JS)) + + driver_path = os.path.join(work_dir, "driver.mjs") + with open(driver_path, "w", encoding="utf-8") as handle: + handle.write(NODE_DRIVER_SOURCE) + + completed = subprocess.run( + [node_executable, driver_path, json.dumps(scenarios)], + capture_output=True, + text=True, + timeout=60, + ) + + if completed.returncode != 0: + raise AssertionError( + f"Node helper execution failed ({completed.returncode}): {completed.stderr.strip()}" + ) + + return {result["name"]: result for result in json.loads(completed.stdout)} + + +def test_citation_tracking_helper_resolves_cited_subsets(): + """Verify the browser helper matches the server tracking-detection rules.""" + print("Testing chat-citation-tracking.js resolution rules...") + + scenarios = [ + { + "name": "legacy_untracked", + "message": {"id": "m1", "role": "assistant"}, + "hybridSources": HYBRID_SOURCES, + "webSources": WEB_SOURCES, + }, + { + "name": "tracked_with_citations", + "message": { + "id": "m2", + "citation_tracking_version": 1, + "cited_hybrid_citations": CITED_HYBRID, + "cited_web_search_citations": CITED_WEB, + }, + "hybridSources": HYBRID_SOURCES, + "webSources": WEB_SOURCES, + }, + { + "name": "tracked_without_citations", + "message": { + "id": "m3", + "citation_tracking_version": 1, + "cited_hybrid_citations": [], + "cited_web_search_citations": [], + }, + "hybridSources": HYBRID_SOURCES, + "webSources": WEB_SOURCES, + }, + { + "name": "tracked_by_key_presence_only", + "message": {"id": "m4", "cited_hybrid_citations": CITED_HYBRID}, + "hybridSources": HYBRID_SOURCES, + "webSources": WEB_SOURCES, + }, + { + "name": "missing_message", + "message": None, + "hybridSources": HYBRID_SOURCES, + "webSources": WEB_SOURCES, + }, + { + "name": "malformed_values", + "message": { + "id": "m6", + "citation_tracking_version": "not-a-version", + "cited_hybrid_citations": "not-an-array", + }, + "hybridSources": "not-an-array", + "webSources": None, + }, + ] + + results = run_citation_tracking_helper(scenarios) + if results is None: + print("Node is unavailable; skipping helper execution checks.") + return True + + legacy = results["legacy_untracked"] + assert legacy["tracked"] is False + assert legacy["hybrid"] == HYBRID_SOURCES, "Legacy messages keep the full source set." + assert legacy["web"] == WEB_SOURCES, "Legacy messages keep the full web source set." + + tracked = results["tracked_with_citations"] + assert tracked["tracked"] is True + assert tracked["hybrid"] == CITED_HYBRID, "Tracked messages expose only cited documents." + assert tracked["web"] == CITED_WEB, "Tracked messages expose only cited web results." + + empty = results["tracked_without_citations"] + assert empty["tracked"] is True + assert empty["hybrid"] == [], "A tracked message that cited nothing renders nothing." + assert empty["web"] == [], "A tracked message that cited nothing renders nothing." + + key_only = results["tracked_by_key_presence_only"] + assert key_only["tracked"] is True, "A cited_* key alone marks the message as tracked." + assert key_only["hybrid"] == CITED_HYBRID + assert key_only["web"] == [], "A tracked message without cited web results yields none." + + missing = results["missing_message"] + assert missing["tracked"] is False + assert missing["hybrid"] == HYBRID_SOURCES + assert missing["web"] == WEB_SOURCES + + malformed = results["malformed_values"] + assert malformed["tracked"] is True, "A cited_* key marks tracking even with a bad version." + assert malformed["hybrid"] == [], "Non-array cited values normalize to an empty list." + assert malformed["web"] == [] + + print("Citation tracking helper resolution rules passed.") + return True + + +def test_inline_galleries_consume_cited_subsets(): + """Verify chat-messages.js feeds cited subsets to both gallery renderers.""" + print("Testing inline gallery wiring in chat-messages.js...") + + messages_source = read_text(CHAT_MESSAGES_JS) + + assert ( + "import { getCitedHybridCitations, getCitedWebCitations } from './chat-citation-tracking.js';" + in messages_source + ) + assert ( + "const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations);" + in messages_source + ) + assert ( + "const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations);" + in messages_source + ) + + video_call = messages_source.split("await renderInlineVideoGalleries(")[1].split(");")[0] + image_call = messages_source.split("await renderInlineImageGalleries(")[1].split(");")[0] + for renderer_name, call_arguments in ( + ("renderInlineVideoGalleries", video_call), + ("renderInlineImageGalleries", image_call), + ): + assert "citedHybridCitations" in call_arguments, ( + f"{renderer_name} must receive the cited document subset." + ) + assert "citedWebCitations" in call_arguments, ( + f"{renderer_name} must receive the cited web subset." + ) + assert "hybridCitations || []" not in call_arguments, ( + f"{renderer_name} must not receive the full retrieved document set." + ) + assert "webCitations || []" not in call_arguments, ( + f"{renderer_name} must not receive the full retrieved web set." + ) + assert "agentCitations || []" in call_arguments, ( + f"{renderer_name} must still receive executed agent citations." + ) + + print("Inline gallery wiring passed.") + return True + + +def test_sources_disclosure_keeps_full_retrieved_sets(): + """Verify the Sources panel still lists every retrieved citation.""" + print("Testing Sources disclosure retains retrieved citations...") + + messages_source = read_text(CHAT_MESSAGES_JS) + + citations_call = messages_source.split("const citationsButtonsHtml = createCitationsHtml(")[1] + citations_call = citations_call.split(");")[0] + assert "hybridCitations," in citations_call, "Sources keeps the full document set." + assert "webCitations," in citations_call, "Sources keeps the full web set." + assert "citedHybridCitations" not in citations_call + assert "citedWebCitations" not in citations_call + + print("Sources disclosure checks passed.") + return True + + +def test_gallery_renderers_declare_cited_inputs(): + """Verify both renderers name their inputs as cited subsets.""" + print("Testing gallery renderer signatures...") + + images_source = read_text(INLINE_IMAGES_JS) + videos_source = read_text(INLINE_VIDEOS_JS) + + assert "export async function renderInlineImageGalleries(\n messageElement,\n citedHybridCitations = [],\n citedWebCitations = [],\n agentCitations = [],\n" in images_source + assert "export async function renderInlineVideoGalleries(\n messageElement,\n citedHybridCitations = [],\n citedWebCitations = [],\n agentCitations = [],\n" in videos_source + + assert "function extractWorkspaceCitationImageItems(citedHybridCitations = []" in images_source + assert "function extractLinkedImageItems(citedWebCitations = []" in images_source + assert "function extractWorkspaceCitationVideoItems(citedHybridCitations = []" in videos_source + assert "function extractLinkedVideoItems(citedWebCitations = []" in videos_source + + assert '"Image links cited in this response."' in images_source + assert '"Video links cited in this response."' in videos_source + assert "returned with this response" not in images_source + assert "returned with this response" not in videos_source + + print("Gallery renderer signature checks passed.") + return True + + +def test_version_is_available(): + """Verify the application includes the inline media gating version.""" + assert_app_version_at_least("0.260.024") + return True + + +if __name__ == "__main__": + tests = [ + test_citation_tracking_helper_resolves_cited_subsets, + test_inline_galleries_consume_cited_subsets, + test_sources_disclosure_keeps_full_retrieved_sets, + test_gallery_renderers_declare_cited_inputs, + test_version_is_available, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + results.append(bool(test())) + except Exception as error: + print(f"Test failed: {error}") + import traceback + + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_inline_video_gallery_visualization.py b/functional_tests/test_inline_video_gallery_visualization.py index e310bf148..cc3ba3f8e 100644 --- a/functional_tests/test_inline_video_gallery_visualization.py +++ b/functional_tests/test_inline_video_gallery_visualization.py @@ -2,7 +2,7 @@ # test_inline_video_gallery_visualization.py """ Functional test for inline video gallery visualization support. -Version: 0.241.066 +Version: 0.260.024 Implemented in: 0.241.066 This test ensures assistant agent citations can expose inline video galleries, @@ -107,6 +107,16 @@ def test_chat_renderer_wires_inline_video_galleries(): assert "max-height: 400px;" in chats_css assert "_contains_inline_video_result(function_result)" in workflow_runner_source assert "mime_type.startswith('video/')" in workflow_runner_source + assert ( + "const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations);" + in messages_js + ) + assert ( + "const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations);" + in messages_js + ) + assert "function extractWorkspaceCitationVideoItems(citedHybridCitations = []" in videos_js + assert "function extractLinkedVideoItems(citedWebCitations = []" in videos_js if __name__ == "__main__": diff --git a/ui_tests/test_chat_inline_image_gallery_rendering.py b/ui_tests/test_chat_inline_image_gallery_rendering.py index 2922c0201..c6b687f47 100644 --- a/ui_tests/test_chat_inline_image_gallery_rendering.py +++ b/ui_tests/test_chat_inline_image_gallery_rendering.py @@ -1,12 +1,16 @@ # test_chat_inline_image_gallery_rendering.py """ UI test for inline image gallery rendering in chat. -Version: 0.241.066 +Version: 0.260.024 Implemented in: 0.241.056 This test ensures assistant messages can hydrate inline image gallery agent citations, render up to five framed images inside the chat bubble, and expose an overlay info button that opens a detail modal for each image. + +It also ensures workspace image galleries render only the images the response +actually cited, so retrieved-but-unreferenced workspace images stay out of the +message bubble while remaining available under Sources. """ import base64 @@ -205,6 +209,167 @@ def test_chat_inline_image_gallery_rendering(playwright): page.locator('[data-message-id="assistant-msg-images-1"] .inline-image-gallery-item-image').nth(1).click() expect(page.locator('#image-modal')).to_be_visible() expect(page.locator('#image-modal img')).to_be_visible() + finally: + context.close() + browser.close() + + +def _build_inline_image_gallery_agent_citation(): + """Return an agent citation whose gallery resolves without an artifact fetch.""" + return { + "tool_name": "Image gallery: Incident Photos", + "function_name": "collect_images", + "plugin_name": "ExternalMediaPlugin", + "function_arguments": {"title": "Incident Photos"}, + "function_result": { + "success": True, + "render_type": "inline_image_gallery", + "image_gallery": { + "title": "Incident Photos", + "summary": "Images returned by the action.", + "source_action_name": "media_collector", + "items": [ + { + "title": "Loading Dock Camera", + "image_url": f"data:image/png;base64,{TINY_PNG_BASE64}", + }, + ], + }, + }, + } + + +@pytest.mark.ui +def test_chat_inline_image_gallery_renders_cited_media_only(playwright): + """Retrieved-but-uncited workspace images must not render as inline media.""" + _require_ui_env() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + + cited_citation = { + "file_name": "cited-photo.png", + "citation_id": "cited-image-001_1", + "page_number": 1, + } + uncited_citation = { + "file_name": "uncited-photo.png", + "citation_id": "uncited-image-001_1", + "page_number": 1, + } + + page.route( + "**/api/user/settings", + lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"selected_agent": None, "settings": {"enable_agents": False}}), + ), + ) + page.route( + "**/api/get_conversations", + lambda route: route.fulfill(status=200, content_type="application/json", body=json.dumps({"conversations": []})), + ) + page.route( + "**/api/enhanced_citations/image**", + lambda route: route.fulfill(status=200, content_type="image/png", body=TINY_PNG_BYTES), + ) + + try: + response = page.goto(f"{BASE_URL}/chats", wait_until="domcontentloaded") + assert response is not None, "Expected a navigation response when loading /chats." + + if response.status in SKIP_RESPONSE_CODES: + pytest.skip(f"Chat page unavailable in this environment (HTTP {response.status}).") + + if "login" in page.url.lower(): + pytest.skip("Inline image gallery UI test requires an authenticated chat session.") + + page.wait_for_selector("#chatbox") + + page.evaluate( + """ + async ({ citedCitation, uncitedCitation, agentCitation }) => { + currentConversationId = 'test-convo'; + window.currentConversationId = 'test-convo'; + const messagesModule = await import('/static/js/chat/chat-messages.js'); + const retrievedCitations = [citedCitation, uncitedCitation]; + + messagesModule.appendMessage( + 'AI', 'Cited image results', null, 'assistant-msg-images-cited', true, + retrievedCitations, [], [], null, null, + { + id: 'assistant-msg-images-cited', + role: 'assistant', + content: 'Cited image results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + citation_tracking_version: 1, + cited_hybrid_citations: [citedCitation], + cited_web_search_citations: [], + }, + true + ); + + messagesModule.appendMessage( + 'AI', 'No cited image results', null, 'assistant-msg-images-uncited', true, + retrievedCitations, [], [agentCitation], null, null, + { + id: 'assistant-msg-images-uncited', + role: 'assistant', + content: 'No cited image results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + agent_citations: [agentCitation], + citation_tracking_version: 1, + cited_hybrid_citations: [], + cited_web_search_citations: [], + }, + true + ); + + messagesModule.appendMessage( + 'AI', 'Legacy image results', null, 'assistant-msg-images-legacy', true, + retrievedCitations, [], [], null, null, + { + id: 'assistant-msg-images-legacy', + role: 'assistant', + content: 'Legacy image results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + }, + true + ); + } + """, + { + "citedCitation": cited_citation, + "uncitedCitation": uncited_citation, + "agentCitation": _build_inline_image_gallery_agent_citation(), + }, + ) + + cited_scope = page.locator('[data-message-id="assistant-msg-images-cited"]') + expect(cited_scope.locator('.inline-image-gallery-card')).to_have_count(1) + expect(cited_scope.locator('.inline-image-gallery-title')).to_have_text('Workspace images') + expect(cited_scope.locator('.inline-image-gallery-item')).to_have_count(1) + expect(cited_scope.locator('.inline-image-gallery-item-title')).to_have_text('cited-photo.png') + + # A tracked response that cited no documents keeps its executed action + # gallery but drops every retrieved workspace image. + uncited_scope = page.locator('[data-message-id="assistant-msg-images-uncited"]') + expect(uncited_scope.locator('.inline-image-gallery-card')).to_have_count(1) + expect(uncited_scope.locator('.inline-image-gallery-title')).to_have_text('Incident Photos') + expect(uncited_scope.locator('.inline-image-gallery-item')).to_have_count(1) + + # Messages saved before citation tracking existed keep every retrieved image. + legacy_scope = page.locator('[data-message-id="assistant-msg-images-legacy"]') + expect(legacy_scope.locator('.inline-image-gallery-card')).to_have_count(1) + expect(legacy_scope.locator('.inline-image-gallery-item')).to_have_count(2) finally: context.close() browser.close() \ No newline at end of file diff --git a/ui_tests/test_chat_inline_video_gallery_rendering.py b/ui_tests/test_chat_inline_video_gallery_rendering.py index b763e986f..d44438946 100644 --- a/ui_tests/test_chat_inline_video_gallery_rendering.py +++ b/ui_tests/test_chat_inline_video_gallery_rendering.py @@ -1,12 +1,16 @@ # test_chat_inline_video_gallery_rendering.py """ UI test for inline video gallery rendering in chat. -Version: 0.241.066 +Version: 0.260.024 Implemented in: 0.241.066 This test ensures assistant messages can hydrate inline video gallery agent citations, render compact inline videos inside the chat bubble, and expose an overlay info button that opens a detail modal for each video. + +It also ensures workspace video galleries render only the videos the response +actually cited, so retrieved-but-unreferenced workspace videos stay out of the +message bubble while remaining available under Sources. """ import json @@ -203,6 +207,171 @@ def test_chat_inline_video_gallery_rendering(playwright): expect(details_modal.locator('#inline-video-details-description')).to_contain_text('loading dock camera') expect(details_modal.locator('#inline-video-details-meta')).to_contain_text('External video (api.example.com)') expect(details_modal.locator('#inline-video-details-preview')).to_be_visible() + finally: + context.close() + browser.close() + + +def _build_inline_video_gallery_agent_citation(): + """Return an agent citation whose gallery resolves without an artifact fetch.""" + return { + "tool_name": "Video gallery: Incident Clips", + "function_name": "collect_videos", + "plugin_name": "ExternalMediaPlugin", + "function_arguments": {"title": "Incident Clips"}, + "function_result": { + "success": True, + "render_type": "inline_video_gallery", + "video_gallery": { + "title": "Incident Clips", + "summary": "Clips returned by the action.", + "source_action_name": "media_collector", + "items": [ + { + "title": "Loading Dock Camera", + "video_url": "https://api.example.com/videos/loading-dock.mp4", + }, + ], + }, + }, + } + + +@pytest.mark.ui +def test_chat_inline_video_gallery_renders_cited_media_only(playwright): + """Retrieved-but-uncited workspace videos must not render as inline media.""" + _require_ui_env() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + + cited_citation = { + "file_name": "cited-evidence.mp4", + "citation_id": "cited-video-001_1", + "page_number": 1, + } + uncited_citation = { + "file_name": "uncited-evidence.mp4", + "citation_id": "uncited-video-001_1", + "page_number": 1, + } + + page.route( + "**/api/user/settings", + lambda route: route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"selected_agent": None, "settings": {"enable_agents": False}}), + ), + ) + page.route( + "**/api/get_conversations", + lambda route: route.fulfill(status=200, content_type="application/json", body=json.dumps({"conversations": []})), + ) + page.route( + "**/api/enhanced_citations/video**", + lambda route: route.fulfill(status=200, content_type="video/mp4", body=DUMMY_VIDEO_BYTES), + ) + page.route( + "https://api.example.com/videos/loading-dock.mp4", + lambda route: route.fulfill(status=200, content_type="video/mp4", body=DUMMY_VIDEO_BYTES), + ) + + try: + response = page.goto(f"{BASE_URL}/chats", wait_until="domcontentloaded") + assert response is not None, "Expected a navigation response when loading /chats." + + if response.status in SKIP_RESPONSE_CODES: + pytest.skip(f"Chat page unavailable in this environment (HTTP {response.status}).") + + if "login" in page.url.lower(): + pytest.skip("Inline video gallery UI test requires an authenticated chat session.") + + page.wait_for_selector("#chatbox") + + page.evaluate( + """ + async ({ citedCitation, uncitedCitation, agentCitation }) => { + currentConversationId = 'test-convo'; + window.currentConversationId = 'test-convo'; + const messagesModule = await import('/static/js/chat/chat-messages.js'); + const retrievedCitations = [citedCitation, uncitedCitation]; + + messagesModule.appendMessage( + 'AI', 'Cited video results', null, 'assistant-msg-videos-cited', true, + retrievedCitations, [], [], null, null, + { + id: 'assistant-msg-videos-cited', + role: 'assistant', + content: 'Cited video results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + citation_tracking_version: 1, + cited_hybrid_citations: [citedCitation], + cited_web_search_citations: [], + }, + true + ); + + messagesModule.appendMessage( + 'AI', 'No cited video results', null, 'assistant-msg-videos-uncited', true, + retrievedCitations, [], [agentCitation], null, null, + { + id: 'assistant-msg-videos-uncited', + role: 'assistant', + content: 'No cited video results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + agent_citations: [agentCitation], + citation_tracking_version: 1, + cited_hybrid_citations: [], + cited_web_search_citations: [], + }, + true + ); + + messagesModule.appendMessage( + 'AI', 'Legacy video results', null, 'assistant-msg-videos-legacy', true, + retrievedCitations, [], [], null, null, + { + id: 'assistant-msg-videos-legacy', + role: 'assistant', + content: 'Legacy video results', + conversation_id: 'test-convo', + hybrid_citations: retrievedCitations, + }, + true + ); + } + """, + { + "citedCitation": cited_citation, + "uncitedCitation": uncited_citation, + "agentCitation": _build_inline_video_gallery_agent_citation(), + }, + ) + + cited_scope = page.locator('[data-message-id="assistant-msg-videos-cited"]') + expect(cited_scope.locator('.inline-video-gallery-card')).to_have_count(1) + expect(cited_scope.locator('.inline-video-gallery-title')).to_have_text('Workspace videos') + expect(cited_scope.locator('.inline-video-gallery-item')).to_have_count(1) + expect(cited_scope.locator('.inline-video-gallery-item-title')).to_have_text('cited-evidence.mp4') + + # A tracked response that cited no documents keeps its executed action + # gallery but drops every retrieved workspace video. + uncited_scope = page.locator('[data-message-id="assistant-msg-videos-uncited"]') + expect(uncited_scope.locator('.inline-video-gallery-card')).to_have_count(1) + expect(uncited_scope.locator('.inline-video-gallery-title')).to_have_text('Incident Clips') + expect(uncited_scope.locator('.inline-video-gallery-item')).to_have_count(1) + + # Messages saved before citation tracking existed keep every retrieved video. + legacy_scope = page.locator('[data-message-id="assistant-msg-videos-legacy"]') + expect(legacy_scope.locator('.inline-video-gallery-card')).to_have_count(1) + expect(legacy_scope.locator('.inline-video-gallery-item')).to_have_count(2) finally: context.close() browser.close() \ No newline at end of file