Gate inline image and video galleries to cited media only

application/single_app/config.py

@@ -96,7 +96,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.260.023"
+VERSION = "0.260.024"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/static/js/chat/chat-citation-tracking.js

@@ -0,0 +1,50 @@
+// chat-citation-tracking.js
+/**
+ * Browser mirror of functions_citation_tracking.py tracking detection.
+ *
+ * Assistant messages carry both the complete retrieved source arrays
+ * (hybrid_citations, web_search_citations) and the smaller subsets that the
+ * final response explicitly cited (cited_hybrid_citations,
+ * cited_web_search_citations). Surfaces that present media or references as
+ * supporting the answer must read the cited subsets, while the Sources
+ * disclosure keeps showing everything that was retrieved.
+ *
+ * Messages saved before citation tracking existed carry no cited arrays. Those
+ * fall back to the full source arrays rather than being parsed at read time,
+ * matching get_message_reference_citation_buckets() on the server.
+ */
+
+const MIN_CITATION_TRACKING_VERSION = 1;
+
+function toCitationArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+
+export function messageHasCitationTracking(message) {
+    if (!message || typeof message !== "object") {
+        return false;
+    }
+
+    const trackingVersion = Number(message.citation_tracking_version);
+    if (Number.isInteger(trackingVersion) && trackingVersion >= MIN_CITATION_TRACKING_VERSION) {
+        return true;
+    }
+
+    return "cited_hybrid_citations" in message || "cited_web_search_citations" in message;
+}
+
+export function getCitedHybridCitations(message, sourceCitations = []) {
+    if (!messageHasCitationTracking(message)) {
+        return toCitationArray(sourceCitations);
+    }
+
+    return toCitationArray(message.cited_hybrid_citations);
+}
+
+export function getCitedWebCitations(message, sourceCitations = []) {
+    if (!messageHasCitationTracking(message)) {
+        return toCitationArray(sourceCitations);
+    }
+
+    return toCitationArray(message.cited_web_search_citations);
+}

application/single_app/static/js/chat/chat-inline-images.js

@@ -246,13 +246,13 @@ function normalizeWorkspaceCitationImageItem(rawCitation, index) {
     };
 }
 
-function extractWorkspaceCitationImageItems(hybridCitations = [], seenKeys = new Set()) {
+function extractWorkspaceCitationImageItems(citedHybridCitations = [], seenKeys = new Set()) {
     const items = [];
-    if (!Array.isArray(hybridCitations) || hybridCitations.length === 0) {
+    if (!Array.isArray(citedHybridCitations) || citedHybridCitations.length === 0) {
         return items;
     }
 
-    hybridCitations.forEach((citation, index) => {
+    citedHybridCitations.forEach((citation, index) => {
         pushUniqueImageItem(items, seenKeys, normalizeWorkspaceCitationImageItem(citation, index));
     });
 
@@ -287,13 +287,13 @@ function normalizeWebCitationImageItem(rawCitation, index) {
     };
 }
 
-function extractLinkedImageItems(webCitations = [], seenKeys = new Set()) {
+function extractLinkedImageItems(citedWebCitations = [], seenKeys = new Set()) {
     const items = [];
-    if (!Array.isArray(webCitations) || webCitations.length === 0) {
+    if (!Array.isArray(citedWebCitations) || citedWebCitations.length === 0) {
         return items;
     }
 
-    webCitations.forEach((citation, index) => {
+    citedWebCitations.forEach((citation, index) => {
         pushUniqueImageItem(items, seenKeys, normalizeWebCitationImageItem(citation, index));
     });
 
@@ -649,10 +649,17 @@ function createImageGalleryCard(result, messageId, index) {
     return { card };
 }
 
+/**
+ * Render inline image galleries for one assistant message.
+ *
+ * The workspace and linked galleries take the cited citation subsets, not the
+ * full retrieved sets, so images only appear when the response referenced them.
+ * Agent citations are executed tool results and are always rendered.
+ */
 export async function renderInlineImageGalleries(
     messageElement,
-    hybridCitations = [],
-    webCitations = [],
+    citedHybridCitations = [],
+    citedWebCitations = [],
     agentCitations = [],
     messageId = "",
     conversationId = ""
@@ -668,8 +675,8 @@ export async function renderInlineImageGalleries(
 
     container.querySelectorAll(".inline-image-gallery-card").forEach((card) => card.remove());
 
-    const hasHybridCitations = Array.isArray(hybridCitations) && hybridCitations.length > 0;
-    const hasWebCitations = Array.isArray(webCitations) && webCitations.length > 0;
+    const hasHybridCitations = Array.isArray(citedHybridCitations) && citedHybridCitations.length > 0;
+    const hasWebCitations = Array.isArray(citedWebCitations) && citedWebCitations.length > 0;
     const hasAgentCitations = Array.isArray(agentCitations) && agentCitations.length > 0;
     if (!hasHybridCitations && !hasWebCitations && !hasAgentCitations) {
         container.classList.toggle("d-none", container.children.length === 0);
@@ -680,7 +687,7 @@ export async function renderInlineImageGalleries(
     let galleryIndex = 0;
     const seenImageKeys = new Set();
 
-    const workspaceItems = extractWorkspaceCitationImageItems(hybridCitations, seenImageKeys);
+    const workspaceItems = extractWorkspaceCitationImageItems(citedHybridCitations, seenImageKeys);
     if (workspaceItems.length > 0 && remainingSlots > 0) {
         const workspaceGallery = buildImageGalleryResult(
             "Workspace images",
@@ -697,11 +704,11 @@ export async function renderInlineImageGalleries(
         }
     }
 
-    const linkedItems = extractLinkedImageItems(webCitations, seenImageKeys);
+    const linkedItems = extractLinkedImageItems(citedWebCitations, seenImageKeys);
     if (linkedItems.length > 0 && remainingSlots > 0) {
         const linkedGallery = buildImageGalleryResult(
             "Linked images",
-            "Direct image links returned with this response.",
+            "Image links cited in this response.",
             linkedItems.slice(0, remainingSlots),
             "Linked sources",
             linkedItems.length

application/single_app/static/js/chat/chat-inline-videos.js

@@ -280,13 +280,13 @@ function normalizeWorkspaceCitationVideoItem(rawCitation, index = 0) {
     };
 }
 
-function extractWorkspaceCitationVideoItems(hybridCitations = [], seenKeys = new Set()) {
+function extractWorkspaceCitationVideoItems(citedHybridCitations = [], seenKeys = new Set()) {
     const items = [];
-    if (!Array.isArray(hybridCitations) || hybridCitations.length === 0) {
+    if (!Array.isArray(citedHybridCitations) || citedHybridCitations.length === 0) {
         return items;
     }
 
-    hybridCitations.forEach((citation, index) => {
+    citedHybridCitations.forEach((citation, index) => {
         pushUniqueVideoItem(items, seenKeys, normalizeWorkspaceCitationVideoItem(citation, index));
     });
 
@@ -343,13 +343,13 @@ function normalizeWebCitationVideoItem(rawCitation, index = 0) {
     };
 }
 
-function extractLinkedVideoItems(webCitations = [], seenKeys = new Set()) {
+function extractLinkedVideoItems(citedWebCitations = [], seenKeys = new Set()) {
     const items = [];
-    if (!Array.isArray(webCitations) || webCitations.length === 0) {
+    if (!Array.isArray(citedWebCitations) || citedWebCitations.length === 0) {
         return items;
     }
 
-    webCitations.forEach((citation, index) => {
+    citedWebCitations.forEach((citation, index) => {
         pushUniqueVideoItem(items, seenKeys, normalizeWebCitationVideoItem(citation, index));
     });
 
@@ -720,10 +720,17 @@ function createVideoGalleryCard(result) {
     return { card };
 }
 
+/**
+ * Render inline video galleries for one assistant message.
+ *
+ * The workspace and linked galleries take the cited citation subsets, not the
+ * full retrieved sets, so videos only appear when the response referenced them.
+ * Agent citations are executed tool results and are always rendered.
+ */
 export async function renderInlineVideoGalleries(
     messageElement,
-    hybridCitations = [],
-    webCitations = [],
+    citedHybridCitations = [],
+    citedWebCitations = [],
     agentCitations = [],
     conversationId = ""
 ) {
@@ -738,8 +745,8 @@ export async function renderInlineVideoGalleries(
 
     container.querySelectorAll(".inline-video-gallery-card").forEach((card) => card.remove());
 
-    const hasHybridCitations = Array.isArray(hybridCitations) && hybridCitations.length > 0;
-    const hasWebCitations = Array.isArray(webCitations) && webCitations.length > 0;
+    const hasHybridCitations = Array.isArray(citedHybridCitations) && citedHybridCitations.length > 0;
+    const hasWebCitations = Array.isArray(citedWebCitations) && citedWebCitations.length > 0;
     const hasAgentCitations = Array.isArray(agentCitations) && agentCitations.length > 0;
     if (!hasHybridCitations && !hasWebCitations && !hasAgentCitations) {
         container.classList.toggle("d-none", container.children.length === 0);
@@ -749,7 +756,7 @@ export async function renderInlineVideoGalleries(
     let remainingSlots = MAX_INLINE_VIDEO_ITEMS;
     const seenVideoKeys = new Set();
 
-    const workspaceItems = extractWorkspaceCitationVideoItems(hybridCitations, seenVideoKeys);
+    const workspaceItems = extractWorkspaceCitationVideoItems(citedHybridCitations, seenVideoKeys);
     if (workspaceItems.length > 0 && remainingSlots > 0) {
         const workspaceGallery = buildVideoGalleryResult(
             "Workspace videos",
@@ -765,11 +772,11 @@ export async function renderInlineVideoGalleries(
         }
     }
 
-    const linkedItems = extractLinkedVideoItems(webCitations, seenVideoKeys);
+    const linkedItems = extractLinkedVideoItems(citedWebCitations, seenVideoKeys);
     if (linkedItems.length > 0 && remainingSlots > 0) {
         const linkedGallery = buildVideoGalleryResult(
             "Linked videos",
-            "Direct video links returned with this response.",
+            "Video links cited in this response.",
             linkedItems.slice(0, remainingSlots),
             "Linked sources",
             linkedItems.length

application/single_app/static/js/chat/chat-messages.js

@@ -32,6 +32,7 @@ import { attachGeneratedImageProposalResults, extractInlineImageProposalBlocks,
 import { renderInlineVideoGalleries } from './chat-inline-videos.js';
 import { renderInlineImageGalleries } from './chat-inline-images.js';
 import { renderInlineAzureMaps } from './chat-inline-maps.js';
+import { getCitedHybridCitations, getCitedWebCitations } from './chat-citation-tracking.js';
 
 // Conditionally import TTS if enabled
 let ttsModule = null;
@@ -6010,17 +6011,23 @@ export function appendMessage(
     }
 
     void (async () => {
+      // Inline galleries present media as supporting the answer, so they render
+      // only what the response cited. The Sources disclosure keeps the complete
+      // retrieved set.
+      const citedHybridCitations = getCitedHybridCitations(fullMessageObject, hybridCitations);
+      const citedWebCitations = getCitedWebCitations(fullMessageObject, webCitations);
+
       await renderInlineVideoGalleries(
         messageDiv,
-        hybridCitations || [],
-        webCitations || [],
+        citedHybridCitations,
+        citedWebCitations,
         agentCitations || [],
         messageConversationId
       );
       await renderInlineImageGalleries(
         messageDiv,
-        hybridCitations || [],
-        webCitations || [],
+        citedHybridCitations,
+        citedWebCitations,
         agentCitations || [],
         messageId,
         messageConversationId

docs/explanation/fixes/INLINE_MEDIA_CITED_ONLY_GATING_FIX.md

@@ -0,0 +1,67 @@
+# Inline Media Cited-Only Gating Fix
+
+Fixed/Implemented in version: **0.260.024**
+
+GitHub issue: [#1329](https://github.com/microsoft/simplechat/issues/1329) (follow-up to [#1249](https://github.com/microsoft/simplechat/issues/1249))
+
+## Issue Description
+
+Assistant messages rendered inline image and video galleries for every media document returned by retrieval, not just the media the response actually cited. A workspace search that surfaced five image files produced five inline gallery tiles even when the answer referenced only one of them, or none at all.
+
+Because the galleries sit directly inside the message bubble, unrelated media was presented as though it supported the answer. The five-item gallery cap could also be consumed entirely by retrieval noise, pushing genuinely cited media out of view, and each unreferenced workspace file triggered an additional enhanced-citation fetch.
+
+## Root Cause Analysis
+
+Issue #1249 introduced the separation between retrieved sources and exact references. Every assistant message now persists `cited_hybrid_citations` and `cited_web_search_citations` alongside the complete `hybrid_citations` and `web_search_citations` arrays, and those cited subsets already reach the browser on all four delivery paths: history load, the streaming terminal event, the legacy non-streaming bridge, and collaboration message serialization.
+
+The inline gallery renderers were never switched over. `appendMessage` passed the full retrieved arrays into `renderInlineImageGalleries` and `renderInlineVideoGalleries`, and those renderers select workspace media purely by file extension through `extractWorkspaceCitationImageItems` and `extractWorkspaceCitationVideoItems`. Any retrieved `.png` or `.mp4` therefore became a gallery tile regardless of whether it was cited. No frontend module read the cited subsets at all.
+
+## Technical Details
+
+Files modified:
+
+- `application/single_app/static/js/chat/chat-citation-tracking.js` (new)
+- `application/single_app/static/js/chat/chat-messages.js`
+- `application/single_app/static/js/chat/chat-inline-images.js`
+- `application/single_app/static/js/chat/chat-inline-videos.js`
+- `application/single_app/config.py`
+- `functional_tests/test_inline_media_cited_only_gating.py` (new)
+- `functional_tests/test_inline_image_gallery_visualization.py`
+- `functional_tests/test_inline_video_gallery_visualization.py`
+- `functional_tests/test_chat_cited_source_tracking.py`
+- `ui_tests/test_chat_inline_image_gallery_rendering.py`
+- `ui_tests/test_chat_inline_video_gallery_rendering.py`
+
+Code changes summary:
+
+- Added `chat-citation-tracking.js`, the browser mirror of `functions_citation_tracking._message_has_citation_tracking()`. It exports `messageHasCitationTracking`, `getCitedHybridCitations`, and `getCitedWebCitations`. A message counts as tracked when `citation_tracking_version` is at least `1` or either `cited_*` key is present, and each getter normalizes non-array values to an empty list.
+- `appendMessage` now derives `citedHybridCitations` and `citedWebCitations` from the assistant message object and passes those to both gallery renderers. The Sources disclosure, its count badges, and the metadata drawer continue to receive the complete retrieved arrays.
+- Renamed the gallery entry-point and extraction helper parameters to `citedHybridCitations` and `citedWebCitations` so the narrowed contract is explicit at the call boundary.
+- Corrected the "Linked images" and "Linked videos" card summaries, which described the links as merely returned with the response rather than cited by it.
+
+No backend change was required. `functions_citation_tracking.build_cited_source_subsets()` already produces the subsets, and conversation and per-message exports already select them through `get_message_reference_citation_buckets()`.
+
+### Scope boundaries
+
+- **Agent and tool galleries still render.** An action that returns an image or video gallery is an executed tool result, not an unused retrieval candidate. This matches how #1249 treats agent records.
+- **Legacy messages keep prior behavior.** Messages saved before citation tracking existed carry no cited arrays and fall back to the full retrieved set, matching the no-migration legacy fallback in `get_message_reference_citation_buckets()` and #1249's decision to avoid read-time history parsing.
+- **A tracked response that cited nothing renders no workspace or linked gallery.** `renderInlineImageGalleries` and `renderInlineVideoGalleries` already collapse `.inline-visualizations-container` with `d-none` when it has no children, so an empty result leaves no visual gap.
+
+## Validation
+
+Test coverage added or updated:
+
+- `functional_tests/test_inline_media_cited_only_gating.py` executes the real `chat-citation-tracking.js` helper under Node across tracked, untracked, empty-cited, key-presence-only, missing-message, and malformed-value inputs, then asserts the wiring in `chat-messages.js`, that the Sources panel still receives full arrays, and that both renderers declare cited inputs.
+- `functional_tests/test_chat_cited_source_tracking.py` gained `test_inline_media_galleries_render_cited_subsets_only`, keeping the inline galleries inside the citation-tracking contract suite (17/17 passing).
+- `functional_tests/test_inline_image_gallery_visualization.py` and `functional_tests/test_inline_video_gallery_visualization.py` had their wiring assertions updated from the retrieved-set call shape to the cited-subset call shape.
+- `ui_tests/test_chat_inline_image_gallery_rendering.py` and `ui_tests/test_chat_inline_video_gallery_rendering.py` gained a gating regression test covering three messages in one page: a tracked response that cited one of two retrieved media files, a tracked response that cited nothing but ran a media action, and a legacy untracked response.
+
+Before and after:
+
+| Scenario | Before | After |
+| --- | --- | --- |
+| Tracked response, 1 of 5 retrieved images cited | 5 inline tiles | 1 inline tile |
+| Tracked response, no documents cited | Every retrieved image tiled | No workspace gallery |
+| Tracked response, no documents cited, media action ran | Retrieved images plus action gallery | Action gallery only |
+| Legacy untracked response | Every retrieved image tiled | Unchanged |
+| Sources disclosure | Complete retrieved set | Unchanged |

docs/explanation/fixes/index.md

@@ -38,3 +38,4 @@ category: Version History
 - [Collaboration Mention Tab Autocomplete Fix](COLLABORATION_MENTION_TAB_AUTOCOMPLETE_FIX.md)
 - [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md)
 - [Admin Settings Pane Variable Scope Fix](ADMIN_SETTINGS_PANE_VARIABLE_SCOPE_FIX.md)
+- [Inline Media Cited-Only Gating Fix](INLINE_MEDIA_CITED_ONLY_GATING_FIX.md)