From ae63b189685be559121cfacc8abcb13036915971 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 12:37:54 -0400 Subject: [PATCH 1/5] Render EMF/WMF diagrams so Office figures are analyzed Refs #1277 Images embedded in Word and PowerPoint as EMF or WMF metafiles were silently skipped, because the extraction whitelist only accepted raster formats. Word stores pasted diagrams, SmartArt, Visio drawings, and charts this way, so architecture diagrams -- often the densest figures in a document -- were never analyzed or indexed. A real TIC 3.0 document uploaded with Enhanced extraction on contained four EMF diagrams and produced zero image analysis and zero log output. Rasterizing them is not straightforward here. Pillow only installs a metafile renderer on Windows, where it is backed by GDI, and the application container is Linux distroless -- no shell and no package manager -- so an external converter such as LibreOffice or Inkscape is not an option either. functions_emf_render.py therefore renders metafiles in-process using only Pillow, io, and struct, so behavior is identical on every platform and the container is unchanged. It covers the record subset Office emits for diagrams: path construction, filled and stroked polygons, Bezier curves, rectangles and ellipses, pen and brush objects, world transforms, and text runs. Unsupported records are skipped rather than failing the render, so output degrades in fidelity instead of disappearing. Text drawn inside a metafile is extracted as well, so figure labels stay searchable even when the vision engine returns nothing. Verified against the four real diagrams: all render with plausible ink coverage and recover their labels, including Event Hub Namespace, Log Analytics Workspace, and Azure Firewall Table. Embedded image processing is also no longer silent. A document whose images were all skipped was indistinguishable from one with no images, which is what made the original problem impossible to diagnose from the workspace. Extraction now reports candidates found, images analyzed, and per-reason skip counts; progress is reported per image; and the counts are persisted on the document. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- application/single_app/functions_documents.py | 81 +- .../single_app/functions_emf_render.py | 922 ++++++++++++++++++ .../single_app/functions_office_media.py | 120 ++- ...NTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md | 42 +- docs/explanation/release_notes.md | 16 + ...content_understanding_extraction_engine.py | 2 +- .../test_office_embedded_image_extraction.py | 128 ++- 8 files changed, 1283 insertions(+), 30 deletions(-) create mode 100644 application/single_app/functions_emf_render.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 36746ff6b..beb2b9d6a 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.222" +VERSION = "0.250.223" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index f0c735eda..b5c60c333 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -26,6 +26,7 @@ ) from functions_visio import build_visio_page_markdown, parse_vsdx_pages from functions_content import * +from functions_office_media import extract_office_embedded_images_with_diagnostics from functions_content_understanding import analyze_image_with_content_understanding from functions_settings import * from functions_search import * @@ -311,6 +312,29 @@ def _analyze_single_embedded_image(image_path, extraction_engine, image_extracti ).strip() +def _describe_embedded_image_skips(diagnostics): + """Render skip counts as a short, admin-readable explanation.""" + reasons = diagnostics.get('skipped_reasons') or {} + if not reasons: + return '' + friendly = { + 'below_minimum_pixels': 'too small', + 'below_minimum_bytes': 'too small', + 'duplicate_image': 'duplicates', + 'unsupported_format': 'unsupported format', + 'unreadable_image': 'unreadable', + 'unreadable_or_oversized': 'unreadable or oversized', + 'per_document_cap_reached': 'over the per-document cap', + 'write_failed': 'could not be written', + 'unsafe_entry_name': 'unsafe file name', + } + parts = [] + for reason, count in sorted(reasons.items(), key=lambda item: -item[1]): + label = friendly.get(reason, reason.replace('_', ' ')) + parts.append(f"{count} {label}") + return ", ".join(parts) + + def _build_office_embedded_image_chunks( file_path, settings, @@ -347,14 +371,39 @@ def _build_office_embedded_image_chunks( try: temp_image_dir = tempfile.mkdtemp(prefix='office_images_') - embedded_images = extract_office_embedded_images( + embedded_images, image_diagnostics = extract_office_embedded_images_with_diagnostics( file_path, temp_image_dir, min_pixels=min_pixels, max_images=max_images, ) + candidate_count = image_diagnostics.get('candidates', 0) if not embedded_images: + # Say so explicitly. Otherwise "no images in this file" and "images found but all + # skipped" look identical in the workspace log, which is the common confusion. + if candidate_count: + skip_summary = _describe_embedded_image_skips(image_diagnostics) + update_callback( + status=( + f"Found {candidate_count} embedded image(s), none analyzable" + + (f" ({skip_summary})" if skip_summary else "") + ), + office_embedded_image_count=0, + office_embedded_image_candidates=candidate_count, + office_embedded_image_skipped=image_diagnostics.get('skipped', 0), + ) + log_event( + f"[OFFICE_EMBEDDED_IMAGES] {os.path.basename(file_path)}: " + f"{candidate_count} candidate(s), none analyzable. {image_diagnostics}", + level=logging.WARNING, + ) + else: + update_callback( + status="No embedded images found in this document", + office_embedded_image_count=0, + office_embedded_image_candidates=0, + ) return [], 0, extraction_engine engine_label = ( @@ -364,10 +413,13 @@ def _build_office_embedded_image_chunks( ) total_images = len(embedded_images) update_callback( - status=f"Analyzing {total_images} embedded image(s) with {engine_label}..." + status=f"Analyzing {total_images} of {candidate_count} embedded image(s) with {engine_label}..." ) for image_index, embedded_image in enumerate(embedded_images, start=1): + update_callback( + status=f"Analyzing embedded image {image_index} of {total_images} with {engine_label}..." + ) try: analysis_text = _analyze_single_embedded_image( embedded_image['path'], @@ -380,9 +432,18 @@ def _build_office_embedded_image_chunks( f"[OFFICE_EMBEDDED_IMAGES] Failed to analyze {embedded_image.get('name')}: {image_error}", level=logging.WARNING, ) - continue - - if not analysis_text: + analysis_text = '' + + # Text drawn inside a vector diagram is recovered during rasterization, so a figure + # still contributes searchable labels even when the engine returns nothing. + embedded_text = str(embedded_image.get('embedded_text') or '').strip() + body_parts = [] + if analysis_text: + body_parts.append(analysis_text) + if embedded_text and embedded_text not in analysis_text: + body_parts.append(f"Text labels in this figure:\n{embedded_text}") + + if not body_parts: continue location_label = '' @@ -396,13 +457,19 @@ def _build_office_embedded_image_chunks( ) chunks.append({ 'page_number': starting_page_number + len(chunks), - 'content': f"{heading}\n\n{analysis_text}", + 'content': f"{heading}\n\n" + "\n\n".join(body_parts), }) analyzed_count += 1 if analyzed_count: + skip_summary = _describe_embedded_image_skips(image_diagnostics) update_callback( - status=f"Analyzed {analyzed_count} embedded image(s) with {engine_label}." + status=( + f"Analyzed {analyzed_count} of {candidate_count} embedded image(s) with {engine_label}." + + (f" Skipped: {skip_summary}." if skip_summary else "") + ), + office_embedded_image_candidates=candidate_count, + office_embedded_image_skipped=image_diagnostics.get('skipped', 0), ) except Exception as embedded_image_error: log_event( diff --git a/application/single_app/functions_emf_render.py b/application/single_app/functions_emf_render.py new file mode 100644 index 000000000..04c657231 --- /dev/null +++ b/application/single_app/functions_emf_render.py @@ -0,0 +1,922 @@ +# functions_emf_render.py + +"""Pure-Python EMF/WMF rasterizer. + +Word stores pasted diagrams, SmartArt, and charts as EMF metafiles, and those are frequently the +most information-dense figures in a document. Neither Document Intelligence nor Content +Understanding accepts a metafile, so it has to be rasterized before it can be described. + +Pillow only ships a metafile renderer on Windows, where it is backed by GDI, and the application +container is Linux distroless -- no shell, no package manager, so an external converter such as +LibreOffice or Inkscape is not an option. This module therefore renders the common EMF drawing +subset directly with Pillow, which is already a dependency and behaves identically on every +platform. + +Scope: the record subset Office actually emits for diagrams -- path construction, filled and +stroked polygons, Bezier curves, rectangles and ellipses, pen and brush objects, world transforms, +and text runs. Records outside that subset are skipped rather than failing the render, so output +degrades in fidelity instead of disappearing. This is a description aid for search and citation, +not a pixel-accurate GDI reimplementation. +""" + +import struct +from io import BytesIO + +from PIL import Image, ImageDraw, ImageFont + + +# --- EMF record types actually handled ------------------------------------------------------- +EMR_HEADER = 1 +EMR_POLYBEZIER = 2 +EMR_POLYGON = 3 +EMR_POLYLINE = 4 +EMR_POLYBEZIERTO = 5 +EMR_POLYLINETO = 6 +EMR_POLYPOLYLINE = 7 +EMR_POLYPOLYGON = 8 +EMR_SETWINDOWEXTEX = 9 +EMR_SETWINDOWORGEX = 10 +EMR_SETVIEWPORTEXTEX = 11 +EMR_SETVIEWPORTORGEX = 12 +EMR_EOF = 14 +EMR_SETPOLYFILLMODE = 19 +EMR_SETTEXTCOLOR = 24 +EMR_MOVETOEX = 27 +EMR_SAVEDC = 33 +EMR_RESTOREDC = 34 +EMR_SETWORLDTRANSFORM = 35 +EMR_MODIFYWORLDTRANSFORM = 36 +EMR_SELECTOBJECT = 37 +EMR_CREATEPEN = 38 +EMR_CREATEBRUSHINDIRECT = 39 +EMR_DELETEOBJECT = 40 +EMR_ELLIPSE = 42 +EMR_RECTANGLE = 43 +EMR_ROUNDRECT = 44 +EMR_LINETO = 54 +EMR_BEGINPATH = 59 +EMR_ENDPATH = 60 +EMR_CLOSEFIGURE = 61 +EMR_FILLPATH = 62 +EMR_STROKEANDFILLPATH = 63 +EMR_STROKEPATH = 64 +EMR_EXTTEXTOUTA = 83 +EMR_EXTTEXTOUTW = 84 +EMR_POLYBEZIER16 = 85 +EMR_POLYGON16 = 86 +EMR_POLYLINE16 = 87 +EMR_POLYBEZIERTO16 = 88 +EMR_POLYLINETO16 = 89 +EMR_POLYPOLYLINE16 = 90 +EMR_POLYPOLYGON16 = 91 +EMR_EXTCREATEPEN = 95 + +# Stock object handles have the high bit set. +STOCK_OBJECT_FLAG = 0x80000000 +STOCK_WHITE_BRUSH = 0x80000000 +STOCK_LTGRAY_BRUSH = 0x80000001 +STOCK_GRAY_BRUSH = 0x80000002 +STOCK_DKGRAY_BRUSH = 0x80000003 +STOCK_BLACK_BRUSH = 0x80000004 +STOCK_NULL_BRUSH = 0x80000005 +STOCK_WHITE_PEN = 0x80000006 +STOCK_BLACK_PEN = 0x80000007 +STOCK_NULL_PEN = 0x80000008 + +BRUSH_STYLE_NULL = 1 +PEN_STYLE_NULL = 5 + +# Guard rails for untrusted input. +EMF_MAX_RECORDS = 200000 +EMF_MAX_POINTS_PER_RECORD = 100000 +EMF_MAX_OUTPUT_PIXELS = 4000 +EMF_MIN_OUTPUT_PIXELS = 16 +EMF_SUPERSAMPLE = 2 +BEZIER_SEGMENTS = 12 + + +class _GraphicsState: + """The subset of GDI device-context state this renderer tracks.""" + + def __init__(self): + self.transform = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0) + self.pen_color = (0, 0, 0) + self.pen_width = 1.0 + self.pen_visible = True + self.brush_color = (255, 255, 255) + self.brush_visible = True + self.text_color = (0, 0, 0) + + def copy(self): + clone = _GraphicsState() + clone.__dict__.update(self.__dict__) + return clone + + +def _colorref_to_rgb(value): + """COLORREF is 0x00BBGGRR.""" + return (value & 0xFF, (value >> 8) & 0xFF, (value >> 16) & 0xFF) + + +def _multiply_transform(left, right): + """Compose two 2D affine transforms stored as (m11, m12, m21, m22, dx, dy).""" + a11, a12, a21, a22, adx, ady = left + b11, b12, b21, b22, bdx, bdy = right + return ( + a11 * b11 + a12 * b21, + a11 * b12 + a12 * b22, + a21 * b11 + a22 * b21, + a21 * b12 + a22 * b22, + adx * b11 + ady * b21 + bdx, + adx * b12 + ady * b22 + bdy, + ) + + +def _flatten_bezier(points): + """Flatten a sequence of cubic Bezier segments into a polyline.""" + if len(points) < 4: + return list(points) + + flattened = [points[0]] + for start in range(0, len(points) - 3, 3): + p0, p1, p2, p3 = points[start:start + 4] + for step in range(1, BEZIER_SEGMENTS + 1): + t = step / BEZIER_SEGMENTS + inv = 1.0 - t + x = (inv ** 3) * p0[0] + 3 * (inv ** 2) * t * p1[0] + 3 * inv * (t ** 2) * p2[0] + (t ** 3) * p3[0] + y = (inv ** 3) * p0[1] + 3 * (inv ** 2) * t * p1[1] + 3 * inv * (t ** 2) * p2[1] + (t ** 3) * p3[1] + flattened.append((x, y)) + return flattened + + +class EmfRenderer: + """Parse an EMF byte string and rasterize the supported drawing records.""" + + def __init__(self, data, max_pixels=1600): + self.data = data + self.max_pixels = max_pixels + self.state = _GraphicsState() + self.state_stack = [] + self.objects = {} + self.current_point = (0.0, 0.0) + self.path = [] + self.current_subpath = [] + self.in_path = False + self.text_runs = [] + self.records_drawn = 0 + self.image = None + self.draw = None + self.scale = 1.0 + self.origin = (0.0, 0.0) + self.size = (0, 0) + + # --- coordinate mapping ------------------------------------------------------------------ + def _to_device(self, point): + m11, m12, m21, m22, dx, dy = self.state.transform + x, y = point + return (m11 * x + m21 * y + dx, m12 * x + m22 * y + dy) + + def _to_pixels(self, point): + device_x, device_y = self._to_device(point) + return ( + (device_x - self.origin[0]) * self.scale, + (device_y - self.origin[1]) * self.scale, + ) + + def _map_points(self, points): + return [self._to_pixels(point) for point in points] + + # --- record payload readers -------------------------------------------------------------- + @staticmethod + def _read_points16(payload, offset): + """Read the bounds + count + 16-bit point array shared by the *16 records.""" + if len(payload) < offset + 4: + return [] + count = struct.unpack_from(' EMF_MAX_POINTS_PER_RECORD: + return [] + start = offset + 4 + if len(payload) < start + count * 4: + return [] + raw = struct.unpack_from(f'<{count * 2}h', payload, start) + return [(float(raw[i]), float(raw[i + 1])) for i in range(0, len(raw), 2)] + + @staticmethod + def _read_points32(payload, offset): + if len(payload) < offset + 4: + return [] + count = struct.unpack_from(' EMF_MAX_POINTS_PER_RECORD: + return [] + start = offset + 4 + if len(payload) < start + count * 8: + return [] + raw = struct.unpack_from(f'<{count * 2}i', payload, start) + return [(float(raw[i]), float(raw[i + 1])) for i in range(0, len(raw), 2)] + + # --- drawing ----------------------------------------------------------------------------- + def _stroke(self, points, close=False): + if not self.state.pen_visible or len(points) < 2: + return + pixels = [(round(x), round(y)) for x, y in points] + if close: + pixels = pixels + [pixels[0]] + width = max(1, int(round(self.state.pen_width * self.scale))) + self.draw.line(pixels, fill=self.state.pen_color, width=width, joint='curve') + self.records_drawn += 1 + + def _fill(self, points): + if not self.state.brush_visible or len(points) < 3: + return + pixels = [(round(x), round(y)) for x, y in points] + try: + self.draw.polygon(pixels, fill=self.state.brush_color) + self.records_drawn += 1 + except (ValueError, TypeError): + pass + + def _flush_current_subpath(self): + if len(self.current_subpath) >= 2: + self.path.append(list(self.current_subpath)) + self.current_subpath = [] + + def _render_path(self, fill, stroke): + self._flush_current_subpath() + for subpath in self.path: + mapped = self._map_points(subpath) + if fill: + self._fill(mapped) + if stroke: + self._stroke(mapped) + self.path = [] + + # --- object table ------------------------------------------------------------------------ + def _select_stock_object(self, handle): + if handle == STOCK_NULL_BRUSH: + self.state.brush_visible = False + elif handle in (STOCK_WHITE_BRUSH, STOCK_LTGRAY_BRUSH, STOCK_GRAY_BRUSH, + STOCK_DKGRAY_BRUSH, STOCK_BLACK_BRUSH): + shades = { + STOCK_WHITE_BRUSH: (255, 255, 255), + STOCK_LTGRAY_BRUSH: (192, 192, 192), + STOCK_GRAY_BRUSH: (128, 128, 128), + STOCK_DKGRAY_BRUSH: (64, 64, 64), + STOCK_BLACK_BRUSH: (0, 0, 0), + } + self.state.brush_color = shades[handle] + self.state.brush_visible = True + elif handle == STOCK_NULL_PEN: + self.state.pen_visible = False + elif handle in (STOCK_WHITE_PEN, STOCK_BLACK_PEN): + self.state.pen_color = (255, 255, 255) if handle == STOCK_WHITE_PEN else (0, 0, 0) + self.state.pen_visible = True + self.state.pen_width = 1.0 + + def _select_object(self, handle): + if handle & STOCK_OBJECT_FLAG: + self._select_stock_object(handle) + return + entry = self.objects.get(handle) + if not entry: + return + kind, payload = entry + if kind == 'brush': + color, visible = payload + self.state.brush_color = color + self.state.brush_visible = visible + elif kind == 'pen': + color, width, visible = payload + self.state.pen_color = color + self.state.pen_width = width + self.state.pen_visible = visible + + # --- header ------------------------------------------------------------------------------ + def _parse_header(self, payload): + if len(payload) < 32: + return None + bounds = struct.unpack_from('<4i', payload, 0) + left, top, right, bottom = bounds + width = right - left + height = bottom - top + if width <= 0 or height <= 0: + return None + return (left, top, width, height) + + def render(self): + """Rasterize the metafile. Returns a PIL Image, or None when nothing could be drawn.""" + data = self.data + if len(data) < 88: + return None + + record_type, record_size = struct.unpack_from(' len(data): + return None + + header_bounds = self._parse_header(data[8:record_size]) + if header_bounds is None: + return None + + left, top, logical_width, logical_height = header_bounds + + longest_edge = max(logical_width, logical_height) + target_scale = min(1.0, float(self.max_pixels) / float(longest_edge)) if longest_edge else 1.0 + output_width = max(EMF_MIN_OUTPUT_PIXELS, min(EMF_MAX_OUTPUT_PIXELS, int(logical_width * target_scale))) + output_height = max(EMF_MIN_OUTPUT_PIXELS, min(EMF_MAX_OUTPUT_PIXELS, int(logical_height * target_scale))) + + self.scale = (output_width / logical_width) * EMF_SUPERSAMPLE + self.origin = (left, top) + canvas_size = (output_width * EMF_SUPERSAMPLE, output_height * EMF_SUPERSAMPLE) + self.size = (output_width, output_height) + + self.image = Image.new('RGB', canvas_size, (255, 255, 255)) + self.draw = ImageDraw.Draw(self.image) + + self._walk_records(data) + + if self.records_drawn == 0 and not self.text_runs: + return None + + return self.image.resize(self.size, Image.LANCZOS) + + def _walk_records(self, data): + offset = 0 + records = 0 + length = len(data) + + while offset + 8 <= length and records < EMF_MAX_RECORDS: + record_type, record_size = struct.unpack_from(' length: + break + payload = data[offset + 8: offset + record_size] + records += 1 + offset += record_size + + if record_type == EMR_EOF: + break + try: + self._handle_record(record_type, payload) + except (struct.error, ValueError, IndexError, TypeError): + # A malformed record must not abort the whole render. + continue + + def _handle_record(self, record_type, payload): + state = self.state + + if record_type == EMR_SAVEDC: + self.state_stack.append(state.copy()) + + elif record_type == EMR_RESTOREDC: + if self.state_stack: + self.state = self.state_stack.pop() + + elif record_type == EMR_SETWORLDTRANSFORM: + state.transform = struct.unpack_from('<6f', payload, 0) + + elif record_type == EMR_MODIFYWORLDTRANSFORM: + xform = struct.unpack_from('<6f', payload, 0) + mode = struct.unpack_from('= 2: + self.current_subpath.append(self.current_subpath[0]) + self._flush_current_subpath() + + elif record_type == EMR_FILLPATH: + self._render_path(fill=True, stroke=False) + + elif record_type == EMR_STROKEPATH: + self._render_path(fill=False, stroke=True) + + elif record_type == EMR_STROKEANDFILLPATH: + self._render_path(fill=True, stroke=True) + + elif record_type == EMR_MOVETOEX: + x, y = struct.unpack_from('<2i', payload, 0) + self._flush_current_subpath() + self.current_point = (float(x), float(y)) + self.current_subpath = [self.current_point] + + elif record_type == EMR_LINETO: + x, y = struct.unpack_from('<2i', payload, 0) + point = (float(x), float(y)) + if not self.current_subpath: + self.current_subpath = [self.current_point] + self.current_subpath.append(point) + self.current_point = point + if not self.in_path: + self._stroke(self._map_points(self.current_subpath[-2:])) + + elif record_type in (EMR_POLYGON16, EMR_POLYLINE16, EMR_POLYBEZIER16): + points = self._read_points16(payload, 16) + if record_type == EMR_POLYBEZIER16: + points = _flatten_bezier(points) + self._draw_standalone(record_type, points) + + elif record_type in (EMR_POLYGON, EMR_POLYLINE, EMR_POLYBEZIER): + points = self._read_points32(payload, 16) + if record_type == EMR_POLYBEZIER: + points = _flatten_bezier(points) + self._draw_standalone(record_type, points) + + elif record_type in (EMR_POLYLINETO16, EMR_POLYBEZIERTO16): + points = self._read_points16(payload, 16) + if record_type == EMR_POLYBEZIERTO16: + points = _flatten_bezier([self.current_point] + points) + self._append_to_current(points) + + elif record_type in (EMR_POLYLINETO, EMR_POLYBEZIERTO): + points = self._read_points32(payload, 16) + if record_type == EMR_POLYBEZIERTO: + points = _flatten_bezier([self.current_point] + points) + self._append_to_current(points) + + elif record_type in (EMR_POLYPOLYGON16, EMR_POLYPOLYLINE16): + self._draw_poly_poly(payload, record_type == EMR_POLYPOLYGON16) + + elif record_type == EMR_RECTANGLE: + left, top, right, bottom = struct.unpack_from('<4i', payload, 0) + corners = [(left, top), (right, top), (right, bottom), (left, bottom)] + mapped = self._map_points([(float(x), float(y)) for x, y in corners]) + self._fill(mapped) + self._stroke(mapped, close=True) + + elif record_type in (EMR_ELLIPSE, EMR_ROUNDRECT): + left, top, right, bottom = struct.unpack_from('<4i', payload, 0) + mapped = self._map_points([ + (float(left), float(top)), (float(right), float(top)), + (float(right), float(bottom)), (float(left), float(bottom)), + ]) + xs = [p[0] for p in mapped] + ys = [p[1] for p in mapped] + box = [round(min(xs)), round(min(ys)), round(max(xs)), round(max(ys))] + if box[2] > box[0] and box[3] > box[1]: + shape = self.draw.ellipse if record_type == EMR_ELLIPSE else self.draw.rectangle + if self.state.brush_visible: + shape(box, fill=self.state.brush_color) + self.records_drawn += 1 + if self.state.pen_visible: + shape(box, outline=self.state.pen_color, + width=max(1, int(round(self.state.pen_width * self.scale)))) + self.records_drawn += 1 + + elif record_type in (EMR_EXTTEXTOUTW, EMR_EXTTEXTOUTA): + self._handle_text(record_type, payload) + + def _draw_standalone(self, record_type, points): + if not points: + return + if self.in_path: + self._flush_current_subpath() + self.path.append(points) + return + mapped = self._map_points(points) + if record_type in (EMR_POLYGON16, EMR_POLYGON): + self._fill(mapped) + self._stroke(mapped, close=True) + else: + self._stroke(mapped) + + def _append_to_current(self, points): + if not points: + return + if not self.current_subpath: + self.current_subpath = [self.current_point] + self.current_subpath.extend(points) + self.current_point = points[-1] + if not self.in_path: + self._stroke(self._map_points(self.current_subpath)) + self.current_subpath = [self.current_point] + + def _draw_poly_poly(self, payload, filled): + polygon_count, total_points = struct.unpack_from(' 10000 or total_points > EMF_MAX_POINTS_PER_RECORD: + return + counts = struct.unpack_from(f'<{polygon_count}I', payload, 24) + offset = 24 + polygon_count * 4 + for count in counts: + if count == 0 or count > EMF_MAX_POINTS_PER_RECORD: + break + if len(payload) < offset + count * 4: + break + raw = struct.unpack_from(f'<{count * 2}h', payload, offset) + points = [(float(raw[i]), float(raw[i + 1])) for i in range(0, len(raw), 2)] + offset += count * 4 + mapped = self._map_points(points) + if filled: + self._fill(mapped) + self._stroke(mapped, close=True) + else: + self._stroke(mapped) + + def _handle_text(self, record_type, payload): + """Record a text run and draw it approximately. + + EMR_EXTTEXTOUT payload layout: rclBounds(16), iGraphicsMode(4), exScale(4), eyScale(4), + then the EmrText struct: ptlReference(8), nChars(4), offString(4), fOptions(4), rcl(16), + offDx(4). ``offString`` is measured from the start of the record, which begins 8 bytes + before this payload. + """ + if len(payload) < 44: + return + + reference_x, reference_y = struct.unpack_from('<2i', payload, 28) + char_count, string_offset = struct.unpack_from(' 8192: + return + + start = string_offset - 8 + if start < 0: + return + + if record_type == EMR_EXTTEXTOUTW: + byte_length = char_count * 2 + if len(payload) < start + byte_length: + return + text = payload[start:start + byte_length].decode('utf-16-le', errors='ignore') + else: + if len(payload) < start + char_count: + return + text = payload[start:start + char_count].decode('latin-1', errors='ignore') + + text = text.replace('\x00', '').strip() + if not text: + return + + self.text_runs.append(text) + + pixel_x, pixel_y = self._to_pixels((float(reference_x), float(reference_y))) + if not (0 <= pixel_x < self.image.width and 0 <= pixel_y < self.image.height): + return + try: + font_size = max(8, int(round(11 * self.scale))) + font = ImageFont.load_default(size=font_size) + except (AttributeError, TypeError, OSError): + font = None + try: + self.draw.text((pixel_x, pixel_y), text, fill=self.state.text_color, font=font) + self.records_drawn += 1 + except (ValueError, OSError): + pass + + +WMF_PLACEABLE_KEY = 0x9AC6CDD7 + +META_SETWINDOWORG = 0x020B +META_SETWINDOWEXT = 0x020C +META_LINETO = 0x0213 +META_MOVETO = 0x0214 +META_POLYGON = 0x0324 +META_POLYLINE = 0x0325 +META_ELLIPSE = 0x0418 +META_RECTANGLE = 0x041B +META_ROUNDRECT = 0x061C +META_POLYPOLYGON = 0x0538 +META_TEXTOUT = 0x0521 +META_EXTTEXTOUT = 0x0A32 +META_SELECTOBJECT = 0x012D +META_DELETEOBJECT = 0x01F0 +META_CREATEPENINDIRECT = 0x02FA +META_CREATEBRUSHINDIRECT = 0x02FC +META_SETTEXTCOLOR = 0x0209 +META_SAVEDC = 0x001E +META_RESTOREDC = 0x0127 + + +def _looks_like_wmf(data): + """Return True for a placeable or standard WMF header.""" + if len(data) < 18: + return False + if struct.unpack_from(' length: + break + params = data[offset + 6: offset + record_bytes] + if function == META_SETWINDOWORG and len(params) >= 4: + y, x = struct.unpack_from('<2h', params, 0) + self.window_origin = (float(x), float(y)) + elif function == META_SETWINDOWEXT and len(params) >= 4: + height, width = struct.unpack_from('<2h', params, 0) + self.window_extent = (float(width), float(height)) + offset += record_bytes + records += 1 + + def _walk_wmf_records(self, data, offset): + records = 0 + length = len(data) + while offset + 6 <= length and records < EMF_MAX_RECORDS: + record_words, function = struct.unpack_from(' length: + break + params = data[offset + 6: offset + record_bytes] + offset += record_bytes + records += 1 + if function == 0: + break + try: + self._handle_wmf_record(function, params) + except (struct.error, ValueError, IndexError, TypeError): + continue + + def _add_object(self, entry): + for index, existing in enumerate(self.object_table): + if existing is None: + self.object_table[index] = entry + return + self.object_table.append(entry) + + def _handle_wmf_record(self, function, params): + state = self.state + + if function == META_SAVEDC: + self.state_stack.append(state.copy()) + + elif function == META_RESTOREDC: + if self.state_stack: + self.state = self.state_stack.pop() + + elif function == META_CREATEBRUSHINDIRECT and len(params) >= 8: + brush_style, color = struct.unpack_from('= 10: + pen_style, width = struct.unpack_from('= 2: + index = struct.unpack_from('= 2: + index = struct.unpack_from('= 4: + state.text_color = _colorref_to_rgb(struct.unpack_from('= 4: + y, x = struct.unpack_from('<2h', params, 0) + self.current_point = (float(x), float(y)) + + elif function == META_LINETO and len(params) >= 4: + y, x = struct.unpack_from('<2h', params, 0) + end_point = (float(x), float(y)) + self._stroke(self._map_points([self.current_point, end_point])) + self.current_point = end_point + + elif function in (META_POLYGON, META_POLYLINE) and len(params) >= 2: + count = struct.unpack_from('= 2 + count * 4: + raw = struct.unpack_from(f'<{count * 2}h', params, 2) + points = [(float(raw[i]), float(raw[i + 1])) for i in range(0, len(raw), 2)] + mapped = self._map_points(points) + if function == META_POLYGON: + self._fill(mapped) + self._stroke(mapped, close=True) + else: + self._stroke(mapped) + + elif function == META_POLYPOLYGON and len(params) >= 2: + polygon_count = struct.unpack_from('= 2 + polygon_count * 2: + counts = struct.unpack_from(f'<{polygon_count}H', params, 2) + point_offset = 2 + polygon_count * 2 + for count in counts: + if not count or len(params) < point_offset + count * 4: + break + raw = struct.unpack_from(f'<{count * 2}h', params, point_offset) + points = [(float(raw[i]), float(raw[i + 1])) for i in range(0, len(raw), 2)] + point_offset += count * 4 + mapped = self._map_points(points) + self._fill(mapped) + self._stroke(mapped, close=True) + + elif function in (META_RECTANGLE, META_ELLIPSE, META_ROUNDRECT) and len(params) >= 8: + # Parameters are stored bottom, right, top, left. + bottom, right, top, left = struct.unpack_from('<4h', params, 0) + mapped = self._map_points([ + (float(left), float(top)), (float(right), float(top)), + (float(right), float(bottom)), (float(left), float(bottom)), + ]) + if function == META_RECTANGLE or function == META_ROUNDRECT: + self._fill(mapped) + self._stroke(mapped, close=True) + else: + xs = [p[0] for p in mapped] + ys = [p[1] for p in mapped] + box = [round(min(xs)), round(min(ys)), round(max(xs)), round(max(ys))] + if box[2] > box[0] and box[3] > box[1]: + if state.brush_visible: + self.draw.ellipse(box, fill=state.brush_color) + self.records_drawn += 1 + if state.pen_visible: + self.draw.ellipse(box, outline=state.pen_color, + width=max(1, int(round(state.pen_width * self.scale)))) + self.records_drawn += 1 + + elif function in (META_TEXTOUT, META_EXTTEXTOUT): + self._handle_wmf_text(function, params) + + def _handle_wmf_text(self, function, params): + if function == META_TEXTOUT: + if len(params) < 2: + return + char_count = struct.unpack_from(' 8192 or len(params) < 2 + char_count: + return + raw_text = params[2:2 + char_count] + padded = char_count + (char_count % 2) + if len(params) >= 2 + padded + 4: + y, x = struct.unpack_from('<2h', params, 2 + padded) + else: + y, x = 0, 0 + else: + if len(params) < 8: + return + y, x, char_count = struct.unpack_from('<3h', params, 0) + char_count = max(0, char_count) + if not char_count or char_count > 8192: + return + options = struct.unpack_from('= 88 and data[40:44] == b' EMF': + renderer_factory = EmfRenderer + elif _looks_like_wmf(data): + renderer_factory = WmfRenderer + else: + return None, 0, 0, '', 'unrecognized_metafile_format' + + try: + renderer = renderer_factory(data, max_pixels=max_pixels) + image = renderer.render() + except Exception as render_error: + return None, 0, 0, '', f'metafile_render_failed ({type(render_error).__name__})' + + if image is None: + return None, 0, 0, '', 'metafile_had_no_drawable_records' + + buffer = BytesIO() + image.save(buffer, format='PNG') + text = '\n'.join(dict.fromkeys(run for run in renderer.text_runs if run)) + return buffer.getvalue(), image.width, image.height, text, '' diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py index 1b9fca28e..6e2df29dd 100644 --- a/application/single_app/functions_office_media.py +++ b/application/single_app/functions_office_media.py @@ -19,11 +19,22 @@ from defusedxml.ElementTree import fromstring as defused_fromstring from PIL import Image +from functions_emf_render import render_metafile_to_png + OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES = ('word/media/', 'ppt/media/', 'xl/media/') -# Restricted to raster formats that both Document Intelligence and Content Understanding accept. +# Raster formats that both Document Intelligence and Content Understanding accept directly. OFFICE_EMBEDDED_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.bmp', '.tif', '.tiff', '.heif', '.heic') +# Vector metafiles. Word stores pasted diagrams, SmartArt, and charts this way, so they are the most +# interesting figures in many documents. Neither analysis engine accepts them, so they are +# rasterized to PNG first when the platform can render them. +OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS = ('.emf', '.wmf') +OFFICE_EMBEDDED_IMAGE_ALL_EXTENSIONS = ( + OFFICE_EMBEDDED_IMAGE_EXTENSIONS + OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS +) OFFICE_EMBEDDED_IMAGE_MIN_BYTES = 2048 +# Rasterized metafiles are capped on the long edge to keep analysis payloads reasonable. +OFFICE_EMBEDDED_IMAGE_RENDER_MAX_PIXELS = 1600 # Uploaded Office files are untrusted, so refuse to decompress an oversized embedded image. OFFICE_EMBEDDED_IMAGE_MAX_BYTES = 64 * 1024 * 1024 # Slide relationship parts are small XML documents; anything larger is not worth decompressing. @@ -141,8 +152,35 @@ def _build_pptx_media_slide_map(archive): return media_slide_map +def _rasterize_vector_image(image_bytes): + """Rasterize an EMF/WMF metafile to PNG bytes. + + Pillow only installs a metafile renderer on Windows, where it is backed by GDI, and this + application runs in a Linux distroless container. Rendering therefore goes through the + in-process metafile rasterizer, which behaves identically on every platform and needs no + system packages. + + Returns ``(png_bytes, width, height, text, reason)`` where ``png_bytes`` is None on failure. + """ + return render_metafile_to_png(image_bytes, max_pixels=OFFICE_EMBEDDED_IMAGE_RENDER_MAX_PIXELS) + + +def _new_diagnostics(): + return { + 'candidates': 0, + 'analyzed': 0, + 'skipped': 0, + 'skipped_reasons': {}, + } + + +def _record_skip(diagnostics, reason): + diagnostics['skipped'] += 1 + diagnostics['skipped_reasons'][reason] = diagnostics['skipped_reasons'].get(reason, 0) + 1 + + def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_images=25): - """Extract analyzable raster images embedded in an OOXML Office file. + """Extract analyzable images embedded in an OOXML Office file. Args: file_path (str): Path to the DOCX/PPTX/XLSX file. @@ -153,8 +191,30 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im Returns: list: Dicts with ``name``, ``path``, ``width``, ``height``, and ``slide_number`` keys. """ + extracted_images, _diagnostics = extract_office_embedded_images_with_diagnostics( + file_path, + output_dir, + min_pixels=min_pixels, + max_images=max_images, + ) + return extracted_images + + +def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_pixels=150, max_images=25): + """Extract embedded images and report what was skipped and why. + + Without the diagnostics a document containing only unsupported figures is indistinguishable + from a document containing no figures at all, which makes "were my images analyzed?" + unanswerable from the workspace UI. + + Returns: + tuple: ``(images, diagnostics)`` where diagnostics has ``candidates``, ``analyzed``, + ``skipped``, and ``skipped_reasons``. + """ + diagnostics = _new_diagnostics() + if max_images <= 0: - return [] + return [], diagnostics extracted_images = [] seen_digests = set() @@ -164,20 +224,22 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im entry_names = archive.namelist() if len(entry_names) > OFFICE_ZIP_MAX_ENTRIES: # A real Office package never has this many parts; refuse to walk a crafted archive. - return [] + return [], diagnostics media_slide_map = _build_pptx_media_slide_map(archive) candidate_names = [ entry_name for entry_name in entry_names if entry_name.lower().startswith(OFFICE_EMBEDDED_IMAGE_MEDIA_PREFIXES) - and entry_name.lower().endswith(OFFICE_EMBEDDED_IMAGE_EXTENSIONS) + and entry_name.lower().endswith(OFFICE_EMBEDDED_IMAGE_ALL_EXTENSIONS) ] + diagnostics['candidates'] = len(candidate_names) inspected_media = 0 for media_name in sorted(candidate_names, key=_office_media_sort_key): if len(extracted_images) >= max_images: - break + _record_skip(diagnostics, 'per_document_cap_reached') + continue # Bound the work a malformed archive can cause, not just successful extractions. if inspected_media >= max_images * 10: break @@ -185,6 +247,7 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im base_name = _safe_media_base_name(media_name) if not base_name: + _record_skip(diagnostics, 'unsafe_entry_name') continue image_bytes = _read_zip_entry_bounded( @@ -193,31 +256,49 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im OFFICE_EMBEDDED_IMAGE_MAX_BYTES, ) if image_bytes is None: + _record_skip(diagnostics, 'unreadable_or_oversized') continue # Small assets are almost always icons, bullets, or spacer graphics. if len(image_bytes) < OFFICE_EMBEDDED_IMAGE_MIN_BYTES: + _record_skip(diagnostics, 'below_minimum_bytes') continue digest = hashlib.sha256(image_bytes).hexdigest() if digest in seen_digests: + _record_skip(diagnostics, 'duplicate_image') continue - try: - with Image.open(BytesIO(image_bytes)) as embedded_image: - width, height = embedded_image.size - except Exception: - continue + source_extension = os.path.splitext(base_name)[1].lower() + is_vector = source_extension in OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS + embedded_text = '' + + if is_vector: + rasterized_bytes, width, height, embedded_text, rasterize_reason = _rasterize_vector_image(image_bytes) + if rasterized_bytes is None: + _record_skip(diagnostics, rasterize_reason or 'vector_not_rasterizable') + continue + image_bytes = rasterized_bytes + output_extension = '.png' + else: + if source_extension not in OFFICE_EMBEDDED_IMAGE_EXTENSIONS: + _record_skip(diagnostics, 'unsupported_format') + continue + try: + with Image.open(BytesIO(image_bytes)) as embedded_image: + width, height = embedded_image.size + except Exception: + _record_skip(diagnostics, 'unreadable_image') + continue + output_extension = source_extension if width < min_pixels or height < min_pixels: + _record_skip(diagnostics, 'below_minimum_pixels') continue # The output name is generated rather than taken from the archive, so a crafted # entry name can never influence where the file is written. - extension = os.path.splitext(base_name)[1].lower() - if extension not in OFFICE_EMBEDDED_IMAGE_EXTENSIONS: - continue - output_path = os.path.join(output_dir, f"{len(extracted_images) + 1:03d}{extension}") + output_path = os.path.join(output_dir, f"{len(extracted_images) + 1:03d}{output_extension}") seen_digests.add(digest) @@ -225,16 +306,21 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im with open(output_path, 'wb') as output_file: output_file.write(image_bytes) except OSError: + _record_skip(diagnostics, 'write_failed') continue + diagnostics['analyzed'] += 1 extracted_images.append({ 'name': base_name, 'path': output_path, 'width': width, 'height': height, 'slide_number': media_slide_map.get(media_name), + 'source_format': source_extension.lstrip('.'), + 'rasterized': is_vector, + 'embedded_text': embedded_text, }) except (zipfile.BadZipFile, FileNotFoundError, OSError): - return [] + return [], diagnostics - return extracted_images + return extracted_images, diagnostics diff --git a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md index 63a513bc3..a6412d51a 100644 --- a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md +++ b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md @@ -11,7 +11,7 @@ does not produce. Enhanced extraction always degrades gracefully: when Content Understanding is unavailable or unconfigured, Enhanced automatically uses Document Intelligence `prebuilt-layout` instead. -**Implemented in version: 0.250.221** +**Implemented in version: 0.250.221** (EMF/WMF diagram support added in 0.250.223) **Tracking issue:** [#1277](https://github.com/microsoft/simplechat/issues/1277) @@ -162,8 +162,10 @@ PDFs and images. SimpleChat therefore extracts embedded images from the OOXML pa them separately. - Sources scanned: `word/media/*`, `ppt/media/*`, `xl/media/*`. -- Accepted formats: PNG, JPG/JPEG, BMP, TIF/TIFF, HEIF/HEIC. Vector formats such as EMF and WMF are - skipped because neither engine accepts them. +- Accepted raster formats: PNG, JPG/JPEG, BMP, TIF/TIFF, HEIF/HEIC. +- Accepted vector formats: **EMF and WMF**. Word stores pasted diagrams, SmartArt, Visio drawings, + and charts as metafiles, so these are frequently the most information-dense figures in a + document. Neither analysis engine accepts a metafile, so they are rasterized to PNG first. - Filtering: assets under 2 KB or smaller than `office_embedded_image_min_pixels` in either dimension are skipped as icons, bullets, or spacers. Byte-identical images are analyzed once, so a logo repeated in a header does not multiply cost. `office_embedded_image_max_per_document` caps the @@ -177,6 +179,34 @@ them separately. Embedded image analysis never fails a document. Individual image failures are logged and skipped. +### Metafile rasterization + +`functions_emf_render.py` renders EMF and WMF in-process, on top of Pillow only. This matters +because the application container is Linux distroless — no shell and no package manager — so an +external converter such as LibreOffice or Inkscape is not an option, and Pillow's own metafile +handler is Windows-only because it is backed by GDI. + +The renderer covers the record subset Office actually emits for diagrams: path construction, +filled and stroked polygons, Bezier curves, rectangles and ellipses, pen and brush objects, world +transforms, and text runs. Records outside that subset are skipped rather than failing the render, +so output degrades in fidelity instead of disappearing. It is a description aid for search and +citation, not a pixel-accurate GDI reimplementation. + +Text drawn inside a metafile is also recovered and attached to the chunk, so figure labels such as +service and resource names stay searchable even when the vision engine returns no description. + +### Confirming that embedded images were processed + +Processing reports counts rather than staying silent, because a document whose images were all +skipped otherwise looks identical to a document with no images: + +- `office_embedded_image_candidates` — image parts found in the package +- `office_embedded_image_count` — images successfully analyzed +- `office_embedded_image_skipped` — images skipped + +Status messages name the engine, report progress per image, and list skip reasons, for example +`Analyzed 4 of 6 embedded image(s) with Content Understanding. Skipped: 2 too small.` + ## API Content Understanding reuses the existing admin test-connection endpoint. No new Flask route is @@ -207,6 +237,7 @@ Enhanced extraction is disabled. | --- | --- | | `application/single_app/functions_content_understanding.py` | Content Understanding REST client, page reconstruction, image analysis, connection test | | `application/single_app/functions_office_media.py` | Embedded image extraction from OOXML packages | +| `application/single_app/functions_emf_render.py` | In-process EMF/WMF rasterizer, no system packages required | | `application/single_app/functions_content.py` | `extract_content_with_extraction_engine()` engine dispatch with fallback | | `application/single_app/functions_documents.py` | Ingestion pipeline, Auto-mode detection, embedded image chunks | | `application/single_app/functions_settings.py` | Settings, normalizers, engine resolution | @@ -254,3 +285,8 @@ Enhanced extraction is disabled. - Content Understanding markdown is denser than Document Intelligence Read output because it includes HTML tables and figure descriptions, so chunks are larger. - Legacy `.doc` and `.ppt` files do not support embedded image analysis. +- Metafile rasterization covers the drawing records Office emits for diagrams. Gradients, complex + clipping regions, and embedded bitmap blits inside a metafile are not reproduced, and text is + drawn with a default font rather than the original typeface, so a rasterized diagram is a + faithful-enough likeness rather than an exact reproduction. The text drawn inside the metafile is + extracted separately and attached to the chunk, so labels remain accurate regardless. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index ab015de75..095606733 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,22 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.223)** + +#### Bug Fixes + +* **Diagrams in Word and PowerPoint Files Are Now Analyzed** + * Images embedded in Office documents as EMF or WMF metafiles were silently skipped. Word stores pasted diagrams, SmartArt, Visio drawings, and charts in this format, so architecture diagrams — often the most information-dense figures in a document — were never analyzed or indexed. + * Metafiles are now rasterized in-process and sent to the configured extraction engine like any other image. Text drawn inside the diagram is recovered as well, so figure labels such as service and resource names become searchable even when the vision engine returns no description. + * The renderer is pure Python on top of Pillow, with no system packages or external converters, so it behaves the same in the Linux container as it does locally. Fidelity is intentionally a description aid rather than a pixel-accurate reproduction; unsupported drawing records are skipped rather than failing the document. + * (Ref: #1277, `functions_emf_render.py`, `functions_office_media.py`, embedded Office image analysis) + +* **Embedded Image Processing Is Now Visible in the Workspace Log** + * A document whose images were all skipped looked exactly like a document with no images at all, so there was no way to tell whether embedded image analysis had run. + * Processing now reports how many embedded images were found, how many were analyzed, and why any were skipped — too small, duplicates, unsupported format, or over the per-document cap. Progress is reported per image rather than only once at the start. + * The found, analyzed, and skipped counts are stored on the document so the outcome can be confirmed after processing completes. + * (Ref: #1277, `functions_documents.py`, `functions_office_media.py`, embedded image diagnostics) + ### **(v0.250.222)** #### New Features diff --git a/functional_tests/test_content_understanding_extraction_engine.py b/functional_tests/test_content_understanding_extraction_engine.py index 0c88365c7..d58e4dfd8 100644 --- a/functional_tests/test_content_understanding_extraction_engine.py +++ b/functional_tests/test_content_understanding_extraction_engine.py @@ -2,7 +2,7 @@ # test_content_understanding_extraction_engine.py """ Functional test for Enhanced extraction backed by Azure AI Content Understanding. -Version: 0.250.221 +Version: 0.250.223 Implemented in: 0.250.221 This test ensures that the Content Understanding client parses analyzer results into the same diff --git a/functional_tests/test_office_embedded_image_extraction.py b/functional_tests/test_office_embedded_image_extraction.py index 91e042845..5618eaed7 100644 --- a/functional_tests/test_office_embedded_image_extraction.py +++ b/functional_tests/test_office_embedded_image_extraction.py @@ -2,7 +2,7 @@ # test_office_embedded_image_extraction.py """ Functional test for embedded image extraction from Office files. -Version: 0.250.221 +Version: 0.250.223 Implemented in: 0.250.221 This test ensures that images embedded in DOCX and PPTX packages are pulled out for analysis, @@ -498,6 +498,129 @@ def test_archive_entry_count_is_capped(): return True +def test_emf_metafiles_are_rasterized_and_text_recovered(): + """EMF diagrams must rasterize to PNG and surface their text labels, with no OS dependency.""" + print("Testing EMF rasterization...") + + import functions_office_media + from functions_emf_render import render_metafile_to_png + + # A minimal but valid EMF: header, a filled polygon, and EOF. + emf_bytes = _build_minimal_emf() + + png, width, height, text, reason = render_metafile_to_png(emf_bytes) + if png is None: + raise AssertionError(f"Minimal EMF failed to render: {reason}") + if not png.startswith(b"\x89PNG"): + raise AssertionError("Rasterizer did not emit PNG bytes.") + if width <= 0 or height <= 0: + raise AssertionError(f"Unexpected raster size {width}x{height}") + + if '.emf' not in functions_office_media.OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: + raise AssertionError("EMF must be an accepted embedded image format.") + if '.wmf' not in functions_office_media.OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: + raise AssertionError("WMF must be an accepted embedded image format.") + + print(f"EMF rasterization test passed! ({width}x{height})") + return True + + +def _build_minimal_emf(): + """Build a small valid EMF containing one filled polygon.""" + import struct + + records = [] + + # EMR_POLYGON16: rclBounds(16), cpts(4), points + points = [(10, 10), (200, 10), (200, 150), (10, 150)] + poly_payload = struct.pack('<4i', 10, 10, 200, 150) + poly_payload += struct.pack(' Date: Tue, 18 Aug 2026 13:29:11 -0400 Subject: [PATCH 2/5] Potential fix for pull request finding 'CodeQL / Empty except' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- application/single_app/functions_emf_render.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/application/single_app/functions_emf_render.py b/application/single_app/functions_emf_render.py index 04c657231..ed7d27a08 100644 --- a/application/single_app/functions_emf_render.py +++ b/application/single_app/functions_emf_render.py @@ -885,8 +885,9 @@ def _handle_wmf_text(self, function, params): try: self.draw.text((pixel_x, pixel_y), text, fill=self.state.text_color, font=font) self.records_drawn += 1 - except (ValueError, OSError): - pass + except (ValueError, OSError) as draw_error: + # Best-effort renderer: keep going if a single WMF text run cannot be drawn. + self.skip_reasons.append(f'wmf_text_draw_failed:{type(draw_error).__name__}') def render_metafile_to_png(data, max_pixels=1600): From 871e1c06ad2f5cf1db86983076f1bde826ca6458 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 13:29:19 -0400 Subject: [PATCH 3/5] Potential fix for pull request finding 'CodeQL / Empty except' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- application/single_app/functions_emf_render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/single_app/functions_emf_render.py b/application/single_app/functions_emf_render.py index ed7d27a08..b4e106676 100644 --- a/application/single_app/functions_emf_render.py +++ b/application/single_app/functions_emf_render.py @@ -605,7 +605,8 @@ def _handle_text(self, record_type, payload): self.draw.text((pixel_x, pixel_y), text, fill=self.state.text_color, font=font) self.records_drawn += 1 except (ValueError, OSError): - pass + # Best-effort renderer: ignore text draw failures and continue with remaining records. + return WMF_PLACEABLE_KEY = 0x9AC6CDD7 From 205f9c1c2b44ded0e2458b74de35b84de1f27413 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 13:29:28 -0400 Subject: [PATCH 4/5] Potential fix for pull request finding 'CodeQL / Empty except' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- application/single_app/functions_emf_render.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/single_app/functions_emf_render.py b/application/single_app/functions_emf_render.py index b4e106676..f6d3e8e8b 100644 --- a/application/single_app/functions_emf_render.py +++ b/application/single_app/functions_emf_render.py @@ -233,7 +233,9 @@ def _fill(self, points): self.draw.polygon(pixels, fill=self.state.brush_color) self.records_drawn += 1 except (ValueError, TypeError): - pass + # Best-effort renderer: malformed/self-intersecting geometry can fail in Pillow. + # Skip this fill and continue processing remaining records. + self.fill_errors = getattr(self, 'fill_errors', 0) + 1 def _flush_current_subpath(self): if len(self.current_subpath) >= 2: From dbe9cfa85572827b2e8d2aa99c1b8dbbc6ba3f41 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 13:29:44 -0400 Subject: [PATCH 5/5] Potential fix for pull request finding 'CodeQL / Module is imported with 'import' and 'import from'' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../test_office_embedded_image_extraction.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/functional_tests/test_office_embedded_image_extraction.py b/functional_tests/test_office_embedded_image_extraction.py index 5618eaed7..7b6afa845 100644 --- a/functional_tests/test_office_embedded_image_extraction.py +++ b/functional_tests/test_office_embedded_image_extraction.py @@ -26,7 +26,11 @@ from PIL import Image # noqa: E402 -from functions_office_media import extract_office_embedded_images # noqa: E402 +from functions_office_media import ( # noqa: E402 + OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS, + OFFICE_ZIP_MAX_ENTRIES, + extract_office_embedded_images, +) from test_support.versioning import assert_app_version_at_least # noqa: E402 @@ -476,8 +480,6 @@ def test_archive_entry_count_is_capped(): """An archive with an absurd number of entries is refused outright.""" print("Testing archive entry cap...") - import functions_office_media - with tempfile.TemporaryDirectory() as work_dir: docx_path = os.path.join(work_dir, "many_entries.docx") output_dir = os.path.join(work_dir, "out") @@ -486,7 +488,7 @@ def test_archive_entry_count_is_capped(): with zipfile.ZipFile(docx_path, "w") as archive: archive.writestr("[Content_Types].xml", "") archive.writestr("word/media/image1.png", build_png_bytes(300, 300, (10, 10, 200))) - for index in range(functions_office_media.OFFICE_ZIP_MAX_ENTRIES + 10): + for index in range(OFFICE_ZIP_MAX_ENTRIES + 10): archive.writestr(f"word/junk/{index}.txt", "x") extracted = extract_office_embedded_images(docx_path, output_dir, min_pixels=150, max_images=25) @@ -502,7 +504,6 @@ def test_emf_metafiles_are_rasterized_and_text_recovered(): """EMF diagrams must rasterize to PNG and surface their text labels, with no OS dependency.""" print("Testing EMF rasterization...") - import functions_office_media from functions_emf_render import render_metafile_to_png # A minimal but valid EMF: header, a filled polygon, and EOF. @@ -516,9 +517,9 @@ def test_emf_metafiles_are_rasterized_and_text_recovered(): if width <= 0 or height <= 0: raise AssertionError(f"Unexpected raster size {width}x{height}") - if '.emf' not in functions_office_media.OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: + if '.emf' not in OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: raise AssertionError("EMF must be an accepted embedded image format.") - if '.wmf' not in functions_office_media.OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: + if '.wmf' not in OFFICE_EMBEDDED_IMAGE_VECTOR_EXTENSIONS: raise AssertionError("WMF must be an accepted embedded image format.") print(f"EMF rasterization test passed! ({width}x{height})")