From 22bc6c41ff8e62a1f09fc7b04b12c60a1b4ab773 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 18 Aug 2026 13:50:04 -0400 Subject: [PATCH] Support legacy Office images and optional formula extraction Refs #1277 Two gaps surfaced while answering questions about equation and legacy format coverage. Legacy .doc and .ppt images. Embedded image analysis covered only DOCX and PPTX, because legacy Office files are OLE compound documents rather than zip packages and have no media parts to enumerate. Their pictures and embedded equation previews are still stored as intact metafile blobs, so they are now carved out by signature using the length recorded in the metafile's own header, then rasterized and analyzed like any other embedded image. Validation checks the record type, the signature position, and that the declared length fits the remaining bytes, so a coincidental byte sequence is not mistaken for an image. Duplicate collapsing and the per-document cap still apply, and a file that is neither zip nor OLE yields nothing rather than raising. Optional formula extraction. Equations were never extracted: no formula feature was requested anywhere, so they arrived as whatever OCR guessed. Document Intelligence exposes a FORMULAS add-on, present in the pinned SDK, which returns equations as LaTeX. It is billed per page, so it is behind a new admin toggle that defaults to off, and it is guarded by the layout-mode check because the add-on applies to that model only. The admin UI states plainly that it is a billed add-on. This does not cover equations authored in modern Word, which are stored as OMML markup rather than images. Legacy Equation Editor and MathType objects are stored with a metafile preview and are covered by the carving change above. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- application/single_app/functions_content.py | 6 + application/single_app/functions_documents.py | 7 +- .../single_app/functions_office_media.py | 137 ++++++++++++++++++ application/single_app/functions_settings.py | 11 ++ .../route_frontend_admin_settings.py | 1 + .../static/js/admin/admin_settings.js | 1 + .../single_app/templates/admin_settings.html | 20 +++ ...NTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md | 12 +- docs/explanation/release_notes.md | 17 +++ ...content_understanding_extraction_engine.py | 29 +++- .../test_office_embedded_image_extraction.py | 117 ++++++++++++++- 12 files changed, 350 insertions(+), 10 deletions(-) diff --git a/application/single_app/config.py b/application/single_app/config.py index beb2b9d6a..5393dc20a 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.223" +VERSION = "0.250.224" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index 50051cf16..5caf36c3c 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -10,6 +10,7 @@ from functions_debug import debug_print from config import * +from azure.ai.documentintelligence.models import DocumentAnalysisFeature from functions_office_media import extract_office_embedded_images import functions_settings from functions_settings import * @@ -453,6 +454,11 @@ def extract_content_with_azure_di(file_path, extraction_mode='read', pages=None) analyze_options["output_content_format"] = "markdown" if pages: analyze_options["pages"] = str(pages) + + # Formula extraction is a billed Document Intelligence add-on, so it is opt-in and only + # applies to Layout, which is the model that supports it. + if normalized_extraction_mode == "layout" and functions_settings.is_document_intelligence_formula_extraction_enabled(): + analyze_options["features"] = [DocumentAnalysisFeature.FORMULAS] # Debug logging for troubleshooting debug_print(f"Starting Azure DI extraction for: {os.path.basename(file_path)}") diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index b5c60c333..a4a7fcf92 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -7498,10 +7498,11 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, final_chunks_to_save = di_extracted_pages else: final_chunks_to_save = [] # No text extracted - # --- Embedded Office image analysis (DOCX/DOC/PPTX) --- + # --- Embedded Office image analysis (DOCX/DOC/PPTX/PPT) --- # Neither extraction engine describes figures inside Office files, so embedded images are - # analyzed separately and appended as their own citable chunks. - if (is_word or is_ppt) and not is_legacy_doc and not is_legacy_ppt: + # analyzed separately and appended as their own citable chunks. Legacy binary formats are + # included because their pictures are carved from the OLE container by signature. + if is_word or is_ppt: next_chunk_page_number = max( (int(chunk.get('page_number') or 0) for chunk in final_chunks_to_save), default=0, diff --git a/application/single_app/functions_office_media.py b/application/single_app/functions_office_media.py index 6e2df29dd..72ef5db8d 100644 --- a/application/single_app/functions_office_media.py +++ b/application/single_app/functions_office_media.py @@ -12,6 +12,7 @@ import hashlib import os import re +import struct import zipfile from io import BytesIO @@ -46,6 +47,9 @@ OFFICE_ZIP_MAX_RELS_ENTRIES = 500 # Only the compression methods real OOXML packages use. OFFICE_ZIP_ALLOWED_COMPRESSION = (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) +# Legacy .doc/.ppt are OLE compound documents, so their pictures are carved by signature instead. +OLE_COMPOUND_FILE_SIGNATURE = b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1' +OFFICE_BINARY_SCAN_MAX_BYTES = 256 * 1024 * 1024 _OFFICE_MEDIA_INDEX_PATTERN = re.compile(r'(\d+)') _PPTX_SLIDE_RELS_PATTERN = re.compile(r'^ppt/slides/_rels/slide(\d+)\.xml\.rels$', re.IGNORECASE) @@ -200,6 +204,130 @@ def extract_office_embedded_images(file_path, output_dir, min_pixels=150, max_im return extracted_images +def _carve_metafiles_from_binary(data, max_metafiles): + """Locate EMF and placeable WMF blobs inside a non-zip Office container. + + Legacy ``.doc`` and ``.ppt`` files are OLE compound documents rather than zip packages, so + there is no ``word/media`` part to enumerate. Their pictures and embedded equation previews are + still stored as intact metafile blobs, which can be located by signature and carved using the + length recorded in the metafile's own header. + + Validation is deliberately strict -- record type, signature position, and a length that fits + inside the remaining bytes -- so a coincidental byte sequence is not mistaken for an image. + """ + carved = [] + offset = 0 + length = len(data) + + while offset < length and len(carved) < max_metafiles: + emf_index = data.find(b' EMF', offset) + wmf_index = data.find(b'\xd7\xcd\xc6\x9a', offset) + + candidates = [index for index in (emf_index, wmf_index) if index != -1] + if not candidates: + break + next_index = min(candidates) + + if next_index == emf_index: + record_start = emf_index - 40 + offset = emf_index + 4 + if record_start < 0 or record_start + 88 > length: + continue + record_type, _record_size = struct.unpack_from(' length: + continue + carved.append(('emf', data[record_start:record_start + total_bytes])) + offset = record_start + total_bytes + else: + record_start = wmf_index + offset = wmf_index + 4 + if record_start + 30 > length: + continue + # Placeable header is 22 bytes; the standard header's mtSize is measured in words. + size_words = struct.unpack_from(' length: + continue + carved.append(('wmf', data[record_start:record_start + total_bytes])) + offset = record_start + total_bytes + + return carved + + +def _extract_from_binary_office_file(file_path, output_dir, min_pixels, max_images, diagnostics): + """Extract embedded metafiles from a legacy binary Office document.""" + try: + file_size = os.path.getsize(file_path) + if file_size > OFFICE_BINARY_SCAN_MAX_BYTES: + return [] + with open(file_path, 'rb') as handle: + data = handle.read() + except OSError: + return [] + + if not data.startswith(OLE_COMPOUND_FILE_SIGNATURE): + return [] + + carved = _carve_metafiles_from_binary(data, max_images * 4) + diagnostics['candidates'] = len(carved) + + extracted_images = [] + seen_digests = set() + + for source_format, blob in carved: + if len(extracted_images) >= max_images: + _record_skip(diagnostics, 'per_document_cap_reached') + continue + + if len(blob) < OFFICE_EMBEDDED_IMAGE_MIN_BYTES: + _record_skip(diagnostics, 'below_minimum_bytes') + continue + + digest = hashlib.sha256(blob).hexdigest() + if digest in seen_digests: + _record_skip(diagnostics, 'duplicate_image') + continue + + png_bytes, width, height, embedded_text, reason = _rasterize_vector_image(blob) + if png_bytes is None: + _record_skip(diagnostics, reason or 'vector_not_rasterizable') + continue + + if width < min_pixels or height < min_pixels: + _record_skip(diagnostics, 'below_minimum_pixels') + continue + + seen_digests.add(digest) + output_path = os.path.join(output_dir, f"{len(extracted_images) + 1:03d}.png") + try: + with open(output_path, 'wb') as output_file: + output_file.write(png_bytes) + except OSError: + _record_skip(diagnostics, 'write_failed') + continue + + diagnostics['analyzed'] += 1 + extracted_images.append({ + 'name': f"embedded_{len(extracted_images) + 1}.{source_format}", + 'path': output_path, + 'width': width, + 'height': height, + 'slide_number': None, + 'source_format': source_format, + 'rasterized': True, + 'embedded_text': embedded_text, + }) + + return extracted_images + + def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_pixels=150, max_images=25): """Extract embedded images and report what was skipped and why. @@ -216,6 +344,15 @@ def extract_office_embedded_images_with_diagnostics(file_path, output_dir, min_p if max_images <= 0: return [], diagnostics + # Legacy .doc and .ppt are OLE compound documents, not zip packages. + if not zipfile.is_zipfile(file_path): + try: + return _extract_from_binary_office_file( + file_path, output_dir, min_pixels, max_images, diagnostics + ), diagnostics + except (OSError, ValueError, struct.error): + return [], diagnostics + extracted_images = [] seen_digests = set() diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index c65039416..0f542118e 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -866,6 +866,15 @@ def normalize_extraction_engine(value): return normalized_value +def is_document_intelligence_formula_extraction_enabled(settings=None): + """Return whether the Document Intelligence formula add-on should be requested. + + Formula extraction is a billed add-on, so it defaults to off and must be enabled explicitly. + """ + resolved_settings = settings if settings is not None else get_settings() + return bool((resolved_settings or {}).get('enable_document_intelligence_formula_extraction', False)) + + def normalize_app_role_claims(user_roles): """Normalize app role claims into a flat string list.""" if not user_roles: @@ -1727,6 +1736,8 @@ def get_settings(use_cosmos=False, include_source=False): 'azure_document_intelligence_authentication_type': 'key', 'document_intelligence_pdf_image_extraction_mode': 'read', 'document_intelligence_auto_sample_pages': DOCUMENT_INTELLIGENCE_AUTO_SAMPLE_PAGES_DEFAULT, + # Formula extraction is a billed Document Intelligence add-on, so it stays off by default. + 'enable_document_intelligence_formula_extraction': False, 'enable_document_intelligence_apim': False, 'azure_apim_document_intelligence_endpoint': '', 'azure_apim_document_intelligence_subscription_key': '', diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 0703e81e3..610cea6d0 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -2761,6 +2761,7 @@ def is_valid_url(url): 'azure_document_intelligence_authentication_type': form_data.get('azure_document_intelligence_authentication_type', 'key'), 'document_intelligence_pdf_image_extraction_mode': document_intelligence_pdf_image_extraction_mode, 'document_intelligence_auto_sample_pages': document_intelligence_auto_sample_pages, + 'enable_document_intelligence_formula_extraction': form_data.get('enable_document_intelligence_formula_extraction') == 'on', 'enable_document_intelligence_apim': form_data.get('enable_document_intelligence_apim') == 'on', 'azure_apim_document_intelligence_endpoint': form_data.get('azure_apim_document_intelligence_endpoint', '').strip(), 'azure_apim_document_intelligence_subscription_key': admin_secret('azure_apim_document_intelligence_subscription_key'), diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index aefe16ff1..26726f5f3 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -10552,6 +10552,7 @@ function setupWalkthroughFieldListeners() { {selector: '#azure_content_understanding_analyzer_id', event: 'input'}, {selector: '#azure_content_understanding_image_analyzer_id', event: 'input'}, {selector: '#enable_office_embedded_image_analysis', event: 'change'}, + {selector: '#enable_document_intelligence_formula_extraction', event: 'change'}, {selector: '#office_embedded_image_min_pixels', event: 'input'}, {selector: '#office_embedded_image_max_per_document', event: 'input'} ], diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index e6e68fde8..4f2f14ca7 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -11798,6 +11798,26 @@
Document Intelligence
+
+ + + +
+
+ Captures equations in PDFs and images as LaTeX rather than approximate OCR text. This is a + billed Document Intelligence add-on that adds per-page cost to every Enhanced + extraction, so it is off by default. It applies to the Layout model only, so it has no effect + while extraction is set to Standard. +
+ {% if content_understanding_supported %}
diff --git a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md index a6412d51a..5e02aa4a0 100644 --- a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md +++ b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md @@ -94,6 +94,7 @@ All settings live in **Admin Settings → Extract**. | Enable Enhanced extraction | `enable_enhanced_extraction` | `False` | | PDF and Image Extraction Mode | `document_intelligence_pdf_image_extraction_mode` | `read` | | Auto Sample Pages | `document_intelligence_auto_sample_pages` | `3` | +| Extract mathematical formulas | `enable_document_intelligence_formula_extraction` | `False` | | Content Understanding Endpoint | `azure_content_understanding_endpoint` | `""` | | Authentication Type | `azure_content_understanding_authentication_type` | `key` | | Content Understanding Key | `azure_content_understanding_key` | `""` | @@ -175,7 +176,10 @@ them separately. `ppt/slides/_rels/slideN.xml.rels`. - Each analyzed image becomes its own citable chunk with a heading such as `### Embedded image 2 of 5: image2.png on slide 3`. -- Legacy `.doc` and `.ppt` files are OLE containers rather than zip packages, so they are skipped. +- Legacy `.doc` and `.ppt` files are OLE compound documents rather than zip packages, so their + pictures are carved out by metafile signature instead of being enumerated from media parts. The + carve validates the record type, signature position, and declared length before accepting a blob, + so a coincidental byte sequence is not mistaken for an image. Embedded image analysis never fails a document. Individual image failures are logged and skipped. @@ -284,9 +288,13 @@ Enhanced extraction is disabled. - There is no APIM passthrough option for Content Understanding yet. - Content Understanding markdown is denser than Document Intelligence Read output because it includes HTML tables and figure descriptions, so chunks are larger. -- Legacy `.doc` and `.ppt` files do not support embedded image analysis. - Metafile rasterization covers the drawing records Office emits for diagrams. Gradients, complex clipping regions, and embedded bitmap blits inside a metafile are not reproduced, and text is drawn with a default font rather than the original typeface, so a rasterized diagram is a faithful-enough likeness rather than an exact reproduction. The text drawn inside the metafile is extracted separately and attached to the chunk, so labels remain accurate regardless. +- Equations authored in modern Word are stored as OMML markup rather than images, so they are not + covered by embedded image analysis. Legacy Equation Editor and MathType objects are stored with a + metafile preview and are covered. +- Formula extraction requires the billed Document Intelligence add-on and applies to the Layout + model only, so it has no effect while extraction is set to Standard. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 095606733..51c30f02b 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,23 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.224)** + +#### New Features + +* **Optional Mathematical Formula Extraction** + * Added an **Extract mathematical formulas** toggle to the Document Intelligence settings. When enabled, equations in PDFs and images are captured as LaTeX instead of being approximated as OCR text. + * This requests a **billed Document Intelligence add-on**, so it is off by default and must be turned on deliberately. It applies to the Layout model only, so it has no effect while extraction is set to Standard. + * (Ref: #1277, `functions_content.py`, `functions_settings.py`, `admin_settings.html`, Document Intelligence formulas add-on) + +#### Bug Fixes + +* **Images in Legacy `.doc` and `.ppt` Files Are Now Analyzed** + * Embedded image analysis previously covered only DOCX and PPTX, because legacy Office files are OLE compound documents rather than zip packages and have no media parts to enumerate. + * Pictures and embedded equation previews are now carved out of the legacy container by metafile signature, using the length recorded in the metafile's own header, then rasterized and analyzed like any other embedded image. + * Validation is strict — record type, signature position, and a length that fits the remaining bytes — so a coincidental byte sequence is not mistaken for an image. Duplicate images are still collapsed and the per-document cap still applies. + * (Ref: #1277, `functions_office_media.py`, `functions_documents.py`, legacy Office image extraction) + ### **(v0.250.223)** #### Bug Fixes diff --git a/functional_tests/test_content_understanding_extraction_engine.py b/functional_tests/test_content_understanding_extraction_engine.py index d58e4dfd8..28246e0a4 100644 --- a/functional_tests/test_content_understanding_extraction_engine.py +++ b/functional_tests/test_content_understanding_extraction_engine.py @@ -2,7 +2,7 @@ # test_content_understanding_extraction_engine.py """ Functional test for Enhanced extraction backed by Azure AI Content Understanding. -Version: 0.250.223 +Version: 0.250.224 Implemented in: 0.250.221 This test ensures that the Content Understanding client parses analyzer results into the same @@ -759,6 +759,32 @@ def test_figures_survive_when_the_result_has_no_pages(): return True +def test_formula_extraction_is_opt_in_and_layout_only(): + """The Document Intelligence formula add-on is billed, so it must default off and be opt-in.""" + print("Testing formula extraction opt-in contract...") + + settings = read_repo_file("application/single_app/functions_settings.py") + content = read_repo_file("application/single_app/functions_content.py") + admin_route = read_repo_file("application/single_app/route_frontend_admin_settings.py") + admin_html = read_repo_file("application/single_app/templates/admin_settings.html") + + assert_contains(settings, "'enable_document_intelligence_formula_extraction': False", "formula add-on defaults to off") + assert_contains(settings, "def is_document_intelligence_formula_extraction_enabled", "formula gate helper") + assert_contains(content, "DocumentAnalysisFeature.FORMULAS", "formula add-on requested") + assert_contains(admin_route, "'enable_document_intelligence_formula_extraction': form_data.get(", "formula toggle persisted") + assert_contains(admin_html, 'id="enable_document_intelligence_formula_extraction"', "formula toggle input") + assert_contains(admin_html, "billed Document Intelligence add-on", "cost warning shown to admins") + + # The add-on only applies to Layout, so it must sit behind the layout branch. + formula_index = content.index("DocumentAnalysisFeature.FORMULAS") + guard_index = content.index('if normalized_extraction_mode == "layout" and functions_settings.is_document_intelligence_formula_extraction_enabled()') + if guard_index > formula_index: + raise AssertionError("Formula feature must be guarded by the layout-mode check.") + + print("Formula extraction opt-in test passed!") + return True + + def test_version_is_at_least_implementation_version(): """The app version must be at or beyond the version this feature shipped in.""" print("Testing application version...") @@ -786,6 +812,7 @@ def test_version_is_at_least_implementation_version(): test_settings_and_admin_surface_contract, test_auto_mode_detects_figures, test_enhanced_extraction_upgrade_migration_contract, + test_formula_extraction_is_opt_in_and_layout_only, test_version_is_at_least_implementation_version, ] diff --git a/functional_tests/test_office_embedded_image_extraction.py b/functional_tests/test_office_embedded_image_extraction.py index 7b6afa845..2d37051ec 100644 --- a/functional_tests/test_office_embedded_image_extraction.py +++ b/functional_tests/test_office_embedded_image_extraction.py @@ -2,7 +2,7 @@ # test_office_embedded_image_extraction.py """ Functional test for embedded image extraction from Office files. -Version: 0.250.223 +Version: 0.250.224 Implemented in: 0.250.221 This test ensures that images embedded in DOCX and PPTX packages are pulled out for analysis, @@ -526,8 +526,12 @@ def test_emf_metafiles_are_rasterized_and_text_recovered(): return True -def _build_minimal_emf(): - """Build a small valid EMF containing one filled polygon.""" +def _build_minimal_emf(min_total_bytes=4096): + """Build a small valid EMF containing one filled polygon. + + Padded past the minimum-bytes filter with a comment record, because real diagrams are far + larger than the floor and the filter exists to drop icons and spacers. + """ import struct records = [] @@ -541,6 +545,14 @@ def _build_minimal_emf(): poly_size = 8 + len(poly_payload) records.append(struct.pack('