diff --git a/sphinx/util/inventory.py b/sphinx/util/inventory.py
index 6e4c648bd3f..99fd8780903 100644
--- a/sphinx/util/inventory.py
+++ b/sphinx/util/inventory.py
@@ -191,13 +191,31 @@ def escape(string: str) -> str:
 
             # body
             compressor = zlib.compressobj(9)
+            # Special pages not generated by singlehtml builder
+            singlehtml_excluded_pages = {'genindex', 'search', 'modindex', 'py-modindex'}
             for domain in env.domains.sorted():
                 sorted_objects = sorted(domain.get_objects())
                 for fullname, dispname, type, docname, anchor, prio in sorted_objects:
+                    # Skip pages that singlehtml builder doesn't generate
+                    if builder.name == 'singlehtml' and docname in singlehtml_excluded_pages:
+                        continue
+
                     if anchor.endswith(fullname):
                         # this can shorten the inventory by as much as 25%
                         anchor = anchor.removesuffix(fullname) + '$'
                     uri = builder.get_target_uri(docname)
+
+                    # For singlehtml builder, prepend the output filename to anchor-only URIs
+                    # so intersphinx knows which file to reference
+                    if builder.name == 'singlehtml' and uri.startswith('#'):
+                        if anchor:
+                            # When there's a specific anchor, use just the filename
+                            # (not #document-xxx) to avoid double anchors
+                            uri = builder.config.root_doc + builder.out_suffix
+                        else:
+                            # No specific anchor, keep the #document-xxx part
+                            uri = builder.config.root_doc + builder.out_suffix + uri
+
                     if anchor:
                         uri += '#' + anchor
                     if dispname == fullname: