fix: collision-resistant doc_name — same-stem documents no longer overwrite each other (#96)

* feat(state): index registry entries by path for doc identity

* feat(state): match legacy registry entries by stem for path backfill

* docs(state): document first-match-wins + truthy-path semantics in find_legacy_by_stem

* feat(converter): portable registry path key

* feat(converter): collision-resistant doc_name resolution (Scheme A)

* fix(converter,state): NFKC-normalize both sides of name comparisons + edge tests

* feat(converter): name all artifacts by collision-resistant doc_name

* fix(converter): registry-authoritative names; read-only dedup skip

Unclaimed on-disk artifacts no longer force a suffix (fixes doc_name
drift on retry after a failed compile), and the dedup early-return now
derives the name from the stored entry without invoking the resolver
(no legacy path backfill from duplicate copies).

* feat(indexer): accept explicit doc_name for long-doc artifacts

* feat(cli): persist path identity metadata on add

* fix(cli): construct registry after convert so legacy backfills aren't clobbered

* fix(lint): resolve raw files through the registry before stem matching

* fix(cli): remove locates raw copies via recorded raw_path

* fix(cli): restrict raw-name fallback to legacy entries without raw_path

* credit: collision-resistant doc_name groundwork

The identity model (path-keyed registry metadata, collision-resistant
naming) builds on the approach pioneered in PR #30.

Co-authored-by: Xinyan Zhou <xinyanzhou938@gmail.com>

---------

Co-authored-by: Xinyan Zhou <xinyanzhou938@gmail.com>

Author: KylinMountain

openkb/cli.py

@@ -41,7 +41,7 @@ def filter(self, record: logging.LogRecord) -> bool:
 from dotenv import load_dotenv
 
 from openkb.config import DEFAULT_CONFIG, load_config, save_config, load_global_config, register_kb
-from openkb.converter import convert_document
+from openkb.converter import _registry_path, convert_document
 from openkb.log import append_log
 from openkb.schema import AGENTS_MD, INDEX_SEED, PAGE_CONTENT_DIRS
 
@@ -284,7 +284,6 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
     config = load_config(openkb_dir / "config.yaml")
     _setup_llm_key(kb_dir)
     model: str = config.get("model", DEFAULT_CONFIG["model"])
-    registry = HashRegistry(openkb_dir / "hashes.json")
 
     # 2. Convert document
     click.echo(f"Adding: {file_path.name}")
@@ -299,15 +298,15 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
         click.echo(f"  [SKIP] Already in knowledge base: {file_path.name}")
         return "skipped"
 
-    doc_name = file_path.stem
+    doc_name = result.doc_name or file_path.stem
     index_result = None  # populated only on the long-doc branch
 
     # 3/4. Index and compile
     if result.is_long_doc:
         click.echo(f"  Long document detected — indexing with PageIndex...")
         try:
             from openkb.indexer import index_long_document
-            index_result = index_long_document(result.raw_path, kb_dir)
+            index_result = index_long_document(result.raw_path, kb_dir, doc_name=doc_name)
         except Exception as exc:
             click.echo(f"  [ERROR] Indexing failed: {exc}")
             logger.debug("Indexing traceback:", exc_info=True)
@@ -347,17 +346,31 @@ def add_single_file(file_path: Path, kb_dir: Path) -> Literal["added", "skipped"
 
     # Register hash only after successful compilation
     if result.file_hash:
+        # Construct the registry NOW, not earlier: convert_document may have
+        # backfilled a legacy entry (doc_name/path) on disk via its own
+        # instance, and an earlier snapshot would clobber that backfill on
+        # the full rewrite in add().
+        registry = HashRegistry(openkb_dir / "hashes.json")
         doc_type = "long_pdf" if result.is_long_doc else file_path.suffix.lstrip(".")
         meta = {
             "name": file_path.name,
             "doc_name": doc_name,
             "type": doc_type,
+            "path": _registry_path(file_path, kb_dir),
         }
+        if result.raw_path is not None:
+            meta["raw_path"] = _registry_path(result.raw_path, kb_dir)
+        if result.source_path is not None:
+            meta["source_path"] = _registry_path(result.source_path, kb_dir)
         # For long PDFs we also persist the PageIndex doc_id so `openkb
         # remove` can later call ``Collection.delete_document(doc_id)``
         # to free the managed PDF copy + SQLite row.
         if index_result is not None:
             meta["doc_id"] = index_result.doc_id
+        # An edited document arrives with a new content hash; drop the
+        # stale entry for the same doc_name so the registry keeps exactly
+        # one entry per document.
+        registry.remove_by_doc_name(doc_name)
         registry.add(result.file_hash, meta)
 
     append_log(kb_dir / "wiki", "ingest", file_path.name)
@@ -910,7 +923,15 @@ def remove(ctx, identifier, keep_raw, keep_empty, dry_run, yes):
     raw_path = None
     if not keep_raw:
         raw_dir = kb_dir / "raw"
-        candidate = raw_dir / name
+        # Raw copies are named by doc_name since the collision fix: use the
+        # recorded raw_path when present. Only pre-upgrade entries (no
+        # raw_path field) fall back to the original filename — a recorded
+        # path that no longer exists must NOT fall through, or it could
+        # delete a same-named raw file belonging to another document.
+        if meta.get("raw_path"):
+            candidate = kb_dir / meta["raw_path"]
+        else:
+            candidate = raw_dir / name
         if candidate.exists():
             raw_path = candidate
             actions.append(("DELETE", str(candidate.relative_to(kb_dir))))

openkb/converter.py

@@ -1,8 +1,11 @@
 """Document conversion pipeline for OpenKB."""
 from __future__ import annotations
 
+import hashlib
 import logging
+import re
 import shutil
+import unicodedata
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -25,6 +28,81 @@ class ConvertResult:
     is_long_doc: bool = False
     skipped: bool = False
     file_hash: str | None = None  # For deferred hash registration
+    doc_name: str | None = None  # Stable wiki name (collision-resistant)
+
+
+def _registry_path(path: Path, kb_dir: Path) -> str:
+    """Portable path string used as the registry's identity key.
+
+    Relative-to-KB posix when the file lives inside the KB (stable across
+    machines/checkouts), absolute posix otherwise. Both paths are fully
+    resolved (symlinks followed) before comparison.
+    """
+    resolved_path = path.resolve()
+    resolved_kb = kb_dir.resolve()
+    if resolved_path.is_relative_to(resolved_kb):
+        return resolved_path.relative_to(resolved_kb).as_posix()
+    return resolved_path.as_posix()
+
+
+_SAFE_STEM_RE = re.compile(r"[^\w\-]+")
+_SUFFIX_LEN = 8
+
+
+def _sanitize_stem(stem: str) -> str:
+    normalized = unicodedata.normalize("NFKC", stem)
+    return _SAFE_STEM_RE.sub("-", normalized).strip("-") or "document"
+
+
+def _name_taken(candidate: str, registry: HashRegistry) -> bool:
+    """True when ``candidate`` is claimed by another registered document.
+
+    The registry is the single authority on ownership: artifacts on disk
+    without a registry entry are either leftovers of a failed ingest of
+    this same source (must be adoptable so a retry keeps its clean name)
+    or out-of-contract manual drops — both are overwritten, matching
+    pre-collision-fix behaviour for unclaimed files.
+    """
+    for meta in registry.all_entries().values():
+        entry_name = meta.get("doc_name") or Path(meta.get("name", "")).stem
+        if unicodedata.normalize("NFKC", entry_name) == candidate:
+            return True
+    return False
+
+
+def resolve_doc_name(src: Path, kb_dir: Path, registry: HashRegistry) -> str:
+    """Resolve the stable wiki name for ``src`` (Scheme A).
+
+    Identity is keyed by path: a source we've seen before (same path, even
+    with new content) keeps its name so re-ingest overwrites in place.
+    Legacy registry entries (written before the path index) are matched by
+    stem and backfilled with the path. A brand-new source keeps the clean
+    sanitized stem unless another document already owns that name, in which
+    case it gets a deterministic ``-{sha256(path)[:8]}`` suffix.
+    """
+    path_key = _registry_path(src, kb_dir)
+
+    known = registry.get_by_path(path_key)
+    if known is not None:
+        stored = known.get("doc_name") or Path(known.get("name", "")).stem
+        if stored:
+            return stored
+
+    legacy = registry.find_legacy_by_stem(src.stem)
+    if legacy is not None:
+        file_hash, meta = legacy
+        meta = dict(meta)
+        name = meta.get("doc_name") or Path(meta.get("name", "")).stem
+        meta["doc_name"] = name
+        meta["path"] = path_key
+        registry.add(file_hash, meta)  # backfill + persist
+        return name
+
+    candidate = _sanitize_stem(src.stem)
+    if _name_taken(candidate, registry):
+        digest = hashlib.sha256(path_key.encode("utf-8")).hexdigest()[:_SUFFIX_LEN]
+        return f"{candidate}-{digest}"
+    return candidate
 
 
 def get_pdf_page_count(path: Path) -> int:
@@ -53,20 +131,29 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
     registry = HashRegistry(openkb_dir / "hashes.json")
 
     # ------------------------------------------------------------------
-    # 1. Hash check
+    # 1. Hash check + identity resolution
     # ------------------------------------------------------------------
     file_hash = HashRegistry.hash_file(src)
     if registry.is_known(file_hash):
         logger.info("Skipping already-known file: %s", src.name)
-        return ConvertResult(skipped=True)
+        stored = registry.get(file_hash) or {}
+        return ConvertResult(
+            skipped=True,
+            file_hash=file_hash,
+            doc_name=stored.get("doc_name") or Path(stored.get("name", src.name)).stem,
+        )
+    doc_name = resolve_doc_name(src, kb_dir, registry)
 
     # ------------------------------------------------------------------
     # 2. Copy to raw/
     # ------------------------------------------------------------------
     raw_dir = kb_dir / "raw"
     raw_dir.mkdir(parents=True, exist_ok=True)
-    raw_dest = raw_dir / src.name
-    if raw_dest.resolve() != src.resolve():
+    if src.resolve().is_relative_to(raw_dir.resolve()):
+        # Watch mode: the file already lives in raw/ — don't copy/rename.
+        raw_dest = src
+    else:
+        raw_dest = raw_dir / f"{doc_name}{src.suffix.lower()}"
         shutil.copy2(src, raw_dest)
 
     # ------------------------------------------------------------------
@@ -81,18 +168,21 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
                 threshold,
                 src.name,
             )
-            return ConvertResult(raw_path=raw_dest, is_long_doc=True, file_hash=file_hash)
+            return ConvertResult(
+                raw_path=raw_dest,
+                is_long_doc=True,
+                file_hash=file_hash,
+                doc_name=doc_name,
+            )
 
     # ------------------------------------------------------------------
     # 4/5. Convert to Markdown
     # ------------------------------------------------------------------
     sources_dir = kb_dir / "wiki" / "sources"
     sources_dir.mkdir(parents=True, exist_ok=True)
-    images_dir = kb_dir / "wiki" / "sources" / "images" / src.stem
+    images_dir = kb_dir / "wiki" / "sources" / "images" / doc_name
     images_dir.mkdir(parents=True, exist_ok=True)
 
-    doc_name = src.stem
-
     if src.suffix.lower() == ".md":
         markdown = src.read_text(encoding="utf-8")
         markdown = copy_relative_images(markdown, src.parent, doc_name, images_dir)
@@ -109,4 +199,9 @@ def convert_document(src: Path, kb_dir: Path) -> ConvertResult:
     dest_md = sources_dir / f"{doc_name}.md"
     dest_md.write_text(markdown, encoding="utf-8")
 
-    return ConvertResult(raw_path=raw_dest, source_path=dest_md, file_hash=file_hash)
+    return ConvertResult(
+        raw_path=raw_dest,
+        source_path=dest_md,
+        file_hash=file_hash,
+        doc_name=doc_name,
+    )

openkb/indexer.py

@@ -86,8 +86,15 @@ def _convert_pdf_to_pages(pdf_path: Path, doc_name: str, images_dir: Path) -> li
     return convert_pdf_to_pages(pdf_path, doc_name, images_dir)
 
 
-def index_long_document(pdf_path: Path, kb_dir: Path) -> IndexResult:
-    """Index a long PDF document using PageIndex and write wiki pages."""
+def index_long_document(
+    pdf_path: Path, kb_dir: Path, doc_name: str | None = None
+) -> IndexResult:
+    """Index a long PDF document using PageIndex and write wiki pages.
+
+    ``doc_name`` is the collision-resistant wiki name used for all written
+    artifacts; defaults to the PDF's stem for backward compatibility.
+    """
+    source_name = doc_name or pdf_path.stem
     openkb_dir = kb_dir / ".openkb"
     config = load_config(openkb_dir / "config.yaml")
 
@@ -123,7 +130,7 @@ def index_long_document(pdf_path: Path, kb_dir: Path) -> IndexResult:
 
     # Fetch complete document (metadata + structure + text)
     doc = col.get_document(doc_id, include_text=True)
-    doc_name: str = doc.get("doc_name", pdf_path.stem)
+    indexed_doc_name: str = doc.get("doc_name", pdf_path.stem)
     description: str = doc.get("doc_description", "")
     structure: list = doc.get("structure", [])
 
@@ -132,15 +139,15 @@ def index_long_document(pdf_path: Path, kb_dir: Path) -> IndexResult:
     logger.info("page_count from doc: %s", doc.get("page_count", "NOT PRESENT"))
 
     tree = {
-        "doc_name": doc_name,
+        "doc_name": indexed_doc_name,
         "doc_description": description,
         "structure": structure,
     }
 
     # Write wiki/sources/ — per-page content
     sources_dir = kb_dir / "wiki" / "sources"
     sources_dir.mkdir(parents=True, exist_ok=True)
-    images_dir = sources_dir / "images" / pdf_path.stem
+    images_dir = sources_dir / "images" / source_name
 
     all_pages: list[dict[str, Any]] = []
     if pageindex_api_key:
@@ -155,19 +162,19 @@ def index_long_document(pdf_path: Path, kb_dir: Path) -> IndexResult:
     if not all_pages:
         if pageindex_api_key:
             logger.warning("Cloud returned no pages for %s; falling back to local pymupdf", pdf_path.name)
-        all_pages = _normalize_page_content(_convert_pdf_to_pages(pdf_path, pdf_path.stem, images_dir))
+        all_pages = _normalize_page_content(_convert_pdf_to_pages(pdf_path, source_name, images_dir))
 
     if not all_pages:
         raise RuntimeError(f"No page content extracted for {pdf_path.name}")
 
-    (sources_dir / f"{pdf_path.stem}.json").write_text(
+    (sources_dir / f"{source_name}.json").write_text(
         json_mod.dumps(all_pages, ensure_ascii=False, indent=2), encoding="utf-8",
     )
 
     # Write wiki/summaries/ (no images, just summaries)
     summaries_dir = kb_dir / "wiki" / "summaries"
     summaries_dir.mkdir(parents=True, exist_ok=True)
-    summary_md = render_summary_md(tree, pdf_path.stem, doc_id)
-    (summaries_dir / f"{pdf_path.stem}.md").write_text(summary_md, encoding="utf-8")
+    summary_md = render_summary_md(tree, source_name, doc_id)
+    (summaries_dir / f"{source_name}.md").write_text(summary_md, encoding="utf-8")
 
     return IndexResult(doc_id=doc_id, description=description, tree=tree)

openkb/lint.py

@@ -336,15 +336,25 @@ def find_orphans(wiki: Path) -> list[str]:
     return sorted(orphans)
 
 
-def find_missing_entries(raw: Path, wiki: Path) -> list[str]:
+def find_missing_entries(
+    raw: Path, wiki: Path, *, kb_dir: Path | None = None,
+) -> list[str]:
     """Find files in raw/ that have no corresponding wiki entries.
 
-    A file is considered "present" if it has either a sources/ or summaries/
-    page with the same stem.
+    When ``kb_dir`` is provided, each raw file is first resolved through
+    the hash registry (``kb_dir/.openkb/hashes.json``): registered files
+    are checked against their ``doc_name`` artifacts, which can differ
+    from the raw stem (watch-mode/URL ingests keep the original filename
+    in raw/ — e.g. ``2509.11420.pdf`` → ``2509-11420.md`` — and collision
+    suffixes rename artifacts too). Files with no registry entry fall back
+    to the stem heuristic: "present" means a sources/ or summaries/ page
+    with the same stem.
 
     Args:
         raw: Path to the raw documents directory.
         wiki: Path to the wiki root directory.
+        kb_dir: Root of the knowledge base. When None, only the legacy
+            stem heuristic is used.
 
     Returns:
         List of filenames in raw/ with no wiki entry.
@@ -356,10 +366,39 @@ def find_missing_entries(raw: Path, wiki: Path) -> list[str]:
     summary_stems = {p.stem for p in summaries_dir.glob("*.md")} if summaries_dir.exists() else set()
     known_stems = sources_stems | summary_stems
 
+    registry = None
+    if kb_dir is not None:
+        registry_file = kb_dir / ".openkb" / "hashes.json"
+        if registry_file.exists():
+            # Deferred import: converter pulls in pymupdf/markitdown,
+            # which lint doesn't otherwise need at import time.
+            from openkb.converter import _registry_path
+            from openkb.state import HashRegistry
+
+            registry = HashRegistry(registry_file)
+
     missing: list[str] = []
     if raw.exists():
         for f in raw.iterdir():
-            if f.is_file() and f.stem not in known_stems:
+            if not f.is_file():
+                continue
+            if registry is not None:
+                meta = registry.get_by_path(_registry_path(f, kb_dir))
+                if meta is not None:
+                    # Registered file — the registry's doc_name is the
+                    # single source of truth for artifact names.
+                    doc_name = meta.get("doc_name") or Path(
+                        meta.get("name", f.name)
+                    ).stem
+                    present = (
+                        (sources_dir / f"{doc_name}.md").exists()
+                        or (sources_dir / f"{doc_name}.json").exists()
+                        or (summaries_dir / f"{doc_name}.md").exists()
+                    )
+                    if not present:
+                        missing.append(f.name)
+                    continue
+            if f.stem not in known_stems:
                 missing.append(f.name)
 
     return sorted(missing)
@@ -460,7 +499,7 @@ def run_structural_lint(kb_dir: Path) -> str:
 
     broken = find_broken_links(wiki)
     orphans = find_orphans(wiki)
-    missing = find_missing_entries(raw, wiki)
+    missing = find_missing_entries(raw, wiki, kb_dir=kb_dir)
     sync_issues = check_index_sync(wiki)
     bad_frontmatter = find_invalid_frontmatter(wiki)