feat(cli): orchestrate raw-less PageIndex Cloud import

Author: KylinMountain

openkb/cli.py

@@ -41,8 +41,10 @@ def filter(self, record: logging.LogRecord) -> bool:
 litellm.suppress_debug_info = True
 from dotenv import load_dotenv
 
+from openkb.agent.compiler import compile_long_doc
 from openkb.config import DEFAULT_CONFIG, load_config, save_config, load_global_config, register_kb
 from openkb.converter import _registry_path, convert_document
+from openkb.indexer import import_cloud_document
 from openkb.locks import atomic_write_json, atomic_write_text, kb_ingest_lock, kb_read_lock
 from openkb.log import append_log
 from openkb.schema import AGENTS_MD, INDEX_SEED, PAGE_CONTENT_DIRS
@@ -386,6 +388,83 @@ def _add_single_file_locked(file_path: Path, kb_dir: Path) -> Literal["added", "
     return "added"
 
 
+def import_from_pageindex_cloud(
+    doc_id: str, kb_dir: Path
+) -> Literal["added", "skipped", "failed"]:
+    """Import an existing PageIndex Cloud document into the KB by ``doc_id``.
+
+    Fetches structure + page content from the cloud (no local PDF), compiles
+    concepts, and registers a raw-less ``pageindex_cloud`` entry. Idempotent:
+    re-importing the same ``doc_id`` is skipped. The user's cloud corpus is
+    never modified.
+    """
+    import hashlib
+    from openkb.state import HashRegistry
+
+    logger = logging.getLogger(__name__)
+    openkb_dir = kb_dir / ".openkb"
+    config = load_config(openkb_dir / "config.yaml")
+    _setup_llm_key(kb_dir)
+    model: str = config.get("model", DEFAULT_CONFIG["model"])
+
+    path_key = f"pageindex-cloud:{doc_id}"
+    synthetic_hash = hashlib.sha256(path_key.encode("utf-8")).hexdigest()
+
+    registry = HashRegistry(openkb_dir / "hashes.json")
+    if registry.is_known(synthetic_hash):
+        click.echo(f"  [SKIP] Already imported from PageIndex Cloud: {doc_id}")
+        return "skipped"
+
+    click.echo(f"Importing from PageIndex Cloud: {doc_id}")
+    try:
+        import_result = import_cloud_document(doc_id, kb_dir, path_key)
+    except Exception as exc:
+        click.echo(f"  [ERROR] Import failed: {exc}")
+        logger.debug("Cloud import traceback:", exc_info=True)
+        return "failed"
+
+    doc_name = import_result.doc_name
+    summary_path = kb_dir / "wiki" / "summaries" / f"{doc_name}.md"
+    click.echo(f"  Compiling imported doc (doc_id={doc_id})...")
+    for attempt in range(2):
+        try:
+            asyncio.run(
+                compile_long_doc(
+                    doc_name, summary_path, doc_id, kb_dir, model,
+                    doc_description=import_result.description,
+                )
+            )
+            break
+        except Exception as exc:
+            if attempt == 0:
+                click.echo("  Retrying compilation in 2s...")
+                time.sleep(2)
+            else:
+                click.echo(f"  [ERROR] Compilation failed: {exc}")
+                logger.debug("Compilation traceback:", exc_info=True)
+                return "failed"
+
+    # Register the raw-less cloud entry only after successful compilation.
+    registry = HashRegistry(openkb_dir / "hashes.json")
+    meta = {
+        "name": import_result.name,
+        "doc_name": doc_name,
+        "type": "pageindex_cloud",
+        "origin": "cloud",
+        "path": path_key,
+        "source_path": _registry_path(
+            kb_dir / "wiki" / "sources" / f"{doc_name}.json", kb_dir
+        ),
+        "doc_id": doc_id,
+    }
+    registry.remove_by_doc_name(doc_name)
+    registry.add(synthetic_hash, meta)
+
+    append_log(kb_dir / "wiki", "ingest", doc_name)
+    click.echo(f"  [OK] {doc_name} imported from PageIndex Cloud.")
+    return "added"
+
+
 # ---------------------------------------------------------------------------
 # CLI
 # ---------------------------------------------------------------------------

tests/test_add_command.py

@@ -203,3 +203,78 @@ def test_add_oldest_legacy_entry_converges_to_single_entry(self, tmp_path):
         new_entries = [m for m in hashes.values() if m.get("doc_name") == "notes"]
         assert len(new_entries) == 1             # …exactly one entry survives
         assert new_entries[0]["path"]            # with path identity persisted
+
+
+class TestImportFromPageindexCloud:
+    def _setup_kb(self, tmp_path):
+        (tmp_path / "raw").mkdir()
+        (tmp_path / "wiki" / "sources" / "images").mkdir(parents=True)
+        (tmp_path / "wiki" / "summaries").mkdir(parents=True)
+        (tmp_path / "wiki" / "concepts").mkdir(parents=True)
+        (tmp_path / "wiki" / "reports").mkdir(parents=True)
+        openkb_dir = tmp_path / ".openkb"
+        openkb_dir.mkdir()
+        (openkb_dir / "config.yaml").write_text("model: gpt-4o-mini\n")
+        (openkb_dir / "hashes.json").write_text(json.dumps({}))
+        return tmp_path
+
+    def test_registers_rawless_cloud_entry(self, tmp_path):
+        import hashlib
+        from openkb.cli import import_from_pageindex_cloud
+        from openkb.indexer import CloudImportResult
+        from openkb.state import HashRegistry
+
+        kb_dir = self._setup_kb(tmp_path)
+        result = CloudImportResult(
+            doc_id="cloud-1", doc_name="Cloud-Paper", name="Cloud Paper.pdf",
+            description="desc",
+        )
+
+        with patch("openkb.cli.import_cloud_document", return_value=result), \
+             patch("openkb.cli.compile_long_doc", return_value=None) as mock_compile, \
+             patch("openkb.cli._setup_llm_key"):
+            outcome = import_from_pageindex_cloud("cloud-1", kb_dir)
+
+        assert outcome == "added"
+        mock_compile.assert_called_once()
+        registry = HashRegistry(kb_dir / ".openkb" / "hashes.json")
+        synthetic = hashlib.sha256(b"pageindex-cloud:cloud-1").hexdigest()
+        meta = registry.get(synthetic)
+        assert meta is not None
+        assert meta["type"] == "pageindex_cloud"
+        assert meta["origin"] == "cloud"
+        assert meta["doc_id"] == "cloud-1"
+        assert meta["path"] == "pageindex-cloud:cloud-1"
+        assert "raw_path" not in meta
+
+    def test_second_import_is_skipped(self, tmp_path):
+        from openkb.cli import import_from_pageindex_cloud
+        from openkb.indexer import CloudImportResult
+
+        kb_dir = self._setup_kb(tmp_path)
+        result = CloudImportResult(
+            doc_id="cloud-1", doc_name="Cloud-Paper", name="Cloud Paper.pdf",
+            description="desc",
+        )
+
+        with patch("openkb.cli.import_cloud_document", return_value=result) as mock_import, \
+             patch("openkb.cli.compile_long_doc", return_value=None), \
+             patch("openkb.cli._setup_llm_key"):
+            import_from_pageindex_cloud("cloud-1", kb_dir)
+            second = import_from_pageindex_cloud("cloud-1", kb_dir)
+
+        assert second == "skipped"
+        assert mock_import.call_count == 1  # not fetched again
+
+    def test_import_failure_returns_failed_and_registers_nothing(self, tmp_path):
+        from openkb.cli import import_from_pageindex_cloud
+        from openkb.state import HashRegistry
+
+        kb_dir = self._setup_kb(tmp_path)
+        with patch("openkb.cli.import_cloud_document", side_effect=RuntimeError("boom")), \
+             patch("openkb.cli._setup_llm_key"):
+            outcome = import_from_pageindex_cloud("cloud-9", kb_dir)
+
+        assert outcome == "failed"
+        registry = HashRegistry(kb_dir / ".openkb" / "hashes.json")
+        assert registry.all_entries() == {}