feat(indexer): expose pageindex_max_concurrency to cap indexing concurrency

PageIndex fans out one indexing LLM call per structure node; on a large
document that can open a socket per node and exhaust the process
file-descriptor limit ("[Errno 24] Too many open files"). This wires an
OpenKB config knob through to PageIndex's IndexConfig.max_concurrency cap.

- New `pageindex_max_concurrency` KB config key (default None = let PageIndex
  apply its own default).
- indexer forwards it via `_build_index_config` only when set AND the installed
  PageIndex's IndexConfig declares the field, so OpenKB keeps working against a
  pinned PageIndex that predates it (IndexConfig forbids unknown kwargs).

Author: KylinMountain

openkb/config.py

@@ -17,6 +17,10 @@
     "model": "gpt-5.4",
     "language": "en",
     "pageindex_threshold": 20,
+    # Cap on concurrent indexing LLM calls PageIndex makes for a single long
+    # document. None = let PageIndex apply its own default. Raise it to index
+    # faster, lower it if you hit provider rate limits or "too many open files".
+    "pageindex_max_concurrency": None,
 }
 
 # Default entity-type vocabulary. Overridable per-KB via the optional

openkb/indexer.py

@@ -153,6 +153,27 @@ def _write_long_doc_artifacts(
     return summary_path
 
 
+def _build_index_config(config: dict[str, Any]) -> IndexConfig:
+    """Build the PageIndex ``IndexConfig`` for local indexing.
+
+    Forwards the optional ``pageindex_max_concurrency`` KB setting to PageIndex,
+    which caps how many indexing LLM calls run at once (guarding against the
+    "too many open files" fd exhaustion on large documents). The value is only
+    passed when set *and* the installed PageIndex's ``IndexConfig`` declares the
+    field, so OpenKB keeps working against a pinned PageIndex that predates it
+    (``IndexConfig`` forbids unknown kwargs).
+    """
+    kwargs: dict[str, Any] = {
+        "if_add_node_text": True,
+        "if_add_node_summary": True,
+        "if_add_doc_description": True,
+    }
+    max_concurrency = config.get("pageindex_max_concurrency")
+    if max_concurrency is not None and "max_concurrency" in IndexConfig.model_fields:
+        kwargs["max_concurrency"] = max_concurrency
+    return IndexConfig(**kwargs)
+
+
 def index_long_document(pdf_path: Path, kb_dir: Path, doc_name: str | None = None) -> IndexResult:
     """Index a long PDF document using PageIndex and write wiki pages.
 
@@ -166,11 +187,7 @@ def index_long_document(pdf_path: Path, kb_dir: Path, doc_name: str | None = Non
     model: str = config.get("model", "gpt-5.4")
     pageindex_api_key = os.environ.get("PAGEINDEX_API_KEY", "")
 
-    index_config = IndexConfig(
-        if_add_node_text=True,
-        if_add_node_summary=True,
-        if_add_doc_description=True,
-    )
+    index_config = _build_index_config(config)
 
     client = PageIndexClient(
         api_key=pageindex_api_key or None,

tests/test_config.py

@@ -26,6 +26,16 @@ def test_default_config_values():
     assert DEFAULT_CONFIG["pageindex_threshold"] == 20
 
 
+def test_pageindex_max_concurrency_defaults_to_none():
+    assert DEFAULT_CONFIG["pageindex_max_concurrency"] is None
+
+
+def test_load_pageindex_max_concurrency_override(tmp_path):
+    config_path = tmp_path / "config.yaml"
+    config_path.write_text("pageindex_max_concurrency: 12\n", encoding="utf-8")
+    assert load_config(config_path)["pageindex_max_concurrency"] == 12
+
+
 def test_load_missing_file_returns_defaults(tmp_path):
     missing = tmp_path / "nonexistent" / "config.yaml"
     config = load_config(missing)

tests/test_indexer.py

@@ -5,8 +5,34 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from pageindex import IndexConfig
+
+from openkb.indexer import (
+    IndexResult,
+    _build_index_config,
+    _normalize_page_content,
+    index_long_document,
+)
+
+
+class TestBuildIndexConfig:
+    def test_sets_base_flags(self):
+        cfg = _build_index_config({})
+        assert cfg.if_add_node_text is True
+        assert cfg.if_add_node_summary is True
+        assert cfg.if_add_doc_description is True
+
+    def test_forwards_max_concurrency_when_supported(self):
+        cfg = _build_index_config({"pageindex_max_concurrency": 8})
+        if "max_concurrency" in IndexConfig.model_fields:
+            assert cfg.max_concurrency == 8
+        else:
+            # A PageIndex predating the field: forwarded as a no-op, never raised.
+            assert not hasattr(cfg, "max_concurrency")
 
-from openkb.indexer import IndexResult, _normalize_page_content, index_long_document
+    def test_none_value_is_left_to_pageindex_default(self):
+        cfg = _build_index_config({"pageindex_max_concurrency": None})
+        assert getattr(cfg, "max_concurrency", None) is None
 
 
 class TestNormalizePageContent: