fix(config): address xhigh code-review findings on the concurrency knobs

- resolve_compile_concurrency moves to config.py (matching resolve_timeout's
  convention) and now rejects bool (bool is an int subclass, so
  `compile_concurrency: true` previously silently became concurrency=1) and
  logs a warning on any malformed value, same as the other resolvers.
- _build_index_config now warns when pageindex_max_concurrency is configured
  but the installed PageIndex doesn't support the field yet, instead of
  silently dropping it with no signal to the user.
- Replaced the tautological test_forwards_max_concurrency_when_supported
  (which branched on the same runtime condition as the code under test, so it
  never exercised the forwarding assertion under CI's pinned PageIndex) with
  fake IndexConfig doubles that make both branches deterministic regardless
  of the installed pageindex version.
- Added a cross-check test pinning DEFAULT_CONFIG["compile_concurrency"]
  against the compiler's own DEFAULT_COMPILE_CONCURRENCY so the two literals
  can't silently drift apart.
- Added an end-to-end test proving index_long_document's own loaded config
  (not just a hand-built dict) reaches PageIndexClient's IndexConfig.
- Hoisted the compile-concurrency resolution out of `recompile --all`'s
  per-document loop (was recomputed every iteration despite being
  loop-invariant).
- Documented both new config.yaml keys in config.yaml.example and
  examples/configuration/README.md (kept in sync).

Author: KylinMountain

config.yaml.example

@@ -2,6 +2,11 @@ model: gpt-5.4                   # LLM model (any LiteLLM-supported provider)
 language: en                     # Wiki output language
 pageindex_threshold: 20          # PDF pages threshold for PageIndex
 
+# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many
+# open files" on large PDFs. Omit either to use the default.
+# pageindex_max_concurrency: 10  # PageIndex indexing concurrency (null = PageIndex's own default)
+# compile_concurrency: 5         # concept/entity page generation concurrency
+
 # Optional: override the entity-type vocabulary used for entity pages.
 # Omit this key to use the default 7 types
 # (person, organization, place, product, work, event, other).

examples/configuration/README.md

@@ -70,6 +70,11 @@ model: gpt-5.4                   # LLM model (any LiteLLM-supported provider)
 language: en                     # Wiki output language
 pageindex_threshold: 20          # PDF pages threshold for PageIndex
 
+# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many
+# open files" on large PDFs. Omit either to use the default.
+# pageindex_max_concurrency: 10  # PageIndex indexing concurrency (null = PageIndex's own default)
+# compile_concurrency: 5         # concept/entity page generation concurrency
+
 # Optional: override the entity-type vocabulary used for entity pages.
 # Omit this key to use the default 7 types
 # (person, organization, place, product, work, event, other).
@@ -95,6 +100,8 @@ pageindex_threshold: 20          # PDF pages threshold for PageIndex
 | `model` | `gpt-5.4` | LLM used for all compile/query/chat work. |
 | `language` | `en` | Language the wiki is written in. |
 | `pageindex_threshold` | `20` | PDFs with this many pages **or more** take the long-doc (PageIndex) path; shorter ones go through the short-doc path. See [`pageindex-cloud/`](../pageindex-cloud/). |
+| `pageindex_max_concurrency` | `null` | Caps concurrent indexing LLM calls PageIndex makes for a single long document. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets PageIndex apply its own default. |
+| `compile_concurrency` | `5` | Caps concurrent LLM calls OpenKB makes while generating concept/entity pages. Lower it if your LLM provider rate-limits. |
 | `entity_types` | 7 defaults | Custom vocabulary for entity pages. `other` is always kept. |
 | `litellm:` | – | A pass-through block for LiteLLM. See below. |
 

openkb/cli.py

@@ -47,13 +47,14 @@ def filter(self, record: logging.LogRecord) -> bool:
 litellm.suppress_debug_info = True
 from dotenv import load_dotenv
 
-from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY, compile_long_doc
+from openkb.agent.compiler import compile_long_doc
 from openkb.config import (
     DEFAULT_CONFIG,
     load_config,
     save_config,
     load_global_config,
     register_kb,
+    resolve_compile_concurrency,
     resolve_extra_headers,
     set_extra_headers,
     resolve_timeout,
@@ -439,19 +440,6 @@ def _run_compile_with_retry(coro_factory, label: str) -> None:
                 raise
 
 
-def _compile_concurrency(config: dict) -> int:
-    """Concurrency cap for the compile step (concept/entity generation).
-
-    Configurable per KB via ``compile_concurrency`` in config.yaml; a missing,
-    null, or non-positive value falls back to the compiler's built-in default.
-    Lower it when the LLM provider rate-limits.
-    """
-    value = config.get("compile_concurrency")
-    if isinstance(value, int) and value > 0:
-        return value
-    return DEFAULT_COMPILE_CONCURRENCY
-
-
 def add_single_file(
     file_path: Path, kb_dir: Path, *, stage: bool = True
 ) -> Literal["added", "skipped", "failed"]:
@@ -577,7 +565,7 @@ def _add_single_file_locked(
                     kb_dir,
                     model,
                     doc_description=index_result.description,
-                    max_concurrency=_compile_concurrency(config),
+                    max_concurrency=resolve_compile_concurrency(config),
                 ),
                 label=f"Compiling long doc (doc_id={index_result.doc_id})",
             )
@@ -591,7 +579,7 @@ def _add_single_file_locked(
                     source_path,
                     kb_dir,
                     model,
-                    max_concurrency=_compile_concurrency(config),
+                    max_concurrency=resolve_compile_concurrency(config),
                 ),
                 label="Compiling short doc",
             )
@@ -719,7 +707,7 @@ def import_from_pageindex_cloud(doc_id: str, kb_dir: Path) -> Literal["added", "
                 kb_dir,
                 model,
                 doc_description=cloud.description,
-                max_concurrency=_compile_concurrency(config),
+                max_concurrency=resolve_compile_concurrency(config),
             ),
             label=f"Compiling imported doc (doc_id={doc_id})",
         )
@@ -1665,6 +1653,7 @@ def _classify(meta: dict) -> str:
     _setup_llm_key(kb_dir)
     config = load_config(openkb_dir / "config.yaml")
     model: str = config.get("model", DEFAULT_CONFIG["model"])
+    max_concurrency = resolve_compile_concurrency(config)
 
     # Import lazily and reference via the module so tests can patch
     # ``openkb.agent.compiler.compile_*`` and see the call.
@@ -1707,7 +1696,7 @@ def _classify(meta: dict) -> str:
                         doc_id,
                         kb_dir,
                         model,
-                        max_concurrency=_compile_concurrency(config),
+                        max_concurrency=max_concurrency,
                     )
                 )
             except Exception as exc:
@@ -1735,7 +1724,7 @@ def _classify(meta: dict) -> str:
                         source_path,
                         kb_dir,
                         model,
-                        max_concurrency=_compile_concurrency(config),
+                        max_concurrency=max_concurrency,
                     )
                 )
             except Exception as exc:

openkb/config.py

@@ -183,6 +183,27 @@ def resolve_timeout(config: dict) -> float | None:
     return value
 
 
+def resolve_compile_concurrency(config: dict) -> int:
+    """Resolve the optional ``compile_concurrency:`` key for the compile step
+    (concept/entity generation).
+
+    Returns ``DEFAULT_CONFIG["compile_concurrency"]`` when absent, ``None``, or
+    invalid; rejects bools and non-positive values, warning when present but
+    unusable (an explicit ``null`` is the normal "use the default" case and
+    warns silently, matching ``resolve_timeout``).
+    """
+    value = config.get("compile_concurrency")
+    if value is None:
+        return DEFAULT_CONFIG["compile_concurrency"]
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        logger.warning(
+            "config: 'compile_concurrency' must be a positive integer, got %r — using default.",
+            value,
+        )
+        return DEFAULT_CONFIG["compile_concurrency"]
+    return value
+
+
 def resolve_litellm_settings(config: dict) -> dict[str, Any]:
     """Resolve the optional ``litellm:`` mapping of LiteLLM module settings.
 

openkb/indexer.py

@@ -169,8 +169,14 @@ def _build_index_config(config: dict[str, Any]) -> IndexConfig:
         "if_add_doc_description": True,
     }
     max_concurrency = config.get("pageindex_max_concurrency")
-    if max_concurrency is not None and "max_concurrency" in IndexConfig.model_fields:
-        kwargs["max_concurrency"] = max_concurrency
+    if max_concurrency is not None:
+        if "max_concurrency" in IndexConfig.model_fields:
+            kwargs["max_concurrency"] = max_concurrency
+        else:
+            logger.warning(
+                "config: 'pageindex_max_concurrency' is set but the installed "
+                "PageIndex version does not support it yet — ignoring it."
+            )
     return IndexConfig(**kwargs)
 
 

tests/test_add_command.py

@@ -97,16 +97,6 @@ def test_add_single_file_compile_failure_rolls_back_converted_artifacts(self, tm
         assert not (kb_dir / "wiki" / "sources" / "notes.md").exists()
         assert HashRegistry(kb_dir / ".openkb" / "hashes.json").all_entries() == {}
 
-    def test_compile_concurrency_resolution(self):
-        from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY
-        from openkb.cli import _compile_concurrency
-
-        assert _compile_concurrency({}) == DEFAULT_COMPILE_CONCURRENCY
-        assert _compile_concurrency({"compile_concurrency": 3}) == 3
-        # None / non-positive / non-int fall back to the compiler default.
-        assert _compile_concurrency({"compile_concurrency": None}) == DEFAULT_COMPILE_CONCURRENCY
-        assert _compile_concurrency({"compile_concurrency": 0}) == DEFAULT_COMPILE_CONCURRENCY
-
     def test_add_forwards_compile_concurrency_from_config(self, tmp_path):
         from unittest.mock import AsyncMock
 

tests/test_config.py

@@ -5,6 +5,7 @@
     get_extra_headers,
     get_timeout,
     load_config,
+    resolve_compile_concurrency,
     resolve_extra_headers,
     resolve_litellm_settings,
     resolve_timeout,
@@ -46,6 +47,59 @@ def test_load_compile_concurrency_override(tmp_path):
     assert load_config(config_path)["compile_concurrency"] == 3
 
 
+def test_resolve_compile_concurrency_absent_uses_default():
+    assert resolve_compile_concurrency({}) == DEFAULT_CONFIG["compile_concurrency"]
+
+
+def test_resolve_compile_concurrency_valid_value():
+    assert resolve_compile_concurrency({"compile_concurrency": 3}) == 3
+
+
+def test_resolve_compile_concurrency_rejects_bool(caplog):
+    with caplog.at_level(logging.WARNING, logger="openkb.config"):
+        result = resolve_compile_concurrency({"compile_concurrency": True})
+    assert result == DEFAULT_CONFIG["compile_concurrency"]
+    assert "compile_concurrency" in caplog.text
+
+
+def test_resolve_compile_concurrency_rejects_non_positive(caplog):
+    with caplog.at_level(logging.WARNING, logger="openkb.config"):
+        assert (
+            resolve_compile_concurrency({"compile_concurrency": 0})
+            == DEFAULT_CONFIG["compile_concurrency"]
+        )
+    assert "compile_concurrency" in caplog.text
+    caplog.clear()
+    with caplog.at_level(logging.WARNING, logger="openkb.config"):
+        assert (
+            resolve_compile_concurrency({"compile_concurrency": -1})
+            == DEFAULT_CONFIG["compile_concurrency"]
+        )
+    assert "compile_concurrency" in caplog.text
+
+
+def test_resolve_compile_concurrency_rejects_non_int():
+    assert (
+        resolve_compile_concurrency({"compile_concurrency": "3"})
+        == DEFAULT_CONFIG["compile_concurrency"]
+    )
+
+
+def test_resolve_compile_concurrency_none_is_silent(caplog):
+    # Explicit null / absent is the normal "use the default" case — no warning.
+    with caplog.at_level(logging.WARNING, logger="openkb.config"):
+        resolve_compile_concurrency({"compile_concurrency": None})
+    assert caplog.text == ""
+
+
+def test_compile_concurrency_defaults_match_compiler_default():
+    """DEFAULT_CONFIG and the compiler's own DEFAULT_COMPILE_CONCURRENCY are two
+    independent literals; catch drift immediately if only one is ever updated."""
+    from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY
+
+    assert DEFAULT_CONFIG["compile_concurrency"] == DEFAULT_COMPILE_CONCURRENCY
+
+
 def test_load_missing_file_returns_defaults(tmp_path):
     missing = tmp_path / "nonexistent" / "config.yaml"
     config = load_config(missing)

tests/test_indexer.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
+import logging
 from unittest.mock import MagicMock, patch
 
 import pytest
-from pageindex import IndexConfig
 
 from openkb.indexer import (
     IndexResult,
@@ -15,25 +15,78 @@
 )
 
 
+class _FakeIndexConfigWithConcurrency:
+    """Stand-in for a PageIndex ``IndexConfig`` that declares ``max_concurrency``.
+
+    Used instead of relying on whatever ``pageindex`` happens to be installed in
+    this environment, so the forwarding tests are deterministic regardless of
+    the currently-pinned PageIndex version (see ``test_forwards_...`` below).
+    """
+
+    model_fields = {
+        "if_add_node_text": None,
+        "if_add_node_summary": None,
+        "if_add_doc_description": None,
+        "max_concurrency": None,
+    }
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+class _FakeIndexConfigWithoutConcurrency:
+    """Stand-in for a PageIndex ``IndexConfig`` predating ``max_concurrency``."""
+
+    model_fields = {
+        "if_add_node_text": None,
+        "if_add_node_summary": None,
+        "if_add_doc_description": None,
+    }
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
 class TestBuildIndexConfig:
     def test_sets_base_flags(self):
         cfg = _build_index_config({})
         assert cfg.if_add_node_text is True
         assert cfg.if_add_node_summary is True
         assert cfg.if_add_doc_description is True
 
-    def test_forwards_max_concurrency_when_supported(self):
+    def test_forwards_max_concurrency_when_supported(self, monkeypatch):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
         cfg = _build_index_config({"pageindex_max_concurrency": 8})
-        if "max_concurrency" in IndexConfig.model_fields:
-            assert cfg.max_concurrency == 8
-        else:
-            # A PageIndex predating the field: forwarded as a no-op, never raised.
-            assert not hasattr(cfg, "max_concurrency")
+        assert cfg.max_concurrency == 8
 
-    def test_none_value_is_left_to_pageindex_default(self):
+    def test_does_not_forward_when_unsupported(self, monkeypatch):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
+        cfg = _build_index_config({"pageindex_max_concurrency": 8})
+        assert not hasattr(cfg, "max_concurrency")
+
+    def test_none_value_is_left_to_pageindex_default(self, monkeypatch):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
         cfg = _build_index_config({"pageindex_max_concurrency": None})
         assert getattr(cfg, "max_concurrency", None) is None
 
+    def test_warns_when_configured_but_unsupported(self, monkeypatch, caplog):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
+        with caplog.at_level(logging.WARNING, logger="openkb.indexer"):
+            _build_index_config({"pageindex_max_concurrency": 8})
+        assert "pageindex_max_concurrency" in caplog.text
+
+    def test_no_warning_when_unset(self, monkeypatch, caplog):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
+        with caplog.at_level(logging.WARNING, logger="openkb.indexer"):
+            _build_index_config({})
+        assert caplog.text == ""
+
+    def test_no_warning_when_supported(self, monkeypatch, caplog):
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
+        with caplog.at_level(logging.WARNING, logger="openkb.indexer"):
+            _build_index_config({"pageindex_max_concurrency": 8})
+        assert caplog.text == ""
+
 
 class TestNormalizePageContent:
     def test_normalizes_pageindex_dicts(self):
@@ -204,6 +257,32 @@ def test_localclient_called_with_index_config(self, kb_dir, sample_tree, tmp_pat
         assert ic.if_add_node_summary is True
         assert ic.if_add_doc_description is True
 
+    def test_pageindex_max_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path):
+        """The KB's real config.yaml, loaded by index_long_document itself, must
+        reach the IndexConfig passed to PageIndexClient — not just the isolated
+        _build_index_config unit tested directly with a hand-built dict."""
+        (kb_dir / ".openkb" / "config.yaml").write_text(
+            "model: gpt-4o-mini\npageindex_max_concurrency: 7\n", encoding="utf-8"
+        )
+
+        doc_id = "conc-789"
+        fake_col = self._make_fake_collection(doc_id, sample_tree)
+        fake_client = MagicMock()
+        fake_client.collection.return_value = fake_col
+
+        pdf_path = tmp_path / "report.pdf"
+        pdf_path.write_bytes(b"%PDF-1.4 fake")
+
+        with (
+            patch("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency),
+            patch("openkb.indexer.PageIndexClient", return_value=fake_client) as mock_cls,
+            patch("openkb.images.convert_pdf_to_pages", return_value=self._fake_pages()),
+        ):
+            index_long_document(pdf_path, kb_dir)
+
+        _, kwargs = mock_cls.call_args
+        assert kwargs["index_config"].max_concurrency == 7
+
     def test_cloud_page_content_is_normalized(self, kb_dir, sample_tree, tmp_path, monkeypatch):
         doc_id = "cloud-123"
         fake_col = self._make_fake_collection(doc_id, sample_tree)