refactor(config): unify pageindex_max_concurrency + compile_concurrency into concurrency

PageIndex indexing and OpenKB's own concept/entity compilation never run
concurrently with each other for the same document (they're sequential
phases of one add), and both knobs exist for the exact same reason — the
user hit a provider rate limit or an fd ceiling. Splitting them by internal
subsystem leaked OpenKB's architecture into the config surface for no
practical benefit, since a user tuning one for a rate limit would set the
other to the same value anyway.

- Single `concurrency` config key (default null = each stage keeps its own
  built-in default).
- config.resolve_concurrency(config) -> int | None: validates (rejects bool
  and non-positive values with a warning), returns None on unset/invalid —
  no default substitution inside; callers decide what None means for them.
- cli.py's compile call sites: `resolve_concurrency(config) or
  DEFAULT_COMPILE_CONCURRENCY`.
- indexer.py's _build_index_config now routes through resolve_concurrency
  (openkb validates before forwarding to PageIndex, rather than relying
  solely on PageIndex's own future validator) instead of raw config.get.
- This also removes the prior dual-literal-default drift risk entirely:
  there's no second numeric default to keep in sync, since an unset
  `concurrency` simply forwards nothing to PageIndex and lets the compiler
  use its own DEFAULT_COMPILE_CONCURRENCY.
- Updated config.yaml.example and examples/configuration/README.md (kept in
  sync) to the single key.

Author: KylinMountain

config.yaml.example

@@ -2,10 +2,11 @@ model: gpt-5.4                   # LLM model (any LiteLLM-supported provider)
 language: en                     # Wiki output language
 pageindex_threshold: 20          # PDF pages threshold for PageIndex
 
-# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many
-# open files" on large PDFs. Omit either to use the default.
-# pageindex_max_concurrency: 10  # PageIndex indexing concurrency (null = PageIndex's own default)
-# compile_concurrency: 5         # concept/entity page generation concurrency
+# Optional: cap concurrent LLM calls during ingest (PageIndex indexing and
+# concept/entity compilation — they never overlap, so one setting covers
+# both). Lower it if you hit provider rate limits or "too many open files" on
+# large PDFs. Omit to let each stage apply its own default.
+# concurrency: 5
 
 # Optional: override the entity-type vocabulary used for entity pages.
 # Omit this key to use the default 7 types

examples/configuration/README.md

@@ -70,10 +70,11 @@ model: gpt-5.4                   # LLM model (any LiteLLM-supported provider)
 language: en                     # Wiki output language
 pageindex_threshold: 20          # PDF pages threshold for PageIndex
 
-# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many
-# open files" on large PDFs. Omit either to use the default.
-# pageindex_max_concurrency: 10  # PageIndex indexing concurrency (null = PageIndex's own default)
-# compile_concurrency: 5         # concept/entity page generation concurrency
+# Optional: cap concurrent LLM calls during ingest (PageIndex indexing and
+# concept/entity compilation — they never overlap, so one setting covers
+# both). Lower it if you hit provider rate limits or "too many open files" on
+# large PDFs. Omit to let each stage apply its own default.
+# concurrency: 5
 
 # Optional: override the entity-type vocabulary used for entity pages.
 # Omit this key to use the default 7 types
@@ -100,8 +101,7 @@ pageindex_threshold: 20          # PDF pages threshold for PageIndex
 | `model` | `gpt-5.4` | LLM used for all compile/query/chat work. |
 | `language` | `en` | Language the wiki is written in. |
 | `pageindex_threshold` | `20` | PDFs with this many pages **or more** take the long-doc (PageIndex) path; shorter ones go through the short-doc path. See [`pageindex-cloud/`](../pageindex-cloud/). |
-| `pageindex_max_concurrency` | `null` | Caps concurrent indexing LLM calls PageIndex makes for a single long document. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets PageIndex apply its own default. |
-| `compile_concurrency` | `5` | Caps concurrent LLM calls OpenKB makes while generating concept/entity pages. Lower it if your LLM provider rate-limits. |
+| `concurrency` | `null` | Caps concurrent LLM calls OpenKB makes during ingest — both PageIndex's indexing of a long document and OpenKB's own concept/entity compilation. The two never run at once for the same document, so one setting covers both. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets each stage apply its own default. |
 | `entity_types` | 7 defaults | Custom vocabulary for entity pages. `other` is always kept. |
 | `litellm:` | – | A pass-through block for LiteLLM. See below. |
 

openkb/cli.py

@@ -47,14 +47,14 @@ def filter(self, record: logging.LogRecord) -> bool:
 litellm.suppress_debug_info = True
 from dotenv import load_dotenv
 
-from openkb.agent.compiler import compile_long_doc
+from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY, compile_long_doc
 from openkb.config import (
     DEFAULT_CONFIG,
     load_config,
     save_config,
     load_global_config,
     register_kb,
-    resolve_compile_concurrency,
+    resolve_concurrency,
     resolve_extra_headers,
     set_extra_headers,
     resolve_timeout,
@@ -556,7 +556,7 @@ def commit_body(snapshot) -> None:
                     kb_dir,
                     model,
                     doc_description=index_result.description,
-                    max_concurrency=resolve_compile_concurrency(config),
+                    max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY,
                 ),
                 label=f"Compiling long doc (doc_id={index_result.doc_id})",
             )
@@ -570,7 +570,7 @@ def commit_body(snapshot) -> None:
                     source_path,
                     kb_dir,
                     model,
-                    max_concurrency=resolve_compile_concurrency(config),
+                    max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY,
                 ),
                 label="Compiling short doc",
             )
@@ -692,7 +692,7 @@ def commit_body(_snapshot) -> None:
                         kb_dir,
                         model,
                         doc_description=cloud.description,
-                        max_concurrency=resolve_compile_concurrency(config),
+                        max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY,
                     ),
                     label=f"Compiling imported doc (doc_id={doc_id})",
                 )
@@ -1645,7 +1645,7 @@ def _classify(meta: dict) -> str:
     _setup_llm_key(kb_dir)
     config = load_config(openkb_dir / "config.yaml")
     model: str = config.get("model", DEFAULT_CONFIG["model"])
-    max_concurrency = resolve_compile_concurrency(config)
+    max_concurrency = resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY
 
     # Import lazily and reference via the module so tests can patch
     # ``openkb.agent.compiler.compile_*`` and see the call.

openkb/config.py

@@ -17,13 +17,13 @@
     "model": "gpt-5.4",
     "language": "en",
     "pageindex_threshold": 20,
-    # Cap on concurrent indexing LLM calls PageIndex makes for a single long
-    # document. None = let PageIndex apply its own default. Raise it to index
-    # faster, lower it if you hit provider rate limits or "too many open files".
-    "pageindex_max_concurrency": None,
-    # Cap on concurrent compile LLM calls OpenKB makes when generating concept
-    # and entity pages. Lower it if your LLM provider rate-limits.
-    "compile_concurrency": 5,
+    # Cap on concurrent LLM calls OpenKB makes during ingest — both PageIndex's
+    # indexing of a long document and OpenKB's own concept/entity compilation.
+    # These never run concurrently with each other for the same document, so
+    # one knob covers both. None = each stage applies its own built-in default.
+    # Lower it if you hit provider rate limits or "too many open files"; raise
+    # it to go faster.
+    "concurrency": None,
 }
 
 # Default entity-type vocabulary. Overridable per-KB via the optional
@@ -183,24 +183,27 @@ def resolve_timeout(config: dict) -> float | None:
     return value
 
 
-def resolve_compile_concurrency(config: dict) -> int:
-    """Resolve the optional ``compile_concurrency:`` key for the compile step
-    (concept/entity generation).
+def resolve_concurrency(config: dict) -> int | None:
+    """Resolve the optional ``concurrency:`` key — the cap on concurrent LLM
+    calls OpenKB makes during ingest (PageIndex indexing and concept/entity
+    compilation alike; they never run at the same time for one document, so
+    one setting covers both).
 
-    Returns ``DEFAULT_CONFIG["compile_concurrency"]`` when absent, ``None``, or
-    invalid; rejects bools and non-positive values, warning when present but
-    unusable (an explicit ``null`` is the normal "use the default" case and
-    warns silently, matching ``resolve_timeout``).
+    Returns ``None`` when absent, explicitly ``null``, or invalid (rejecting
+    bools and non-positive values with a warning when present but unusable —
+    an explicit ``null``/absent key is the normal "unset" case and stays
+    silent, matching ``resolve_timeout``). Callers apply their own default (or
+    omit the setting entirely) when this returns ``None``.
     """
-    value = config.get("compile_concurrency")
+    value = config.get("concurrency")
     if value is None:
-        return DEFAULT_CONFIG["compile_concurrency"]
+        return None
     if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
         logger.warning(
-            "config: 'compile_concurrency' must be a positive integer, got %r — using default.",
+            "config: 'concurrency' must be a positive integer, got %r — ignoring it.",
             value,
         )
-        return DEFAULT_CONFIG["compile_concurrency"]
+        return None
     return value
 
 

openkb/indexer.py

@@ -11,7 +11,7 @@
 
 from pageindex import IndexConfig, PageIndexClient
 
-from openkb.config import load_config
+from openkb.config import load_config, resolve_concurrency
 from openkb.tree_renderer import render_summary_md
 
 logger = logging.getLogger(__name__)
@@ -156,26 +156,26 @@ def _write_long_doc_artifacts(
 def _build_index_config(config: dict[str, Any]) -> IndexConfig:
     """Build the PageIndex ``IndexConfig`` for local indexing.
 
-    Forwards the optional ``pageindex_max_concurrency`` KB setting to PageIndex,
-    which caps how many indexing LLM calls run at once (guarding against the
-    "too many open files" fd exhaustion on large documents). The value is only
-    passed when set *and* the installed PageIndex's ``IndexConfig`` declares the
-    field, so OpenKB keeps working against a pinned PageIndex that predates it
-    (``IndexConfig`` forbids unknown kwargs).
+    Forwards the KB's ``concurrency`` setting to PageIndex, which caps how many
+    indexing LLM calls run at once (guarding against "too many open files" fd
+    exhaustion on large documents). The value is only passed when set *and* the
+    installed PageIndex's ``IndexConfig`` declares the field, so OpenKB keeps
+    working against a pinned PageIndex that predates it (``IndexConfig``
+    forbids unknown kwargs).
     """
     kwargs: dict[str, Any] = {
         "if_add_node_text": True,
         "if_add_node_summary": True,
         "if_add_doc_description": True,
     }
-    max_concurrency = config.get("pageindex_max_concurrency")
-    if max_concurrency is not None:
+    concurrency = resolve_concurrency(config)
+    if concurrency is not None:
         if "max_concurrency" in IndexConfig.model_fields:
-            kwargs["max_concurrency"] = max_concurrency
+            kwargs["max_concurrency"] = concurrency
         else:
             logger.warning(
-                "config: 'pageindex_max_concurrency' is set but the installed "
-                "PageIndex version does not support it yet — ignoring it."
+                "config: 'concurrency' is set but the installed PageIndex "
+                "version does not support it yet — ignoring it."
             )
     return IndexConfig(**kwargs)
 

tests/test_add_command.py

@@ -97,14 +97,14 @@ def test_add_single_file_compile_failure_rolls_back_converted_artifacts(self, tm
         assert not (kb_dir / "wiki" / "sources" / "notes.md").exists()
         assert HashRegistry(kb_dir / ".openkb" / "hashes.json").all_entries() == {}
 
-    def test_add_forwards_compile_concurrency_from_config(self, tmp_path):
+    def test_add_forwards_concurrency_from_config(self, tmp_path):
         from unittest.mock import AsyncMock
 
         from openkb.cli import add_single_file
 
         kb_dir = self._setup_kb(tmp_path)
         (kb_dir / ".openkb" / "config.yaml").write_text(
-            "model: gpt-4o-mini\ncompile_concurrency: 3\n", encoding="utf-8"
+            "model: gpt-4o-mini\nconcurrency: 3\n", encoding="utf-8"
         )
         doc = tmp_path / "notes.md"
         doc.write_text("# Notes\n\nBody", encoding="utf-8")

tests/test_config.py

@@ -5,7 +5,7 @@
     get_extra_headers,
     get_timeout,
     load_config,
-    resolve_compile_concurrency,
+    resolve_concurrency,
     resolve_extra_headers,
     resolve_litellm_settings,
     resolve_timeout,
@@ -27,79 +27,53 @@ def test_default_config_values():
     assert DEFAULT_CONFIG["pageindex_threshold"] == 20
 
 
-def test_pageindex_max_concurrency_defaults_to_none():
-    assert DEFAULT_CONFIG["pageindex_max_concurrency"] is None
+def test_concurrency_defaults_to_none():
+    assert DEFAULT_CONFIG["concurrency"] is None
 
 
-def test_load_pageindex_max_concurrency_override(tmp_path):
+def test_load_concurrency_override(tmp_path):
     config_path = tmp_path / "config.yaml"
-    config_path.write_text("pageindex_max_concurrency: 12\n", encoding="utf-8")
-    assert load_config(config_path)["pageindex_max_concurrency"] == 12
+    config_path.write_text("concurrency: 12\n", encoding="utf-8")
+    assert load_config(config_path)["concurrency"] == 12
 
 
-def test_compile_concurrency_defaults_to_5():
-    assert DEFAULT_CONFIG["compile_concurrency"] == 5
+def test_resolve_concurrency_absent_is_none():
+    assert resolve_concurrency({}) is None
 
 
-def test_load_compile_concurrency_override(tmp_path):
-    config_path = tmp_path / "config.yaml"
-    config_path.write_text("compile_concurrency: 3\n", encoding="utf-8")
-    assert load_config(config_path)["compile_concurrency"] == 3
-
-
-def test_resolve_compile_concurrency_absent_uses_default():
-    assert resolve_compile_concurrency({}) == DEFAULT_CONFIG["compile_concurrency"]
-
+def test_resolve_concurrency_valid_value():
+    assert resolve_concurrency({"concurrency": 3}) == 3
 
-def test_resolve_compile_concurrency_valid_value():
-    assert resolve_compile_concurrency({"compile_concurrency": 3}) == 3
 
-
-def test_resolve_compile_concurrency_rejects_bool(caplog):
+def test_resolve_concurrency_rejects_bool(caplog):
     with caplog.at_level(logging.WARNING, logger="openkb.config"):
-        result = resolve_compile_concurrency({"compile_concurrency": True})
-    assert result == DEFAULT_CONFIG["compile_concurrency"]
-    assert "compile_concurrency" in caplog.text
+        result = resolve_concurrency({"concurrency": True})
+    assert result is None
+    assert "concurrency" in caplog.text
 
 
-def test_resolve_compile_concurrency_rejects_non_positive(caplog):
+def test_resolve_concurrency_rejects_non_positive(caplog):
     with caplog.at_level(logging.WARNING, logger="openkb.config"):
-        assert (
-            resolve_compile_concurrency({"compile_concurrency": 0})
-            == DEFAULT_CONFIG["compile_concurrency"]
-        )
-    assert "compile_concurrency" in caplog.text
+        assert resolve_concurrency({"concurrency": 0}) is None
+    assert "concurrency" in caplog.text
     caplog.clear()
     with caplog.at_level(logging.WARNING, logger="openkb.config"):
-        assert (
-            resolve_compile_concurrency({"compile_concurrency": -1})
-            == DEFAULT_CONFIG["compile_concurrency"]
-        )
-    assert "compile_concurrency" in caplog.text
+        assert resolve_concurrency({"concurrency": -1}) is None
+    assert "concurrency" in caplog.text
 
 
-def test_resolve_compile_concurrency_rejects_non_int():
-    assert (
-        resolve_compile_concurrency({"compile_concurrency": "3"})
-        == DEFAULT_CONFIG["compile_concurrency"]
-    )
+def test_resolve_concurrency_rejects_non_int():
+    assert resolve_concurrency({"concurrency": "3"}) is None
 
 
-def test_resolve_compile_concurrency_none_is_silent(caplog):
-    # Explicit null / absent is the normal "use the default" case — no warning.
+def test_resolve_concurrency_none_is_silent(caplog):
+    # Explicit null / absent is the normal "unset — caller applies its own
+    # default, or omits the setting entirely" case — no warning.
     with caplog.at_level(logging.WARNING, logger="openkb.config"):
-        resolve_compile_concurrency({"compile_concurrency": None})
+        assert resolve_concurrency({"concurrency": None}) is None
     assert caplog.text == ""
 
 
-def test_compile_concurrency_defaults_match_compiler_default():
-    """DEFAULT_CONFIG and the compiler's own DEFAULT_COMPILE_CONCURRENCY are two
-    independent literals; catch drift immediately if only one is ever updated."""
-    from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY
-
-    assert DEFAULT_CONFIG["compile_concurrency"] == DEFAULT_COMPILE_CONCURRENCY
-
-
 def test_load_missing_file_returns_defaults(tmp_path):
     missing = tmp_path / "nonexistent" / "config.yaml"
     config = load_config(missing)

tests/test_indexer.py

@@ -54,26 +54,33 @@ def test_sets_base_flags(self):
         assert cfg.if_add_node_summary is True
         assert cfg.if_add_doc_description is True
 
-    def test_forwards_max_concurrency_when_supported(self, monkeypatch):
+    def test_forwards_concurrency_when_supported(self, monkeypatch):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
-        cfg = _build_index_config({"pageindex_max_concurrency": 8})
+        cfg = _build_index_config({"concurrency": 8})
         assert cfg.max_concurrency == 8
 
     def test_does_not_forward_when_unsupported(self, monkeypatch):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
-        cfg = _build_index_config({"pageindex_max_concurrency": 8})
+        cfg = _build_index_config({"concurrency": 8})
         assert not hasattr(cfg, "max_concurrency")
 
     def test_none_value_is_left_to_pageindex_default(self, monkeypatch):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
-        cfg = _build_index_config({"pageindex_max_concurrency": None})
+        cfg = _build_index_config({"concurrency": None})
+        assert getattr(cfg, "max_concurrency", None) is None
+
+    def test_invalid_value_is_left_to_pageindex_default(self, monkeypatch):
+        # resolve_concurrency() rejects bools/non-positive values — same as an
+        # unset key, just via the shared config-level validation.
+        monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
+        cfg = _build_index_config({"concurrency": 0})
         assert getattr(cfg, "max_concurrency", None) is None
 
     def test_warns_when_configured_but_unsupported(self, monkeypatch, caplog):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
         with caplog.at_level(logging.WARNING, logger="openkb.indexer"):
-            _build_index_config({"pageindex_max_concurrency": 8})
-        assert "pageindex_max_concurrency" in caplog.text
+            _build_index_config({"concurrency": 8})
+        assert "concurrency" in caplog.text
 
     def test_no_warning_when_unset(self, monkeypatch, caplog):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency)
@@ -84,7 +91,7 @@ def test_no_warning_when_unset(self, monkeypatch, caplog):
     def test_no_warning_when_supported(self, monkeypatch, caplog):
         monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency)
         with caplog.at_level(logging.WARNING, logger="openkb.indexer"):
-            _build_index_config({"pageindex_max_concurrency": 8})
+            _build_index_config({"concurrency": 8})
         assert caplog.text == ""
 
 
@@ -280,12 +287,12 @@ def test_localclient_called_with_index_config(self, kb_dir, sample_tree, tmp_pat
         assert ic.if_add_node_summary is True
         assert ic.if_add_doc_description is True
 
-    def test_pageindex_max_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path):
+    def test_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path):
         """The KB's real config.yaml, loaded by index_long_document itself, must
         reach the IndexConfig passed to PageIndexClient — not just the isolated
         _build_index_config unit tested directly with a hand-built dict."""
         (kb_dir / ".openkb" / "config.yaml").write_text(
-            "model: gpt-4o-mini\npageindex_max_concurrency: 7\n", encoding="utf-8"
+            "model: gpt-4o-mini\nconcurrency: 7\n", encoding="utf-8"
         )
 
         doc_id = "conc-789"