From e225c708f34b36c22a754eb912ff4bb738d89b76 Mon Sep 17 00:00:00 2001 From: Dipesh Mittal Date: Tue, 11 Aug 2026 08:27:08 +0530 Subject: [PATCH 1/2] feat(ui): serve the explorer as web pages instead of Streamlit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Streamlit was capping how good this could look and how easily it could be changed. Every visual decision went through framework selectors, the map went through a wrapper that could not be styled, and nothing had a URL. The rendering layer is replaced; the view-model is not. `view.py` was already free of Streamlit and fully tested, so it carries over untouched and stays the contract — only `app.py`'s 642 lines of widget calls were thrown away. What this buys beyond appearance: every view has a URL //schema, //explore?q=…&t=…, //entity/, //map?focus=. Tabs, filters, an entity and a focused map are all linkable, the back button works, and a demo can be sent as a link. Under Streamlit all of this was session state. the index is a route / is a path parameter rather than something parsed out of a URL the browser reported over a websocket. The bug where every path served whichever brain sorted first is now unrepresentable. no websocket pages are HTML and render on first byte; the map's data is fetched separately, so a large graph cannot stop the page from appearing. a real 404 an unknown index or a deleted entity says so with a status code, rather than a 200 that reads as empty. The endpoint on the help tab now comes from the request and honours X-Forwarded-Proto, so a TLS-terminating proxy no longer advertises an http:// endpoint for an https:// page. Design is a token sheet — light and dark, no build step. Tailwind would have meant a Node toolchain in a pip-installable package, which is the property that made server-rendering attractive in the first place. The map is Cytoscape, vendored so the package stays installable and usable offline. Deployment is unchanged on purpose: same `open-index ui`, same port, same OPEN_INDEX_BRAINS_ROOT, so containers and proxy config carry over as-is. Drops the streamlit and streamlit-agraph dependencies. 513 tests pass; view.py at 100%, web.py at 96%. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/tests.yml | 8 +- README.md | 6 +- docker-compose.yml | 2 - docs/quickstart.mdx | 4 +- docs/reference/cli.mdx | 2 +- open_index/cli.py | 33 +- open_index/storage/sqlite_backend.py | 7 +- open_index/ui/app.py | 642 ------------------- open_index/ui/static/app.css | 333 ++++++++++ open_index/ui/static/app.js | 47 ++ open_index/ui/static/map.js | 152 +++++ open_index/ui/static/vendor/cytoscape.min.js | 32 + open_index/ui/templates/analytics.html | 115 ++++ open_index/ui/templates/base.html | 83 +++ open_index/ui/templates/directory.html | 37 ++ open_index/ui/templates/entity.html | 82 +++ open_index/ui/templates/explore.html | 101 +++ open_index/ui/templates/help.html | 127 ++++ open_index/ui/templates/jobs.html | 39 ++ open_index/ui/templates/map.html | 71 ++ open_index/ui/templates/missing.html | 25 + open_index/ui/templates/schema.html | 86 +++ open_index/ui/view.py | 160 +---- open_index/ui/web.py | 426 ++++++++++++ pyproject.toml | 20 +- tests/test_cli.py | 40 +- tests/test_ui_app.py | 325 ---------- tests/test_ui_view.py | 149 ----- tests/test_ui_web.py | 270 ++++++++ 29 files changed, 2102 insertions(+), 1322 deletions(-) delete mode 100644 open_index/ui/app.py create mode 100644 open_index/ui/static/app.css create mode 100644 open_index/ui/static/app.js create mode 100644 open_index/ui/static/map.js create mode 100644 open_index/ui/static/vendor/cytoscape.min.js create mode 100644 open_index/ui/templates/analytics.html create mode 100644 open_index/ui/templates/base.html create mode 100644 open_index/ui/templates/directory.html create mode 100644 open_index/ui/templates/entity.html create mode 100644 open_index/ui/templates/explore.html create mode 100644 open_index/ui/templates/help.html create mode 100644 open_index/ui/templates/jobs.html create mode 100644 open_index/ui/templates/map.html create mode 100644 open_index/ui/templates/missing.html create mode 100644 open_index/ui/templates/schema.html create mode 100644 open_index/ui/web.py delete mode 100644 tests/test_ui_app.py create mode 100644 tests/test_ui_web.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a5b87e7..b3bdaef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,10 +40,10 @@ jobs: run: pip install -e '.[ui,mcp,opensearch,serve,dev]' numpy - name: Check the optional deps are importable - # Four test modules call pytest.importorskip("mcp") / ("streamlit"). If - # the install above ever breaks, those ~60 tests would skip and the job - # would still go green — so fail here instead. - run: python -c "import mcp, streamlit, streamlit_agraph, opensearchpy, numpy" + # Several test modules call pytest.importorskip(...). If the install + # above ever breaks, those tests would skip and the job would still go + # green — so fail here instead. + run: python -c "import mcp, starlette, jinja2, opensearchpy, numpy" - name: Run tests # -rs lists skip reasons, so a suite that stops running is visible in diff --git a/README.md b/README.md index 94ea41f..b212bee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A brain is built from four primitives: ## Quickstart ```bash -pip install -e '.[all]' # core + UI (Streamlit) + MCP server +pip install -e '.[all]' # core + explorer UI + MCP server # Try the bundled example (support brain: products, issues, segments, comments) open-index index --brain examples/support-brain @@ -47,7 +47,7 @@ Prefer containers, or need a brain several agents share? → | `open-index ingest ` | Run a connector now to pull entities from an MCP server. | | `open-index run [--force] [--loop N]` | Run every connector whose `schedule` is due (wire into cron/CI). | | `open-index search [-t doc_type]` | Search from the terminal. | -| `open-index ui` | Launch the Streamlit explorer (Explore / **Map** / Analytics / Jobs). | +| `open-index ui` | Launch the explorer (How to use / Schema / Explore / **Map** / Analytics / Jobs). | | `open-index mcp [--read-only]` | Run the MCP context layer over stdio. **Read+write by default**; `--read-only` opts out of writes. | | `open-index serve [--port --token --read-only]` | Serve the MCP context layer over **HTTP** for remote agents (bearer-token auth). | | `open-index serve --brains ` | Serve **every** brain under a directory from one process, each at `//mcp`. | @@ -374,7 +374,7 @@ fastest way to get an answer and the best place to sanity-check a bigger change. ```bash git clone https://github.com/DrDroidLab/open-index cd open-index -pip install -e '.[all]' # core + UI (Streamlit) + MCP server +pip install -e '.[all]' # core + explorer UI + MCP server pytest # run the test suite ``` diff --git a/docker-compose.yml b/docker-compose.yml index 5ae6a88..db80563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -134,8 +134,6 @@ services: <<: *brain-env OPEN_INDEX_SEARCH_BACKEND: ${OPEN_INDEX_SEARCH_BACKEND:-sqlite} OPEN_INDEX_OPENSEARCH_HOSTS: ${OPEN_INDEX_OPENSEARCH_HOSTS:-http://opensearch:9200} - STREAMLIT_SERVER_HEADLESS: "true" - STREAMLIT_SERVER_ADDRESS: 0.0.0.0 ports: - "${UI_PORT:-8501}:8501" diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 9a8d3a4..af72c90 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -6,10 +6,10 @@ description: Install Open Index, run the bundled example brain, and open the exp ## Install ```bash -pip install -e '.[all]' # core + UI (Streamlit) + MCP server +pip install -e '.[all]' # core + explorer UI + MCP server ``` -The `[all]` extra pulls in the Streamlit explorer and the MCP server. For a +The `[all]` extra pulls in the explorer and the MCP server. For a narrower install, pick the extras you need — `[ui]`, `[mcp]`, `[serve]`, `[semantic]`, `[opensearch]`. diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index d110faf..1c924bb 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -36,7 +36,7 @@ changing dimensions. | Command | What it does | |---|---| | `open-index search [-t doc_type]` | Search from the terminal. | -| `open-index ui` | Launch the Streamlit explorer (Explore / **Map** / Analytics / Jobs). | +| `open-index ui` | Launch the explorer (How to use / Schema / Explore / **Map** / Analytics / Jobs). | ## Serving over MCP diff --git a/open_index/cli.py b/open_index/cli.py index 60fba11..fcf5c88 100644 --- a/open_index/cli.py +++ b/open_index/cli.py @@ -6,7 +6,7 @@ open-index index (re)load entities/ into the search index open-index ingest run a connector to pull entities from MCP open-index search search from the terminal - open-index ui launch the Streamlit map explorer + open-index ui launch the explorer in a browser open-index mcp run the MCP server (stdio) `--brain ` selects the brain directory (default: current directory). @@ -15,7 +15,6 @@ from __future__ import annotations import json -import sys from pathlib import Path from typing import Optional @@ -357,25 +356,29 @@ def list_connectors(brain: str = BrainOpt): @app.command() def ui( brain: str = BrainOpt, - port: int = typer.Option(8501, help="Streamlit port."), + host: str = typer.Option("0.0.0.0", help="Address to bind."), + port: int = typer.Option(8501, help="Port to serve the explorer on."), ): - """Launch the Streamlit map explorer.""" + """Launch the explorer: schema, search, and the relationship map.""" import os - import subprocess - - app_path = Path(__file__).parent / "ui" / "app.py" - env = dict(os.environ, OPEN_INDEX_DIR=str(Path(brain).resolve())) - cmd = [ - sys.executable, "-m", "streamlit", "run", str(app_path), - "--server.port", str(port), - ] + + os.environ.setdefault("OPEN_INDEX_DIR", str(Path(brain).resolve())) + try: - subprocess.run(cmd, env=env, check=True) - except FileNotFoundError: - typer.secho("Streamlit not installed: pip install 'open-index[ui]'", + from open_index.ui.web import serve as serve_ui + except ImportError: + typer.secho("The explorer needs its extras: pip install 'open-index[ui]'", fg=typer.colors.RED, err=True) raise typer.Exit(1) + root = os.environ.get("OPEN_INDEX_BRAINS_ROOT") + where = f"every brain under {root}" if root else os.environ["OPEN_INDEX_DIR"] + typer.echo(f"open-index explorer · {where}") + # The bind address is not necessarily reachable, so print a URL that is. + shown = "127.0.0.1" if host in ("0.0.0.0", "::") else host + typer.echo(f" http://{shown}:{port}") + serve_ui(host=host, port=port) + @app.command() def mcp( diff --git a/open_index/storage/sqlite_backend.py b/open_index/storage/sqlite_backend.py index 65b1671..814823e 100644 --- a/open_index/storage/sqlite_backend.py +++ b/open_index/storage/sqlite_backend.py @@ -58,9 +58,10 @@ class SQLiteBackend: def __init__(self, db_path: str | Path, config=None): self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) - # check_same_thread=False: the Streamlit UI caches one Brain (and thus one - # connection) but reruns it across worker threads. Access is serialized - # through _lock so the shared connection stays safe. + # check_same_thread=False: the explorer caches one Brain (and so one + # connection) while Starlette runs its sync endpoints on a threadpool, so + # the connection is reached from several threads. Access is serialized + # through _lock, which is what actually keeps it safe. self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False) self._conn.row_factory = sqlite3.Row self._conn.execute("PRAGMA foreign_keys = ON") diff --git a/open_index/ui/app.py b/open_index/ui/app.py deleted file mode 100644 index 8c859ba..0000000 --- a/open_index/ui/app.py +++ /dev/null @@ -1,642 +0,0 @@ -"""Streamlit explorer for a brain. - -Launched via `open-index ui`; the brain directory arrives in OPEN_INDEX_DIR. - -The UI is for **inspecting** a brain, not editing one — writes go through the -agent (MCP) or the CLI so every change is validated and lands in the source of -truth. Accordingly there are three tabs, and the two questions a newcomer -actually has are answered without clicking anything: - - "what's in here?" the sidebar always lists every doc_type with its count - "show me the map" the map auto-anchors on the most-connected entities - -Both used to require finding the right tab and then making a selection before -anything appeared, which read as an empty or broken screen. -""" - -from __future__ import annotations - -import os -from typing import Optional -from time import perf_counter - -import streamlit as st - -from open_index.brain import Brain -from open_index.graph import ContextGraph, build_graph, build_overview_graph -from open_index.ui import view - -st.set_page_config(page_title="Open Index", page_icon="🧠", layout="wide") - - -@st.cache_resource -def _open_brain(brain_dir: str) -> Brain: - return Brain.open(brain_dir) - - -@st.cache_resource -def _available_brains(root: str) -> dict: - """Brains under a root, cached so every rerun does not re-stat the disk.""" - from open_index.config import discover_brains - - return {name: str(path) for name, path in discover_brains(root).items()} - - -def _current_url(): - """The URL the browser is on, as reported over the websocket. - - Absent on older Streamlit versions and in bare script runs, so callers must - tolerate None. - """ - try: - return st.context.url - except Exception: - return None - - -def _select_brain(): - """The brain this page shows, chosen solely by the URL path. - - With OPEN_INDEX_BRAINS_ROOT set, one process serves every brain under it — - the alternative is a process, and a ~250MB embedding model, per brain. The - index is the first path segment, so / serves . There is no - in-page switcher on purpose: the URL is the selector, which keeps the - address bar and the screen in agreement and makes every view shareable. - """ - root = os.environ.get("OPEN_INDEX_BRAINS_ROOT") - if not root: - return _open_brain(os.environ.get("OPEN_INDEX_DIR", ".")), None - - brains = _available_brains(root) - if not brains: - st.error(f"No brains found under `{_esc(root)}`.") - st.caption("A brain is a directory containing `brain.yaml`. Create one " - "with `open-index init `.") - st.stop() - - chosen = view.brain_from_url(_current_url(), list(brains)) - # The directory name is also the URL segment, so it is what builds this - # index's endpoint — the brain's configured name may differ from it. - return _open_brain(brains[chosen]), chosen - - -def _theme_type() -> str: - """Whether the viewer is in light or dark mode. - - Streamlit reports this per-session, so it follows the browser preference - even when the server was never configured with a theme. Older versions - don't expose it at all, hence the guarded lookup. - """ - try: - return getattr(st.context.theme, "type", None) or "light" - except Exception: - return "light" - - -def _dot(color: str) -> str: - """An inline colored dot matching a doc_type's map color.""" - return (f"") - - -def _esc(text) -> str: - return (str(text).replace("&", "&").replace("<", "<").replace(">", ">")) - - -# --------------------------------------------------------------------------- # -# Sidebar — the brain at a glance. Always visible, on every tab. -# --------------------------------------------------------------------------- # - -def render_sidebar(brain: Brain) -> dict: - """Identity, structure and settings. Returns the chosen search options.""" - summary = view.summarize(brain) - - with st.sidebar: - st.markdown(f"### 🧠 {_esc(summary.name)}") - if summary.description: - st.caption(summary.description) - st.caption(f"**{summary.total_entities:,}** entities · " - f"**{len(summary.doc_types)}** doc_types") - - st.divider() - st.markdown("**Doc types**") - if not summary.has_schema: - st.caption("None yet — define one with `open-index add-doc-type`, " - "or ask your agent.") - for row in summary.doc_types: - st.markdown( - f"{_dot(row.color)} `{_esc(row.name)}`   " - f"{row.count:,} · {row.storage}", - unsafe_allow_html=True, - ) - - st.divider() - with st.expander("Search settings"): - mode = st.radio( - "Mode", list(view.SEARCH_MODES), horizontal=True, - help=("Hybrid: keyword matches dominate, semantic similarity " - "rescues differently-worded queries. Keyword: text only. " - "Semantic: embedding similarity only."), - ) - if mode == "Semantic": - from open_index.embeddings import embedding_provider_available - - if not embedding_provider_available(): - st.warning("No embedding provider — results stay keyword-only. " - "Install `open-index[semantic]`.") - - with st.expander("Connect an agent"): - st.caption("Editing happens through your agent or the CLI, never here — " - "so every write is validated.") - st.code(f"open-index mcp-config --brain {os.environ.get('OPEN_INDEX_DIR', '.')}", - language="bash") - st.caption("Paste the output into `.mcp.json`, then ask the agent to " - "read `navigation_guidelines` and add what you need.") - - return {"semantic_weight": view.semantic_weight_for(mode)} - - -# --------------------------------------------------------------------------- # -# Explore — search and browse in one place (they were duplicate screens). -# --------------------------------------------------------------------------- # - -def _goto(entity_id) -> None: - st.session_state["open_entity"] = entity_id - st.rerun() - - -def render_explore(brain: Brain, options: dict) -> None: - summary = view.summarize(brain) - - if summary.is_empty: - st.info("This brain has no entities yet.") - st.caption("Add some with `open-index import `, a connector, or by " - "asking your agent — then run `open-index index`.") - return - - query = st.text_input( - "Search", label_visibility="collapsed", - placeholder="Search the brain… e.g. payment, checkout, redis", - ) - type_filter = st.multiselect( - "Limit to doc_types", [r.name for r in summary.doc_types], - label_visibility="collapsed", placeholder="All doc types", - ) - - if st.session_state.get("open_entity"): - render_entity(brain, st.session_state["open_entity"]) - return - - if query: - _render_results(brain, query, type_filter or None, options) - else: - _render_browse(brain, summary, type_filter or None) - - -def _render_results(brain: Brain, query: str, doc_types, options: dict) -> None: - try: - # source="ui" so browsing here shows up in the Analytics tab alongside - # CLI and agent traffic — otherwise the usage picture has a hole in it. - results = brain.search(query=query, doc_types=doc_types, limit=50, - semantic_weight=options["semantic_weight"], - source="ui") - except Exception as exc: # a backend that's down shouldn't blank the page - st.error(f"Search failed: {exc}") - return - - st.caption(f"{results.total} result(s) — click one to open") - if not results.results: - st.info("No matches. Try fewer words, or switch to Semantic mode in the sidebar.") - return - for r in results.results: - color = view.color_for(brain, r["doc_type"]) - if st.button(f"{r['name']} · {r['doc_type']} · {r['id']}", - key=f"res_{r['id']}"): - _goto(r["id"]) - - -def _render_browse(brain: Brain, summary: view.BrainSummary, doc_types) -> None: - """No query: list entities by type so the brain is never a blank page.""" - shown = [r for r in summary.doc_types - if r.count and (not doc_types or r.name in doc_types)] - if not shown: - st.info("No entities in the selected doc_types.") - return - - for row in shown: - with st.expander(f"{row.name} · {row.count:,}", - expanded=len(shown) == 1): - if row.description: - st.caption(row.description) - for entity in brain.backend.all_entities([row.name])[:200]: - description = entity.fields.get("description", "") - label = entity.name + (f" · {description[:70]}" if description else "") - if st.button(label, key=f"br_{entity.id}"): - _goto(entity.id) - if row.count > 200: - st.caption(f"showing the first 200 of {row.count:,} — search to narrow.") - - -def render_entity(brain: Brain, entity_id: str) -> None: - """One entity: its fields, its attribution, and both directions of its edges.""" - if st.button("← back", key="entity_back"): - _goto(None) - - entity = brain.get_entity(entity_id, source="ui") - if entity is None: - st.warning(f"No entity `{entity_id}`.") - return - - st.markdown( - f"{_dot(view.color_for(brain, entity.doc_type))} **{_esc(entity.name)}** " - f" `{_esc(entity.id)}`", - unsafe_allow_html=True, - ) - - rows = view.field_rows(entity) - if rows: - st.table(rows) - - provenance = view.provenance_row(entity) - if provenance: - with st.expander("Provenance"): - st.table([provenance]) - if entity.valid_from or entity.valid_to: - st.caption(f"valid: {entity.valid_from or '—'} → {entity.valid_to or 'now'}") - - links = view.neighbours(brain, entity_id) - st.markdown(f"**Relationships** ({len(links)})") - if not links: - st.caption("None yet. Edges are what make this a graph — ask your agent to " - "link this entity to related ones.") - return - for i, link in enumerate(links): - suffix = "" if link.exists else " · (missing)" - if st.button(f"{link.label} · {link.other_id}{suffix}", - key=f"nb_{i}_{link.other_id}"): - _goto(link.other_id) - - -# --------------------------------------------------------------------------- # -# Map — draws immediately, no selection required. -# --------------------------------------------------------------------------- # - -def render_map(brain: Brain) -> None: - """The whole index at a glance: every entity, coloured by doc_type. - - Anchoring on one entity is the wrong default for someone who has never seen - the index — they have no entity in mind. This shows the shape of the whole - thing and lets them subtract from it. - """ - summary = view.summarize(brain) - populated = [r.name for r in summary.doc_types if r.count] - if not populated: - st.info("No entities to map yet. Add some and run `open-index index`.") - return - - chosen = st.multiselect( - "Doc types shown", populated, default=populated, - help="Uncheck a type to drop it and its edges from the map.", - ) - scope = chosen or populated - - focus = st.session_state.get("map_focus") - if focus: - cols = st.columns([5, 1]) - cols[0].caption(f"Focused on `{focus}` and its immediate neighbours.") - if cols[1].button("↩ show all"): - st.session_state["map_focus"] = None - st.rerun() - graph = build_graph(brain, [focus], depth=1) - else: - graph = build_overview_graph(brain, scope, limit=view.MAX_GRAPH_NODES) - - total = sum(r.count for r in summary.doc_types if r.name in scope) - if not focus and len(graph.nodes) < total: - st.warning( - f"Showing the {len(graph.nodes)} most-connected of {total} entities. " - "Narrow the doc types above to see the rest." - ) - - canvas, legend = st.columns([4, 1]) - with canvas: - st.caption(f"{len(graph.nodes)} nodes · {len(graph.edges)} edges — " - "hover for detail, click a node to focus on it") - render_graph(brain, graph) - with legend: - st.markdown("**Legend**") - for row in view.legend_rows(brain, graph): - st.markdown( - f"{_dot(row['color'])} `{_esc(row['doc_type'])}`" - f" {row['count']}", - unsafe_allow_html=True, - ) - if graph.edges: - st.markdown("---") - st.caption("Edges are `related_to` links. Hover one to see which " - "relationship it is.") - - # streamlit-agraph doesn't paint on its first render inside a tab (the canvas - # mounts with zero size). One forced rerun remounts it with the tab active. - if not st.session_state.get("_map_primed"): - st.session_state["_map_primed"] = True - st.rerun() - - -def render_graph(brain: Brain, graph: ContextGraph) -> None: - try: - from streamlit_agraph import Config, Edge, Node, agraph - except ImportError: - st.warning("Install the map renderer: pip install 'open-index[ui]'") - st.write({"nodes": [n.__dict__ for n in graph.nodes], - "edges": [e.__dict__ for e in graph.edges]}) - return - - palette = view.graph_theme(_theme_type()) - - nodes = [Node(**spec) for spec in view.graph_node_specs(graph)] - edges = [Edge(**spec) for spec in view.graph_edge_specs(graph, palette["edge"])] - - busy = len(graph.nodes) > view.BUSY_GRAPH_NODES - config = Config( - # An int, not "100%": the library formats this as f"{width}px", so a CSS - # string yields "100%px" and the canvas never sizes — which is what left - # the graph stranded in a corner instead of centred. - width=view.GRAPH_WIDTH, height=view.GRAPH_HEIGHT, directed=True, - # Physics stays on even for busy graphs: vis only fits the viewport to - # the content as part of stabilisation, so disabling it means the map - # never centres. Slow big graphs down instead of freezing them. - physics=True, stabilization=True, fit=True, - maxVelocity=15 if busy else 50, - hierarchical=False, collapsible=False, - ) - # The canvas is a fixed pixel width, so on a wide page it would sit flush - # left. Centre it in the content area rather than letting it hug the edge. - _left, middle, _right = st.columns([1, 20, 1]) - with middle: - clicked = agraph(nodes=nodes, edges=edges, config=config) - if clicked and clicked != st.session_state.get("map_focus"): - st.session_state["map_focus"] = clicked - st.rerun() - - -# --------------------------------------------------------------------------- # -# Jobs — connectors and their schedules. -# --------------------------------------------------------------------------- # - -def render_analytics(brain: Brain) -> None: - """Which context was fetched, by whom, and how often — across CLI, MCP and UI. - - The number worth watching is zero-result searches: those are the questions - this brain was asked and could not answer, i.e. what to model next. - """ - summary = brain.analytics_summary() - if not summary.get("available", True): - st.warning("Analytics are unavailable — the local state directory is not writable.") - return - - st.caption( - "Stored in `~/.local/state/open-index/`, outside the brain checkout. " - "Search text and entity ids stay on this machine." - ) - - cols = st.columns(4) - cols[0].metric("fetches", summary["total_fetches"]) - cols[1].metric("failed", summary["failed_fetches"]) - cols[2].metric("zero-result searches", summary["zero_result_searches"]) - cols[3].metric("avg latency", f"{summary['average_duration_ms']:.0f} ms") - - if not summary["total_fetches"]: - st.info("Nothing recorded yet. Search here, or query the brain from the " - "CLI or your agent, and the usage shows up in this tab.") - return - - left, right = st.columns(2) - with left: - st.markdown("**By client**") - st.bar_chart([{"client": k, "fetches": v} - for k, v in summary["by_source"].items()], - x="client", y="fetches") - with right: - st.markdown("**By operation**") - st.bar_chart([{"operation": k, "fetches": v} - for k, v in summary["by_operation"].items()], - x="operation", y="fetches") - - st.markdown("**Most requested context**") - if summary["by_context"]: - st.dataframe([{"context": c, "fetches": n} - for c, n in summary["by_context"].items()], - hide_index=True, use_container_width=True) - else: - st.caption("No named context fetched yet.") - - with st.expander("Recent fetches"): - events = brain.analytics_events(limit=100) - if not events: - st.caption("No events yet.") - return - st.dataframe([{ - "time": e["fetched_at"], - "client": e["source"], - "operation": e["operation"], - "context": e["query"] or e["entity_id"] or "navigation guide", - "results": e["result_count"], - "ms": e["duration_ms"], - "ok": bool(e["success"]), - } for e in events], hide_index=True, use_container_width=True) - - -def render_schema(brain: Brain) -> None: - """Every doc_type and the shape of it — the reference before you write.""" - summary = view.summarize(brain) - if not summary.has_schema: - st.info("No doc_types defined yet.") - st.caption("A doc_type is a concept this index tracks, plus the fields it " - "stores. Create one with `open-index add-doc-type`, or ask an " - "agent to call `create_doc_type`.") - return - - st.caption( - f"{len(summary.doc_types)} doc_types · {summary.total_entities:,} entities. " - "Every entity id is `:`, and any entity can link to any " - "other through `related_to`." - ) - - for row in summary.doc_types: - doc_type = brain.config.doc_type(row.name) - noun = "entity" if row.count == 1 else "entities" - with st.expander(f"{row.name} · {row.count:,} {noun}", - expanded=len(summary.doc_types) <= 3): - if row.description: - st.markdown(row.description) - st.caption( - f"{_dot(row.color)} source of truth: " - + ("**files** — JSON under `entities/`, git-trackable" - if row.storage == "file" - else "**search index** — DB-owned, not written to files"), - unsafe_allow_html=True, - ) - - fields = view.schema_field_rows(doc_type) - if fields: - st.markdown("**Fields**") - st.table(fields) - else: - st.caption("No fields declared.") - - relationships = view.schema_relationship_rows(brain, row.name) - st.markdown("**Relationships (optional)**") - if relationships: - st.table(relationships) - st.caption("Declared edges are validated against their target " - "doc_type. Undeclared ones still work — they just " - "aren't checked.") - else: - st.caption("None declared or in use. Entities of this type are " - "valid without any; edges are what make the index " - "traversable rather than just searchable.") - - -def render_how_to_use(brain: Brain, url_name: Optional[str] = None) -> None: - """Connecting an agent, the tools it gets, and what the other tabs are for. - - Deliberately the rightmost tab: it is the page people come back to, not the - one they start on. - """ - summary = view.summarize(brain) - # Derived from the URL this page is on, so it is right for *this* index and - # needs no configuration. Falls back to an explicitly configured URL for a - # single-brain deployment, where the page path carries no index name. - mcp_url = (view.mcp_url_for(_current_url(), url_name) - or os.environ.get("OPEN_INDEX_PUBLIC_URL", "")) - read_only = os.environ.get("OPEN_INDEX_READ_ONLY", "").lower() in ("1", "true", "yes") - - st.markdown("### What an index holds") - st.caption( - "Three ideas, and the whole thing follows from them." - ) - for term, what in view.MODEL_GUIDE: - st.markdown(f"- **{term}** — {what}") - - st.divider() - st.markdown(f"### Connect an agent to `{_esc(summary.name)}`") - st.caption( - "This index speaks MCP, so any MCP-capable agent can query it — and, " - "unless the endpoint is read-only, keep it current." - ) - - if mcp_url: - st.markdown("**1. Point your agent at this URL**") - st.code(view.mcp_client_config(mcp_url, server_name=summary.name), language="json") - st.caption("Paste into `.mcp.json` (Claude Code), `.cursor/mcp.json` (Cursor), " - "or any MCP client's server config. Or generate it:") - st.code(f"open-index mcp-config --url {mcp_url} --name {summary.name} > .mcp.json", - language="bash") - else: - st.info("This explorer isn't configured with a public MCP URL " - "(`OPEN_INDEX_PUBLIC_URL`), so the connection block can't be shown.") - st.code("open-index mcp-config --brain > .mcp.json", language="bash") - - st.markdown("**2. Ask it something**") - st.caption("No briefing needed — the navigation guide below is injected into the " - "MCP handshake, so the agent knows this index's doc_types and " - "relationship vocabulary before its first turn.") - - st.divider() - st.markdown("### Tools the agent gets") - - st.markdown("**Reading**") - for name, what in view.READ_TOOLS: - st.markdown(f"- `{name}` — {what}") - - st.markdown("**Writing**") - if read_only: - st.info("This endpoint is **read-only** — the write tools below are not " - "registered on it. Serve without `--read-only` to enable them.") - for name, what in view.WRITE_TOOLS: - st.markdown(f"- `{name}` — {what}") - st.caption("Entity ids are always `:`. Writes are validated " - "against the doc_type schema, and land in the search index (and on " - "disk, for `storage: file` types).") - - st.divider() - st.markdown("### What each tab does") - for name, what in view.TAB_GUIDE: - st.markdown(f"- **{name}** — {what}") - - st.divider() - st.markdown("### What's in this index right now") - st.caption(f"{summary.total_entities:,} entities across " - f"{len(summary.doc_types)} doc_types.") - if summary.doc_types: - st.table([{"doc_type": r.name, "entities": r.count, - "source of truth": "files (git)" if r.storage == "file" else "search index", - "what it holds": r.description or "—"} - for r in summary.doc_types]) - - with st.expander("The full navigation guide the agent receives"): - st.code(brain.navigation_guidelines(include_writes=not read_only), - language="markdown") - - -def render_jobs(brain: Brain) -> None: - from open_index.connectors.runner import discover_connectors - from open_index.scheduling import RunState - - st.caption("Ingestion scripts in `connectors/*.py` that pull entities from an " - "MCP server on a schedule.") - found = discover_connectors(brain) - if not found: - st.info("No connectors yet.") - st.caption("Add `connectors/*.py` to pull entities from an MCP server — " - "see docs/agents/connectors.mdx.") - return - - import inspect - - state = RunState(brain.config.root) if brain.config.root else None - for name, cls in sorted(found.items()): - meta = (state._data.get(name, {}) if state else {}) - with st.container(border=True): - st.markdown(f"**⚙️ {name}** · schedule `{cls.schedule}`") - cols = st.columns(3) - cols[0].metric("source", "live MCP" if cls.mcp_url else "offline/demo") - cols[1].metric("last run", (meta.get("last_run") or "never")[:19]) - cols[2].metric("entities", meta.get("last_count", "—")) - st.caption(f"endpoint: `{cls.mcp_url or 'offline/demo'}` · " - f"last status: {meta.get('last_status', '—')}") - with st.expander("view script"): - try: - st.code(inspect.getsource(cls), language="python") - except (OSError, TypeError): - st.caption("(source unavailable)") - st.caption(f"Run now: `open-index ingest {name}` · " - "`open-index run` for everything due.") - - -def main() -> None: - brain, url_name = _select_brain() - st.markdown(view.ROW_CSS, unsafe_allow_html=True) - - options = render_sidebar(brain) - # Streamlit opens the first tab, so the help page leftmost means a visitor - # lands on the explanation rather than having to find it. - tab_help, tab_schema, tab_explore, tab_map, tab_analytics, tab_jobs = st.tabs( - [name for name, _ in view.TAB_GUIDE] - ) - with tab_help: - render_how_to_use(brain, url_name) - with tab_schema: - render_schema(brain) - with tab_explore: - render_explore(brain, options) - with tab_map: - render_map(brain) - with tab_analytics: - render_analytics(brain) - with tab_jobs: - render_jobs(brain) - - -main() diff --git a/open_index/ui/static/app.css b/open_index/ui/static/app.css new file mode 100644 index 0000000..4837e1c --- /dev/null +++ b/open_index/ui/static/app.css @@ -0,0 +1,333 @@ +/* Open Index explorer. + * + * Hand-written rather than generated: shipping Tailwind would mean a Node build + * step in a pip-installable package, and this is one small app with a + * consistent design, which is exactly where a token sheet beats a utility + * framework. + * + * Light is the base palette; dark redefines only the tokens. Every colour is + * defined on bare :root so nothing depends on a media query having matched. + */ + +:root { + --bg: #fbfbfa; + --surface: #ffffff; + --surface-2: #f5f5f4; + --border: #e4e4e2; + --border-2: #d3d3d0; + --text: #1a1a19; + --text-2: #5c5c58; + --text-3: #8a8a85; + --accent: #b45309; + --accent-bg: #fef3e7; + --on-accent: #ffffff; + --danger: #b91c1c; + --ok: #15803d; + --shadow: 0 1px 2px rgba(0,0,0,.04), 0 4px 12px rgba(0,0,0,.04); + --radius: 10px; + --radius-sm: 6px; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, Roboto, + "Helvetica Neue", Arial, sans-serif; +} + +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #0f0f0e; + --surface: #191918; + --surface-2: #232322; + --border: #2c2c2a; + --border-2: #3d3d3a; + --text: #ececea; + --text-2: #a3a39d; + --text-3: #74746e; + --accent: #f0a868; + --accent-bg: #2a1d10; + --on-accent: #1a1a19; + --danger: #f87171; + --ok: #4ade80; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 4px 14px rgba(0,0,0,.25); + } +} + +:root[data-theme="dark"] { + --bg: #0f0f0e; + --surface: #191918; + --surface-2: #232322; + --border: #2c2c2a; + --border-2: #3d3d3a; + --text: #ececea; + --text-2: #a3a39d; + --text-3: #74746e; + --accent: #f0a868; + --accent-bg: #2a1d10; + --on-accent: #1a1a19; + --danger: #f87171; + --ok: #4ade80; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 4px 14px rgba(0,0,0,.25); +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font-family: var(--sans); + font-size: 15px; + line-height: 1.6; + -webkit-font-smoothing: antialiased; +} + +a { color: inherit; text-decoration: none; } +a:hover { color: var(--accent); } + +code, pre, .mono { font-family: var(--mono); } + +code { + font-size: .86em; + background: var(--surface-2); + border: 1px solid var(--border); + border-radius: 4px; + padding: .08em .38em; +} + +pre { + background: var(--surface-2); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + padding: 14px 16px; + overflow-x: auto; + font-size: 13px; + line-height: 1.55; + margin: 0; +} +pre code { background: none; border: 0; padding: 0; font-size: inherit; } + +h1, h2, h3, h4 { line-height: 1.25; margin: 0; font-weight: 600; letter-spacing: -.01em; } +h1 { font-size: 1.5rem; } +h2 { font-size: 1.12rem; margin-bottom: 2px; } +h3 { font-size: .95rem; } + +p { margin: 0 0 10px; } +.muted { color: var(--text-2); font-size: .88rem; } +.dim { color: var(--text-3); } +small { font-size: .82rem; } + +/* -- shell ---------------------------------------------------------------- */ + +.shell { display: flex; min-height: 100vh; } + +.sidebar { + width: 260px; + flex: 0 0 260px; + background: var(--surface); + border-right: 1px solid var(--border); + padding: 20px 16px; + position: sticky; + top: 0; + height: 100vh; + overflow-y: auto; +} + +.main { flex: 1; min-width: 0; } + +.topbar { + position: sticky; top: 0; z-index: 20; + display: flex; align-items: center; gap: 4px; + padding: 0 28px; + background: color-mix(in srgb, var(--bg) 88%, transparent); + backdrop-filter: saturate(1.6) blur(8px); + border-bottom: 1px solid var(--border); +} + +.content { padding: 26px 28px 64px; max-width: 1180px; } +.content.wide { max-width: none; } + +/* -- nav ------------------------------------------------------------------ */ + +.tab { + padding: 13px 14px; + font-size: .9rem; + color: var(--text-2); + border-bottom: 2px solid transparent; + white-space: nowrap; +} +.tab:hover { color: var(--text); } +.tab.active { color: var(--text); border-bottom-color: var(--accent); font-weight: 500; } + +.brand { display: flex; align-items: center; gap: 9px; margin-bottom: 4px; } +.brand-mark { + width: 26px; height: 26px; border-radius: 7px; flex: 0 0 26px; + background: linear-gradient(135deg, var(--accent), color-mix(in srgb, var(--accent) 55%, #7c3aed)); +} +.brand-name { font-weight: 600; font-size: 1rem; letter-spacing: -.01em; word-break: break-word; } + +.side-section { margin-top: 22px; } +.side-label { + font-size: .68rem; text-transform: uppercase; letter-spacing: .09em; + color: var(--text-3); font-weight: 600; margin-bottom: 8px; +} + +.dt-row { + display: flex; align-items: center; gap: 8px; + padding: 5px 7px; border-radius: var(--radius-sm); + font-size: .84rem; +} +.dt-row:hover { background: var(--surface-2); } +.dt-row .name { font-family: var(--mono); font-size: .8rem; flex: 1; min-width: 0; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.dt-row .count { color: var(--text-3); font-variant-numeric: tabular-nums; } + +.dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 9px; } + +/* -- cards & tables ------------------------------------------------------- */ + +.card { + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 18px 20px; + margin-bottom: 14px; + box-shadow: var(--shadow); +} +.card > :last-child { margin-bottom: 0; } + +.grid { display: grid; gap: 14px; } +.grid.cols-2 { grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); } +.grid.cols-4 { grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); } + +.stat { background: var(--surface); border: 1px solid var(--border); + border-radius: var(--radius); padding: 14px 16px; } +.stat .n { font-size: 1.6rem; font-weight: 600; font-variant-numeric: tabular-nums; + letter-spacing: -.02em; } +.stat .k { font-size: .78rem; color: var(--text-2); } + +.table-wrap { overflow-x: auto; border: 1px solid var(--border); + border-radius: var(--radius-sm); background: var(--surface); } +table { border-collapse: collapse; width: 100%; font-size: .86rem; } +th, td { text-align: left; padding: 9px 12px; border-bottom: 1px solid var(--border); } +th { font-weight: 600; font-size: .74rem; text-transform: uppercase; + letter-spacing: .06em; color: var(--text-3); background: var(--surface-2); + position: sticky; top: 0; } +tr:last-child td { border-bottom: 0; } +td.mono, th.mono { font-family: var(--mono); font-size: .8rem; } + +/* -- list rows (search results, neighbours) ------------------------------- */ + +.rows { border: 1px solid var(--border); border-radius: var(--radius-sm); + overflow: hidden; background: var(--surface); } +.row { + display: flex; align-items: center; gap: 10px; + padding: 10px 13px; border-bottom: 1px solid var(--border); + font-size: .89rem; +} +.row:last-child { border-bottom: 0; } +.row:hover { background: var(--surface-2); } +.row .title { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; + white-space: nowrap; } +.row .meta { color: var(--text-3); font-family: var(--mono); font-size: .76rem; + white-space: nowrap; } + +.badge { + display: inline-block; font-size: .7rem; padding: 1px 7px; border-radius: 20px; + border: 1px solid var(--border-2); color: var(--text-2); white-space: nowrap; +} +.badge.warn { color: var(--danger); border-color: var(--danger); } + +/* -- forms ---------------------------------------------------------------- */ + +input[type="search"], input[type="text"], select { + width: 100%; padding: 9px 12px; font-size: .92rem; font-family: inherit; + color: var(--text); background: var(--surface); + border: 1px solid var(--border-2); border-radius: var(--radius-sm); +} +input:focus, select:focus { outline: 2px solid var(--accent); outline-offset: -1px; } + +.chips { display: flex; flex-wrap: wrap; gap: 6px; } +.chip { + font-size: .78rem; padding: 4px 11px; border-radius: 20px; cursor: pointer; + border: 1px solid var(--border-2); color: var(--text-2); background: var(--surface); + display: inline-flex; align-items: center; gap: 6px; +} +.chip:hover { border-color: var(--accent); color: var(--text); } +.chip.on { background: var(--accent-bg); border-color: var(--accent); color: var(--accent); } + +.btn { + display: inline-block; padding: 8px 14px; font-size: .87rem; + border: 1px solid var(--border-2); border-radius: var(--radius-sm); + background: var(--surface); color: var(--text); cursor: pointer; +} +.btn:hover { border-color: var(--accent); color: var(--accent); } +/* --on-accent, not a hardcoded white: the dark palette's accent is a light + orange, and white text on it is unreadable. */ +.btn.primary { background: var(--accent); border-color: var(--accent); + color: var(--on-accent); } + +/* -- notices -------------------------------------------------------------- */ + +.note { + border: 1px solid var(--border); border-left: 3px solid var(--accent); + background: var(--surface); border-radius: var(--radius-sm); + padding: 12px 15px; font-size: .88rem; margin-bottom: 14px; +} +.note.warn { border-left-color: var(--danger); } +.note.plain { border-left-color: var(--border-2); color: var(--text-2); } + +/* -- disclosure ----------------------------------------------------------- */ + +details.block { + border: 1px solid var(--border); border-radius: var(--radius); + background: var(--surface); margin-bottom: 10px; overflow: hidden; +} +details.block > summary { + cursor: pointer; padding: 13px 17px; font-weight: 500; font-size: .93rem; + display: flex; align-items: center; gap: 9px; list-style: none; +} +details.block > summary::-webkit-details-marker { display: none; } +details.block > summary::before { + content: "›"; color: var(--text-3); font-size: 1.05rem; line-height: 1; + transition: transform .15s; display: inline-block; +} +details.block[open] > summary::before { transform: rotate(90deg); } +details.block > summary:hover { background: var(--surface-2); } +details.block .body { padding: 0 17px 17px; } + +/* -- map ------------------------------------------------------------------ */ + +.map-layout { display: flex; gap: 16px; align-items: flex-start; } +#cy { + flex: 1; min-width: 0; height: 660px; + background: var(--surface); border: 1px solid var(--border); + border-radius: var(--radius); +} +.legend { width: 210px; flex: 0 0 210px; } + +#tip { + position: fixed; z-index: 50; pointer-events: none; display: none; + max-width: 320px; white-space: pre-line; + background: var(--surface); color: var(--text); + border: 1px solid var(--border-2); border-radius: var(--radius-sm); + padding: 8px 11px; font-size: .8rem; line-height: 1.45; box-shadow: var(--shadow); +} + +/* -- misc ----------------------------------------------------------------- */ + +.copy { float: right; font-size: .74rem; } +.hr { height: 1px; background: var(--border); margin: 26px 0; border: 0; } +.page-head { margin-bottom: 18px; } +.page-head p { color: var(--text-2); font-size: .9rem; margin: 5px 0 0; } +.deflist dt { font-weight: 600; margin-top: 12px; } +.deflist dd { margin: 3px 0 0; color: var(--text-2); } +.deflist code { color: var(--text); } + +@media (max-width: 900px) { + .shell { flex-direction: column; } + .sidebar { width: auto; flex: none; height: auto; position: static; + border-right: 0; border-bottom: 1px solid var(--border); } + .map-layout { flex-direction: column; } + .legend { width: auto; flex: none; } + #cy { width: 100%; height: 460px; } + .topbar { overflow-x: auto; padding: 0 14px; } + .content { padding: 20px 14px 48px; } +} diff --git a/open_index/ui/static/app.js b/open_index/ui/static/app.js new file mode 100644 index 0000000..a3337e6 --- /dev/null +++ b/open_index/ui/static/app.js @@ -0,0 +1,47 @@ +/* Shared behaviour: theme toggle and copy-to-clipboard. + * The map has its own script; this file must stay useful with JS-light pages. */ + +(function () { + var root = document.documentElement; + + function current() { + var explicit = root.getAttribute("data-theme"); + if (explicit) return explicit; + return window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"; + } + + var toggle = document.getElementById("theme-toggle"); + if (toggle) { + toggle.addEventListener("click", function () { + var next = current() === "dark" ? "light" : "dark"; + root.setAttribute("data-theme", next); + try { localStorage.setItem("oi-theme", next); } catch (e) {} + }); + } + + // Copy buttons: + + + {% if multi %} +
+
Other indexes
+ ← all indexes +
+ {% endif %} + + +
+ + +
+ {% block body %}{% endblock %} +
+
+ + + +
+ +{% block scripts %}{% endblock %} + + diff --git a/open_index/ui/templates/directory.html b/open_index/ui/templates/directory.html new file mode 100644 index 0000000..c52ee1b --- /dev/null +++ b/open_index/ui/templates/directory.html @@ -0,0 +1,37 @@ + + + + + +Indexes · Open Index + + + + + +
+
+

Indexes on this host

+

{{ brains | length }} available. Each has its own explorer and its own MCP + endpoint at /<name>/mcp.

+
+
+ {% for name in brains %} + {% set s = summaries[name] %} + +

{{ s.name }}

+

{{ s.description or "—" }}

+
+ {{ s.total_entities | comma }} entities · {{ s.doc_types | length }} doc types +
+
+ {% for r in s.doc_types[:6] %} + {{ r.name }} + {% endfor %} +
+ + {% endfor %} +
+
+ + diff --git a/open_index/ui/templates/entity.html b/open_index/ui/templates/entity.html new file mode 100644 index 0000000..e860423 --- /dev/null +++ b/open_index/ui/templates/entity.html @@ -0,0 +1,82 @@ +{% extends "base.html" %} +{% block title %}{{ entity.name if entity else entity_id }} · {{ summary.name }}{% endblock %} + +{% block body %} +

← back to explore

+ +{% if entity is none %} +
No entity {{ entity_id }} in this index.
+{% else %} + +
+

+ + {{ entity.name }} +

+

{{ entity.id }}

+
+ +{% if fields %} +
+ + + + {% for row in fields %} + + {% endfor %} + +
fieldvalue
{{ row.field }}{{ row.value }}
+
+{% endif %} + +{% if provenance %} +
+ Provenance +
+
+ + + + + + + + +
asserted byasserted atconfidenceevidence
{{ provenance.asserted_by }}{{ provenance.asserted_at }}{{ provenance.confidence }}{{ provenance.evidence }}
+
+
+
+{% endif %} + +{% if entity.valid_from or entity.valid_to %} +

valid: {{ entity.valid_from or "—" }} → {{ entity.valid_to or "now" }}

+{% endif %} + +

Relationships ({{ links | length }})

+{% if links %} +
+ {% for link in links %} + + {{ link.direction }} + + + {{ link.meaning or "(related)" }} · {{ link.other_name }} + + {% if not link.exists %}missing{% endif %} + {{ link.other_id }} + + {% endfor %} +
+{% else %} +
+ None yet. Edges are what make this a graph — ask your agent to link this + entity to related ones. +
+{% endif %} + +

+ Show on the map → +

+ +{% endif %} +{% endblock %} diff --git a/open_index/ui/templates/explore.html b/open_index/ui/templates/explore.html new file mode 100644 index 0000000..9902633 --- /dev/null +++ b/open_index/ui/templates/explore.html @@ -0,0 +1,101 @@ +{% extends "base.html" %} +{% block title %}Explore · {{ summary.name }}{% endblock %} + +{% block body %} +
+

Explore

+

Search the index, or browse by doc type. Open an entity to see its fields + and every relationship in both directions.

+
+ +{% if summary.is_empty %} +
+ This index has no entities yet. Add some with open-index import <file>, + a connector, or by asking your agent — then run open-index index. +
+{% else %} + +
+
+ + + +
+ +
+ {% for row in summary.doc_types %} + {% if row.count %} + {# Each chip is a link that toggles itself in the query string, so the + filter is in the URL and therefore shareable and back-buttonable. #} + + {{ row.name }} + + {% endif %} + {% endfor %} + {% if selected %} + clear + {% endif %} +
+
+ +{% if error %} +
Search failed: {{ error }}
+{% endif %} + +{% if results is not none %} +

{{ results.total }} result{{ "" if results.total == 1 else "s" }}

+ {% if results.rows %} +
+ {% for r in results.rows %} + + + {{ r.name }} + {{ r.doc_type }} + + {% endfor %} +
+ {% else %} +
No matches. Try fewer words, or switch the mode to Semantic.
+ {% endif %} + +{% elif browse is not none %} + {% if not browse %} +
No entities in the selected doc types.
+ {% endif %} + {% for group in browse %} +
+ + + {{ group.row.name }} + · {{ group.row.count | comma }} + +
+ {% if group.row.description %}

{{ group.row.description }}

{% endif %} +
+ {% for e in group.entities %} + + {{ e.name }}{% if e.description %} + · {{ e.description }}{% endif %} + {{ e.id }} + + {% endfor %} +
+ {% if group.truncated %} +

Showing the first 200 of + {{ group.row.count | comma }} — search to narrow.

+ {% endif %} +
+
+ {% endfor %} +{% endif %} + +{% endif %} +{% endblock %} diff --git a/open_index/ui/templates/help.html b/open_index/ui/templates/help.html new file mode 100644 index 0000000..75f4204 --- /dev/null +++ b/open_index/ui/templates/help.html @@ -0,0 +1,127 @@ +{% extends "base.html" %} +{% block title %}How to use {{ summary.name }}{% endblock %} + +{% block body %} +
+

What an index holds

+

Three ideas, and the whole thing follows from them.

+
+ +
+
+ {% for term, what in model_guide %} +
{{ term }}
+
{{ what | md }}
+ {% endfor %} +
+
+ +
+ +
+

Connect an agent to {{ summary.name }}

+

This index speaks MCP, so any MCP-capable agent can query it{% if not read_only %} — and keep it current{% endif %}.

+
+ +{% if mcp_url %} +
+

1. Point your agent at this URL

+

Paste into .mcp.json (Claude Code), + .cursor/mcp.json (Cursor), or any MCP client's server config.

+ + 
{{ client_config }}
+

Or generate it:

+ 
open-index mcp-config --url {{ mcp_url }} --name {{ summary.name }} > .mcp.json
+
+{% else %} +
+ This explorer isn't configured with a public MCP URL, so the connection + block can't be shown. + 
open-index mcp-config --brain <brain-dir> > .mcp.json
+
+{% endif %} + +
+

2. Ask it something

+

No briefing needed — the navigation guide is injected + into the MCP handshake, so the agent knows this index's doc types and + relationship vocabulary before its first turn.

+
+ +
+ +

Tools the agent gets

+ +
+

Reading

+
+ {% for name, what in read_tools %} +
{{ name }}
+
{{ what | md }}
+ {% endfor %} +
+
+ +
+

Writing

+ {% if read_only %} +
+ This endpoint is read-only — the tools below are not + registered on it. Serve without --read-only to enable them. +
+ {% endif %} +
+ {% for name, what in write_tools %} +
{{ name }}
+
{{ what | md }}
+ {% endfor %} +
+

Entity ids are always + <doc_type>:<slug>. Writes are validated against the + doc_type schema, and land in the search index (and on disk, for + storage: file types).

+
+ +
+ +

What each tab does

+
+
+ {% for name, what in tab_guide %} +
{{ name }}
+
{{ what | md }}
+ {% endfor %} +
+
+ +
+ +

What's in this index right now

+

{{ summary.total_entities | comma }} entities across + {{ summary.doc_types | length }} doc types.

+ +{% if summary.doc_types %} +
+ + + + {% for r in summary.doc_types %} + + + + + + + {% endfor %} + +
doc typeentitiessource of truthwhat it holds
+ {{ r.name }} + {{ r.count | comma }}{{ "files (git)" if r.storage == "file" else "search index" }}{{ r.description or "—" }}
+
+{% endif %} + +
+ The full navigation guide the agent receives + 
{{ guide }}
+
+{% endblock %} diff --git a/open_index/ui/templates/jobs.html b/open_index/ui/templates/jobs.html new file mode 100644 index 0000000..f3e1738 --- /dev/null +++ b/open_index/ui/templates/jobs.html @@ -0,0 +1,39 @@ +{% extends "base.html" %} +{% block title %}Jobs · {{ summary.name }}{% endblock %} + +{% block body %} +
+

Jobs

+

Ingestion scripts in connectors/*.py that pull entities from an + MCP server on a schedule.

+
+ +{% if not jobs %} +
+ No connectors yet. Add connectors/*.py to pull entities from an + MCP server. +
+{% endif %} + +{% for job in jobs %} +
+

⚙️ {{ job.name }} · schedule {{ job.schedule }}

+
+
{{ "live MCP" if job.mcp_url else "offline/demo" }}
source
+
{{ job.last_run }}
last run
+
{{ job.last_count }}
entities
+
{{ job.last_status }}
last status
+
+

endpoint: {{ job.mcp_url or "offline/demo" }}

+ {% if job.source %} +
+ View script + 
{{ job.source }}
+
+ {% endif %} +

Run now: + open-index ingest {{ job.name }} · + open-index run for everything due.

+
+{% endfor %} +{% endblock %} diff --git a/open_index/ui/templates/map.html b/open_index/ui/templates/map.html new file mode 100644 index 0000000..0487c44 --- /dev/null +++ b/open_index/ui/templates/map.html @@ -0,0 +1,71 @@ +{% extends "base.html" %} +{% block title %}Map · {{ summary.name }}{% endblock %} +{% block wide %}wide{% endblock %} + +{% block body %} +
+

Map

+

Relationships drawn. Nodes carry colour and shape only — names are long and + overlap badly at any real size, so identity is on hover. Click a node to + focus on it and its immediate neighbours.

+
+ +{% if not populated %} +
No entities to map yet. Add some and run open-index index.
+{% else %} + +
+
+ {% for name in populated %} + + {{ name }} + + {% endfor %} + {% if selected | length != populated | length %} + all types + {% endif %} +
+ {% if focus %} +

+ Focused on {{ focus }} and its immediate neighbours. + ↩ show the whole index +

+ {% endif %} +
+ + + +
+
+
+
+
Legend
+
+ +
+
+
Counts
+
0 nodes · 0 edges
+
+
+
+ + + + + +{% endif %} +{% endblock %} diff --git a/open_index/ui/templates/missing.html b/open_index/ui/templates/missing.html new file mode 100644 index 0000000..552762b --- /dev/null +++ b/open_index/ui/templates/missing.html @@ -0,0 +1,25 @@ + + + + + +Not found · Open Index + + + +
+

No index called {{ wanted }}

+ {% if brains %} +

This host serves:

+
+ {% for name in brains %} + {{ name }} + {% endfor %} +
+ {% else %} +

This host serves no indexes. A brain is a directory + containing brain.yaml.

+ {% endif %} +
+ + diff --git a/open_index/ui/templates/schema.html b/open_index/ui/templates/schema.html new file mode 100644 index 0000000..917ebc8 --- /dev/null +++ b/open_index/ui/templates/schema.html @@ -0,0 +1,86 @@ +{% extends "base.html" %} +{% block title %}Schema · {{ summary.name }}{% endblock %} + +{% block body %} +
+

Schema

+

{{ summary.doc_types | length }} doc types · {{ summary.total_entities | comma }} entities. + Every entity id is <doc_type>:<slug>, and any entity can link + to any other through related_to.

+
+ +{% if not summary.has_schema %} +
+ No doc types defined yet. A doc type is a concept this index tracks, plus the + fields it stores. Create one with open-index add-doc-type, or ask + an agent to call create_doc_type. +
+{% endif %} + +{% for block in blocks %} +
+ + + {{ block.row.name }} + + · {{ block.row.count | comma }} {{ "entity" if block.row.count == 1 else "entities" }} + + +
+ {% if block.row.description %}

{{ block.row.description }}

{% endif %} +

Source of truth: + {% if block.row.storage == "file" %} + files — JSON under entities/, git-trackable + {% else %} + search index — DB-owned, not written to files + {% endif %} +

+ +

Fields

+ {% if block.fields %} +
+ + + {% for key in block.fields[0].keys() %}{% endfor %} + + + {% for row in block.fields %} + {% for key, val in row.items() %} + + {% endfor %} + {% endfor %} + +
{{ key }}
{{ val }}
+
+ {% else %} +

No fields declared.

+ {% endif %} + +

Relationships (optional)

+ {% if block.relationships %} +
+ + + + {% for row in block.relationships %} + + + + + + + {% endfor %} + +
relationshippoints atdeclaredin use
{{ row["relationship"] }}{{ row["points at"] }}{{ row["declared"] }}{{ row["in use"] }}
+
+

Declared edges are validated against + their target doc type. Undeclared ones still work — they just aren't checked.

+ {% else %} +

None declared or in use. Entities of this type are valid + without any; edges are what make the index traversable rather than just + searchable.

+ {% endif %} +
+
+{% endfor %} +{% endblock %} diff --git a/open_index/ui/view.py b/open_index/ui/view.py index bc79714..44ecea1 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -1,9 +1,9 @@ """View-model for the explorer: everything the UI shows, minus the rendering. -Kept free of Streamlit so the decisions that actually matter — which doc_types +Kept free of any rendering library so the decisions that actually matter — which doc_types to list, what to anchor the map on when the user hasn't chosen, how to describe -an entity's neighbours — are plain functions that can be tested. `app.py` is -then a thin layer of widgets over these. +an entity's neighbours — are plain functions that can be tested. `web.py` and the +templates are then a thin rendering layer over these. """ from __future__ import annotations @@ -67,29 +67,6 @@ def has_schema(self) -> bool: return bool(self.doc_types) -def brain_from_url(url: Optional[str], available: list[str]) -> Optional[str]: - """Which brain a URL asks for: the first path segment. - - `/support-index` and `/support-index/anything` both select `support-index`, so - the index name lives in the path exactly as it does for the MCP endpoint at - `//mcp`. - - Read from the browser's URL rather than left to Streamlit's own page router: - that router resolves programmatic pages by an internal identifier, so every - path rendered whichever brain sorted first. An unknown or missing segment - falls back to the first brain — a stale link should land somewhere useful - rather than on an error. - """ - if not available: - return None - if not url: - return available[0] - - from urllib.parse import urlparse - - first = urlparse(url).path.strip("/").split("/")[0] - return first if first in available else available[0] - def color_for(brain: Brain, doc_type: str) -> str: dt = brain.config.doc_type(doc_type) @@ -227,88 +204,7 @@ def semantic_weight_for(mode: str) -> Optional[float]: # -- map rendering ------------------------------------------------------------ -# The canvas width handed to streamlit-agraph. It MUST be an int: the library -# does `f"{width}px"`, so a CSS string like "100%" becomes the invalid value -# "100%px", the canvas fails to size, and the graph is stranded in a corner -# instead of centred. -# Sized to sit beside the legend column rather than to fill the page: at the -# old full-page width the canvas overflowed its column once the legend arrived. -GRAPH_WIDTH = 950 -GRAPH_HEIGHT = 650 - -# Past this many nodes a force layout keeps drifting, so we slow it down rather -# than switching physics off — vis only auto-fits the viewport as part of -# stabilisation, and without stabilisation the graph never centres. -BUSY_GRAPH_NODES = 150 - -_GRAPH_THEMES = { - "dark": { - # vis defaults to near-black labels with a white halo, which on a dark - # canvas is both unreadable and visually noisy. - "node_label": "#e8eaed", - "edge_label": "#aab2bd", - "edge": "#6b7280", - "stroke_width": 0, - }, - "light": { - "node_label": "#1f2328", - "edge_label": "#57606a", - "edge": "#c8ccd2", - "stroke_width": 0, - }, -} - - -# Buttons styled as full-width list rows, so results and neighbours read as a -# list rather than a wall of chrome. -# -# Every value is theme-agnostic on purpose. Hardcoding `background:#fff` painted -# white rows under Streamlit's dark theme, which keeps its light text — white on -# white, and the entity list became invisible. So: transparent background, -# inherited text colour, and translucent grey borders that read correctly -# against either a light or a dark surface. -ROW_CSS = """ - -""" - - -def mcp_url_for(current_url: Optional[str], name: Optional[str]) -> Optional[str]: - """This index's MCP endpoint, derived from the URL the browser is on. - - An explorer serving many brains cannot be handed one correct endpoint as - configuration — the answer depends on which index you are looking at. But - the page already knows: it is at `/`, so the endpoint is - `//mcp` on the same origin. Deriving it also means it stays right - behind any proxy or hostname without anything being configured. - Returns None when the URL or the index name is unknown, so callers can fall - back to explicit configuration. - """ - if not current_url or not name: - return None - - from urllib.parse import urlparse - - parsed = urlparse(current_url) - if not parsed.scheme or not parsed.netloc: - return None - return f"{parsed.scheme}://{parsed.netloc}/{name}/mcp" def mcp_client_config(url: str, server_name: str = "open-index") -> str: @@ -465,7 +361,8 @@ def node_tooltip(entity_id: str, name: str, doc_type: str, fields: Optional[dict] = None) -> str: """What hovering a node shows: the full name, its type, and a few fields. - vis renders this as plain text, so it is newline-separated rather than HTML. + Newline-separated plain text: the tooltip element sets `white-space: + pre-line` and takes it via textContent, so no markup is involved. """ lines = [name, f"{doc_type} · {entity_id}"] for key, value in list((fields or {}).items()): @@ -497,52 +394,5 @@ def legend_rows(brain, graph) -> list[dict]: ] -def graph_node_specs(graph, anchor_size: int = 20, size: int = 13) -> list[dict]: - """Node payloads for the map renderer. - - Nothing is labelled on the canvas. Entity names are long and arbitrary — - drawn beside every dot they overlap each other and their own edges, and no - amount of truncation fixes a dense graph. The picture carries shape and - colour; identity comes from the tooltip, and the legend explains the colours. - - `label` is an empty string rather than None: None serialises to null and vis - draws that literally. - """ - specs = [] - for node in graph.nodes: - fields = {k: v for k, v in (node.data or {}).items() - if k not in ("id", "doc_type", "related_to")} - specs.append({ - "id": node.id, - "label": "", - "color": node.color, - "shape": "dot", - "size": anchor_size if node.is_anchor else size, - "title": node_tooltip(node.id, node.label, node.doc_type, fields), - }) - return specs - - -def graph_edge_specs(graph, color: str) -> list[dict]: - """Edge payloads. Unlabelled for the same reason as nodes — the relationship - is on the tooltip.""" - return [ - { - "source": edge.source, - "target": edge.target, - "label": "", - "title": edge_tooltip(edge.source, edge.target, edge.meaning), - "color": color, - } - for edge in graph.edges - ] -def graph_theme(theme_type: Optional[str]) -> dict: - """Label and edge colours for the map, given Streamlit's active theme. - - Anything unrecognised (including None, which is what Streamlit reports when - the user is following their browser preference and the server was never - told) falls back to the light palette. - """ - return _GRAPH_THEMES.get((theme_type or "").lower(), _GRAPH_THEMES["light"]) diff --git a/open_index/ui/web.py b/open_index/ui/web.py new file mode 100644 index 0000000..fd31ecc --- /dev/null +++ b/open_index/ui/web.py @@ -0,0 +1,426 @@ +"""The explorer, served as ordinary web pages. + +Replaces the Streamlit app. The view-model in `view.py` is unchanged and still +decides *what* is shown; this module only renders it, and `templates/` decides +how it looks. + +Two things follow from being real pages rather than a widget script: + + every view has a URL //schema, //entity/ — so a tab, an + entity, a filtered map are all linkable, the back + button works, and a demo can be sent as a link. + the page is HTML no websocket, no rerun-the-whole-script model, and + the design is CSS rather than framework selectors. + +One process serves every brain under OPEN_INDEX_BRAINS_ROOT, selected by the +first path segment exactly as the MCP endpoint at //mcp is. A single +brain (OPEN_INDEX_DIR) is served at the root instead. +""" + +from __future__ import annotations + +import os +from functools import lru_cache +from pathlib import Path +from typing import Any, Optional + +from open_index.brain import Brain +from open_index.ui import view + +HERE = Path(__file__).parent +TEMPLATES = HERE / "templates" +STATIC = HERE / "static" + +# Tab slug -> (label, template). The label comes from view.TAB_GUIDE so the nav +# and the "what each tab does" list cannot drift apart. +TABS = [ + ("", view.HELP_TAB, "help.html"), + ("schema", "Schema", "schema.html"), + ("explore", "Explore", "explore.html"), + ("map", "Map", "map.html"), + ("analytics", "Analytics", "analytics.html"), + ("jobs", "Jobs", "jobs.html"), +] + + +def inline_markdown(text: str): + """The `**bold**` and `` `code` `` used in view.py's guide strings, as HTML. + + Those strings are written as markdown because they are also read as plain + text (the MCP navigation guide), so the UI has to render the little of it + that appears. Escaping happens first and the result is marked safe, so the + only HTML that can reach the page is the two tags produced here — a full + markdown library would be a dependency and a much wider surface. + """ + import re + + from markupsafe import Markup, escape + + out = str(escape(text)) + out = re.sub(r"\*\*(.+?)\*\*", r"\1", out) + out = re.sub(r"`([^`]+?)`", r"\1", out) + return Markup(out) + + +@lru_cache(maxsize=64) +def open_brain(brain_dir: str) -> Brain: + """Opened once per directory: opening a brain loads an embedding model.""" + return Brain.open(brain_dir) + + +@lru_cache(maxsize=1) +def discover(root: str) -> dict[str, str]: + from open_index.config import discover_brains + + return {name: str(path) for name, path in discover_brains(root).items()} + + +def available_brains() -> dict[str, str]: + """Every brain this process serves, keyed by its URL segment. + + A single-brain deployment has no segment, so it is keyed by "" and lives at + the root. + """ + root = os.environ.get("OPEN_INDEX_BRAINS_ROOT") + if root: + return discover(root) + return {"": os.environ.get("OPEN_INDEX_DIR", ".")} + + +def is_read_only() -> bool: + return os.environ.get("OPEN_INDEX_READ_ONLY", "").lower() in ("1", "true", "yes") + + +def mcp_url_for_request(request, name: str) -> str: + """This index's MCP endpoint, from the URL the browser actually used. + + Derived rather than configured for the same reason as before: one shared + process cannot be handed a single correct URL, and deriving it keeps the + value right behind any proxy. Honours X-Forwarded-Proto so a TLS-terminating + proxy does not turn an https page into an http endpoint. + """ + forwarded = request.headers.get("x-forwarded-proto") + scheme = (forwarded.split(",")[0].strip() if forwarded else request.url.scheme) + host = request.headers.get("host") or request.url.netloc + base = f"{scheme}://{host}" + return f"{base}/{name}/mcp" if name else f"{base}/mcp" + + +# --------------------------------------------------------------------------- # +# Page data — each returns the context its template renders. +# --------------------------------------------------------------------------- # + +def _base_context(request, name: str, brain: Brain, active: str) -> dict[str, Any]: + summary = view.summarize(brain) + return { + "request": request, + "brain": brain, + "name": name, + "summary": summary, + "active": active, + "tabs": TABS, + "base": f"/{name}" if name else "", + "multi": len(available_brains()) > 1 or bool(name), + "read_only": is_read_only(), + } + + +def page_help(request, name: str, brain: Brain) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "") + mcp_url = mcp_url_for_request(request, name) or os.environ.get( + "OPEN_INDEX_PUBLIC_URL", "") + ctx.update( + mcp_url=mcp_url, + client_config=view.mcp_client_config( + mcp_url, server_name=ctx["summary"].name) if mcp_url else "", + model_guide=view.MODEL_GUIDE, + read_tools=view.READ_TOOLS, + write_tools=view.WRITE_TOOLS, + tab_guide=view.TAB_GUIDE, + guide=brain.navigation_guidelines(include_writes=not ctx["read_only"]), + ) + return ctx + + +def page_schema(request, name: str, brain: Brain) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "schema") + blocks = [] + for row in ctx["summary"].doc_types: + doc_type = brain.config.doc_type(row.name) + blocks.append({ + "row": row, + "fields": view.schema_field_rows(doc_type) if doc_type else [], + "relationships": view.schema_relationship_rows(brain, row.name), + }) + ctx["blocks"] = blocks + return ctx + + +def page_explore(request, name: str, brain: Brain) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "explore") + params = request.query_params + query = (params.get("q") or "").strip() + mode = params.get("mode") or "Hybrid" + if mode not in view.SEARCH_MODES: + mode = "Hybrid" + selected = [t for t in params.getlist("t") if t] + + ctx.update(query=query, mode=mode, modes=list(view.SEARCH_MODES), + selected=selected, results=None, browse=None, error=None) + + if query: + try: + found = brain.search( + query=query, doc_types=selected or None, limit=50, + semantic_weight=view.semantic_weight_for(mode), source="ui") + ctx["results"] = { + "total": found.total, + "rows": [ + {**r, "color": view.color_for(brain, r["doc_type"])} + for r in found.results + ], + } + except Exception as exc: # a backend that is down must not blank the page + ctx["error"] = str(exc) + return ctx + + # No query: list by doc_type, so the page is never blank. + groups = [] + for row in ctx["summary"].doc_types: + if not row.count or (selected and row.name not in selected): + continue + entities = brain.backend.all_entities([row.name])[:200] + groups.append({ + "row": row, + "entities": [ + {"id": e.id, "name": e.name, + "description": str(e.fields.get("description", ""))[:120]} + for e in entities + ], + "truncated": row.count > 200, + }) + ctx["browse"] = groups + return ctx + + +def page_entity(request, name: str, brain: Brain, entity_id: str) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "explore") + entity = brain.get_entity(entity_id, source="ui") + ctx.update(entity_id=entity_id, entity=entity) + if entity is None: + return ctx + ctx.update( + color=view.color_for(brain, entity.doc_type), + fields=view.field_rows(entity), + provenance=view.provenance_row(entity), + links=view.neighbours(brain, entity_id), + ) + return ctx + + +def page_map(request, name: str, brain: Brain) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "map") + populated = [r.name for r in ctx["summary"].doc_types if r.count] + selected = [t for t in request.query_params.getlist("t") if t in populated] + focus = request.query_params.get("focus") or None + ctx.update(populated=populated, selected=selected or populated, focus=focus) + return ctx + + +def graph_payload(brain: Brain, scope: list[str], focus: Optional[str]) -> dict[str, Any]: + """Nodes and edges for the map, shaped for the client-side renderer.""" + from open_index.graph import build_graph, build_overview_graph + + if focus: + graph = build_graph(brain, [focus], depth=1) + else: + graph = build_overview_graph(brain, scope, limit=view.MAX_GRAPH_NODES) + + total = sum(count for dt, count in brain.counts().items() if dt in scope) + return { + "nodes": [ + { + "id": n.id, + "label": n.label, + "doc_type": n.doc_type, + "color": n.color, + "anchor": bool(n.is_anchor), + "tooltip": view.node_tooltip( + n.id, n.label, n.doc_type, + {k: v for k, v in (n.data or {}).items() + if k not in ("id", "doc_type", "related_to")}), + } + for n in graph.nodes + ], + "edges": [ + {"source": e.source, "target": e.target, "meaning": e.meaning, + "tooltip": view.edge_tooltip(e.source, e.target, e.meaning)} + for e in graph.edges + ], + "legend": view.legend_rows(brain, graph), + "total": total, + "capped": (not focus) and len(graph.nodes) < total, + } + + +def page_analytics(request, name: str, brain: Brain) -> dict[str, Any]: + ctx = _base_context(request, name, brain, "analytics") + summary = brain.analytics_summary() + ctx["stats"] = summary + ctx["events"] = brain.analytics_events(limit=100) if summary.get( + "total_fetches") else [] + return ctx + + +def page_jobs(request, name: str, brain: Brain) -> dict[str, Any]: + import inspect + + from open_index.connectors.runner import discover_connectors + from open_index.scheduling import RunState + + ctx = _base_context(request, name, brain, "jobs") + found = discover_connectors(brain) + state = RunState(brain.config.root) if brain.config.root else None + jobs = [] + for job_name, cls in sorted(found.items()): + meta = (state._data.get(job_name, {}) if state else {}) + try: + source = inspect.getsource(cls) + except (OSError, TypeError): + source = "" + jobs.append({ + "name": job_name, + "schedule": cls.schedule, + "mcp_url": cls.mcp_url, + "last_run": (meta.get("last_run") or "never")[:19], + "last_count": meta.get("last_count", "—"), + "last_status": meta.get("last_status", "—"), + "source": source, + }) + ctx["jobs"] = jobs + return ctx + + +# --------------------------------------------------------------------------- # +# App +# --------------------------------------------------------------------------- # + +def build_app(): + """The Starlette app serving the explorer.""" + from starlette.applications import Starlette + from starlette.responses import JSONResponse, RedirectResponse + from starlette.routing import Mount, Route + from starlette.staticfiles import StaticFiles + from starlette.templating import Jinja2Templates + + templates = Jinja2Templates(directory=str(TEMPLATES)) + templates.env.filters["comma"] = lambda n: f"{n:,}" + templates.env.filters["md"] = inline_markdown + + def resolve(request): + """(url_segment, Brain) for this request, or None when it cannot be served.""" + brains = available_brains() + if not brains: + return None, None + wanted = request.path_params.get("name") + if wanted is None: # single-brain deployment + key = "" if "" in brains else sorted(brains)[0] + elif wanted in brains: + key = wanted + else: + return None, None + return key, open_brain(brains[key]) + + def render(request, template: str, ctx: dict): + # request first: the two-argument form is the deprecated Starlette + # signature, and on current versions it reads the context as the + # template name. + return templates.TemplateResponse(request, template, ctx) + + def not_found(request, wanted: str): + brains = available_brains() + return templates.TemplateResponse( + request, "missing.html", + {"wanted": wanted, "brains": sorted(n for n in brains if n)}, + status_code=404, + ) + + def make(page_fn, template): + def endpoint(request): + name, brain = resolve(request) + if brain is None: + return not_found(request, request.path_params.get("name", "")) + return render(request, template, page_fn(request, name, brain)) + return endpoint + + def root(request): + brains = available_brains() + if "" in brains: # single brain: serve it here + return make(page_help, "help.html")(request) + if len(brains) == 1: + return RedirectResponse(f"/{next(iter(brains))}") + return templates.TemplateResponse( + request, "directory.html", + {"brains": sorted(brains), + "summaries": {n: view.summarize(open_brain(p)) + for n, p in sorted(brains.items())}}, + ) + + def entity(request): + name, brain = resolve(request) + if brain is None: + return not_found(request, request.path_params.get("name", "")) + ctx = page_entity(request, name, brain, request.path_params["entity_id"]) + # A link to a deleted entity is a broken link, and should read as one to + # anything crawling or checking these pages — not as a successful page + # that happens to say nothing is there. + status = 200 if ctx["entity"] is not None else 404 + return templates.TemplateResponse(request, "entity.html", ctx, + status_code=status) + + def graph_json(request): + name, brain = resolve(request) + if brain is None: + return JSONResponse({"error": "unknown index"}, status_code=404) + populated = [r.name for r in view.summarize(brain).doc_types if r.count] + scope = [t for t in request.query_params.getlist("t") if t in populated] + return JSONResponse(graph_payload( + brain, scope or populated, request.query_params.get("focus") or None)) + + def healthz(request): + from starlette.responses import PlainTextResponse + + return PlainTextResponse("ok") + + pages = [ + ("schema", page_schema, "schema.html"), + ("explore", page_explore, "explore.html"), + ("map", page_map, "map.html"), + ("analytics", page_analytics, "analytics.html"), + ("jobs", page_jobs, "jobs.html"), + ] + + routes = [ + Route("/healthz", healthz), + Mount("/static", app=StaticFiles(directory=str(STATIC)), name="static"), + Route("/", root), + ] + # Single-brain routes live at the root; the multi-brain ones carry /{name}. + for slug, fn, tpl in pages: + routes.append(Route(f"/{slug}", make(fn, tpl))) + routes.append(Route("/entity/{entity_id:path}", entity)) + routes.append(Route("/api/graph", graph_json)) + + routes.append(Route("/{name}", make(page_help, "help.html"))) + for slug, fn, tpl in pages: + routes.append(Route(f"/{{name}}/{slug}", make(fn, tpl))) + routes.append(Route("/{name}/entity/{entity_id:path}", entity)) + routes.append(Route("/{name}/api/graph", graph_json)) + + return Starlette(routes=routes) + + +def serve(host: str = "0.0.0.0", port: int = 8501) -> None: + import uvicorn + + uvicorn.run(build_app(), host=host, port=port) diff --git a/pyproject.toml b/pyproject.toml index 366d705..d9f3da8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,13 @@ dependencies = [ ] [project.optional-dependencies] +# The explorer: ordinary server-rendered pages. Starlette and uvicorn are +# already pulled in by `serve`, so on a deployment that runs both this adds +# only the template engine. ui = [ - "streamlit>=1.36", - "streamlit-agraph>=0.0.45", + "starlette>=0.37", + "jinja2>=3.1", + "uvicorn>=0.30", ] mcp = [ "mcp>=1.2", @@ -32,7 +36,7 @@ opensearch = [ serve = [ "mcp>=1.2", "uvicorn>=0.30", - "starlette>=1.0", + "starlette>=0.37", ] # Semantic/vector search (local ONNX model + OpenAI-compatible API support). semantic = [ @@ -43,12 +47,11 @@ dev = [ "pytest>=8.0", ] all = [ - "streamlit>=1.36", - "streamlit-agraph>=0.0.45", + "jinja2>=3.1", "mcp>=1.2", "opensearch-py>=2.4", "uvicorn>=0.30", - "starlette>=1.0", + "starlette>=0.37", "fastembed>=0.3", "numpy>=1.26", "pytest>=8.0", @@ -60,5 +63,10 @@ open-index = "open_index.cli:app" [tool.setuptools.packages.find] include = ["open_index*"] +# Templates and static assets are part of the package, not just the checkout — +# without this the wheel installs a UI that 500s on every page. +[tool.setuptools.package-data] +open_index = ["ui/templates/*.html", "ui/static/*", "ui/static/vendor/*"] + [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 512e64a..8c6dce7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,6 +6,7 @@ """ import json +import os import shutil from pathlib import Path @@ -366,29 +367,38 @@ def fake_sleep(seconds): # -- ui / mcp (subprocess + server entry points are stubbed) ------------------ -def test_ui_invokes_streamlit(brain_dir, monkeypatch): +def test_ui_serves_the_explorer(brain_dir, monkeypatch): captured = {} - - def fake_run(cmd, env=None, check=None): - captured["cmd"] = cmd - captured["dir"] = env["OPEN_INDEX_DIR"] - - monkeypatch.setattr("subprocess.run", fake_run) + monkeypatch.setattr("open_index.ui.web.serve", + lambda host, port: captured.update(host=host, port=port)) result = run("ui", "--brain", brain_dir, "--port", "9999") assert result.exit_code == 0 - assert "streamlit" in captured["cmd"] - assert "9999" in captured["cmd"] - assert captured["dir"] == str(brain_dir.resolve()) + assert captured["port"] == 9999 + assert os.environ["OPEN_INDEX_DIR"] == str(brain_dir.resolve()) + + +def test_ui_never_advertises_the_bind_address(brain_dir, monkeypatch): + """0.0.0.0 is not connectable; printing it is the same bug `serve` had.""" + monkeypatch.setattr("open_index.ui.web.serve", lambda host, port: None) + result = run("ui", "--brain", brain_dir) + assert "0.0.0.0" not in result.stdout + assert "http://127.0.0.1:8501" in result.stdout + + +def test_ui_reports_missing_extras(brain_dir, monkeypatch): + import builtins + real_import = builtins.__import__ -def test_ui_reports_missing_streamlit(brain_dir, monkeypatch): - def boom(*a, **k): - raise FileNotFoundError + def no_starlette(name, *a, **k): + if name.startswith("open_index.ui.web"): + raise ImportError("no starlette") + return real_import(name, *a, **k) - monkeypatch.setattr("subprocess.run", boom) + monkeypatch.setattr(builtins, "__import__", no_starlette) result = run("ui", "--brain", brain_dir) assert result.exit_code == 1 - assert "Streamlit not installed" in result.output + assert "open-index[ui]" in result.output def test_mcp_stdio_entry_point(brain_dir, monkeypatch): diff --git a/tests/test_ui_app.py b/tests/test_ui_app.py deleted file mode 100644 index a20bb07..0000000 --- a/tests/test_ui_app.py +++ /dev/null @@ -1,325 +0,0 @@ -"""The explorer actually runs. - -Executes the Streamlit script with AppTest, which surfaces exceptions the way a -browser would. These guard the specific complaints that prompted the rebuild: -structure you had to hunt for, and a map that opened blank. -""" - -import os -import shutil -from pathlib import Path - -import pytest - -pytest.importorskip("streamlit") - -from streamlit.testing.v1 import AppTest # noqa: E402 - -from open_index.ui import view # noqa: E402 - -APP = Path(__file__).resolve().parent.parent / "open_index" / "ui" / "app.py" -EXAMPLE = Path(__file__).resolve().parent.parent / "examples" / "support-brain" - - -def _run(brain_dir, monkeypatch): - monkeypatch.setenv("OPEN_INDEX_DIR", str(brain_dir)) - # Generous timeout: each of these boots a real Streamlit script run, and on - # a loaded machine (the full suite, or a CI runner) 60s flaked once. - return AppTest.from_file(str(APP), default_timeout=180).run() - - -@pytest.fixture -def populated(tmp_path, monkeypatch): - from open_index.brain import Brain - - dst = tmp_path / "support" - shutil.copytree(EXAMPLE, dst) - Brain.open(dst).index() - return _run(dst, monkeypatch) - - -@pytest.fixture -def empty(tmp_path, monkeypatch): - root = tmp_path / "fresh" - root.mkdir() - (root / "brain.yaml").write_text("name: fresh\ndescription: New.\n") - return _run(root, monkeypatch) - - -# -- it runs ------------------------------------------------------------------ - - -def test_app_runs_without_exceptions(populated): - assert not populated.exception, [e.value for e in populated.exception] - - -def test_empty_brain_runs_without_exceptions(empty): - """The first thing a new user sees must not be a stack trace.""" - assert not empty.exception, [e.value for e in empty.exception] - - -# -- structure is visible without hunting for it ------------------------------ - - -def test_sidebar_names_the_brain(populated): - assert any("support-brain" in m.value for m in populated.sidebar.markdown) - - -def test_sidebar_lists_every_doc_type(populated): - text = " ".join(m.value for m in populated.sidebar.markdown) - for name in ("issue", "product", "comment", "user_segment"): - assert name in text, f"{name} missing from the sidebar" - - -def test_sidebar_shows_counts_and_storage_policy(populated): - text = " ".join(m.value for m in populated.sidebar.markdown) - assert "entities" in " ".join(c.value for c in populated.sidebar.caption) - assert "file" in text or "index" in text - - -def test_empty_brain_sidebar_explains_what_to_do(empty): - text = " ".join(c.value for c in empty.sidebar.caption) - assert "add-doc-type" in text - - -# -- the map no longer opens blank -------------------------------------------- - - -def test_map_draws_without_any_selection(populated): - """The original complaint: nothing rendered until you made a selection. The - map now opens on the whole index, so there is always something to see.""" - populated.tabs[3].run() # Map - assert not populated.exception - - -def test_map_doc_type_filter_defaults_to_everything(populated): - types = next(m for m in populated.multiselect if m.label == "Doc types shown") - assert set(types.value) == set(types.options) - assert types.options, "every populated doc_type should be filterable" - - -def test_map_filter_lists_only_populated_doc_types(populated, tmp_path): - """A doc_type with no entities would be a dead checkbox.""" - from open_index.brain import Brain - from open_index.schema import DocType - - types = next(m for m in populated.multiselect if m.label == "Doc types shown") - counts = Brain.open(os.environ["OPEN_INDEX_DIR"]).counts() - assert all(counts.get(name, 0) > 0 for name in types.options) - - -# -- explore ------------------------------------------------------------------ - - -def test_search_box_is_present(populated): - assert populated.text_input, "expected a search box" - - -def test_browse_rows_are_rendered_without_a_query(populated): - """Idle state lists entities rather than showing an empty page.""" - assert len(populated.button) > 1 - - -def test_searching_narrows_to_matches(populated): - populated.text_input[0].set_value("payment").run() - assert not populated.exception - labels = " ".join(b.label for b in populated.button) - assert "payment" in labels.lower() - - -def test_opening_an_entity_shows_its_relationships(populated): - populated.text_input[0].set_value("checkout").run() - target = next(b for b in populated.button if "product:checkout" in b.label) - target.click().run() - - assert not populated.exception - text = " ".join(m.value for m in populated.markdown) - assert "Relationships" in text - assert "Checkout" in text - - -def test_back_returns_to_the_list(populated): - populated.text_input[0].set_value("checkout").run() - next(b for b in populated.button if "product:checkout" in b.label).click().run() - next(b for b in populated.button if "back" in b.label).click().run() - assert not populated.exception - - - # -- analytics ------------------------------------------------------------ - - -def test_analytics_tab_renders_on_a_cold_brain(populated): - """No usage recorded yet must be an explanation, not a crash or a blank.""" - text = " ".join(i.value for i in populated.info) + " ".join( - c.value for c in populated.caption) - assert "Analytics" in [t.label for t in populated.tabs] - assert not populated.exception - assert "~/.local/state/open-index/" in text or "recorded" in text - - -def test_searching_is_recorded_as_ui_usage(populated): - """A UI search must show up in analytics, or the usage picture has a hole.""" - populated.text_input[0].set_value("payment").run() - assert not populated.exception - - from open_index.brain import Brain - - import os - summary = Brain.open(os.environ["OPEN_INDEX_DIR"]).analytics_summary() - assert summary["total_fetches"] >= 1 - assert "ui" in summary["by_source"] - - -def test_opening_an_entity_is_recorded_as_ui_usage(populated): - populated.text_input[0].set_value("checkout").run() - next(b for b in populated.button if "product:checkout" in b.label).click().run() - - import os - - from open_index.brain import Brain - - summary = Brain.open(os.environ["OPEN_INDEX_DIR"]).analytics_summary() - assert "get_entity" in summary["by_operation"] - - -def test_empty_brain_explains_the_next_step(empty): - text = " ".join(c.value for c in empty.caption) + " ".join( - i.value for i in empty.info) - assert "no entities" in text.lower() - - -# -- dark mode ---------------------------------------------------------------- - - -def test_row_css_is_theme_agnostic(): - """A hardcoded white row background rendered white-on-white under the dark - theme, which kept its light text — the entity list became invisible.""" - style = view.ROW_CSS.lower() - assert "background:#fff" not in style - assert "background:transparent" in style - assert "color:inherit" in style - - -def test_row_css_inherits_colour_through_the_label_element(): - """Streamlit wraps button labels in

, which otherwise keeps its own - colour and ignores the inherit on the button.""" - assert "> button p{color:inherit" in view.ROW_CSS - - -# -- How to use ---------------------------------------------------------------- - - -def test_help_tab_explains_the_model_before_the_connection_block(populated): - """The vocabulary has to land before "point your agent at this URL" means - anything to a first-time visitor.""" - markdown = [m.value for m in populated.markdown] - headings = [m for m in markdown if m.startswith("###")] - assert any("What an index holds" in h for h in headings) - assert headings.index(next(h for h in headings if "What an index holds" in h)) \ - < headings.index(next(h for h in headings if "Connect an agent" in h)) - - -def test_schema_marks_relationships_optional(populated): - populated.tabs[1].run() # Schema - assert not populated.exception - assert any("Relationships (optional)" in m.value for m in populated.markdown) - - -def test_help_is_the_first_tab_so_it_opens_on_load(populated): - """Streamlit selects the first tab, so a first-time visitor lands on the - explanation instead of having to find it.""" - assert [t.label for t in populated.tabs][0] == view.HELP_TAB == "How to use?" - - -def test_schema_sits_between_help_and_explore(populated): - labels = [t.label for t in populated.tabs] - assert labels[:3] == ["How to use?", "Schema", "Explore"] - - -def test_tab_guide_matches_the_tabs_actually_rendered(populated): - """A stale list of what-each-tab-does is worse than none.""" - assert [t.label for t in populated.tabs] == [n for n, _ in view.TAB_GUIDE] - - -def test_documented_tools_match_the_registered_mcp_tools(brain): - """The tab must not advertise a tool the server does not expose, nor miss one.""" - pytest.importorskip("mcp") - import asyncio as _asyncio - - from open_index.mcp_server import build_server - - server = build_server(brain) - registered = {t.name for t in - _asyncio.new_event_loop().run_until_complete(server.list_tools())} - documented = {name.split("(")[0] for name, _ in view.READ_TOOLS + view.WRITE_TOOLS} - assert documented == registered, ( - f"docs vs server mismatch: only-in-docs={documented - registered}, " - f"only-on-server={registered - documented}") - - -def test_read_only_tools_are_the_documented_read_set(brain): - pytest.importorskip("mcp") - import asyncio as _asyncio - - from open_index.mcp_server import build_server - - server = build_server(brain, read_only=True) - registered = {t.name for t in - _asyncio.new_event_loop().run_until_complete(server.list_tools())} - assert {n.split("(")[0] for n, _ in view.READ_TOOLS} == registered - - -def test_connection_block_is_valid_json_with_the_url(): - import json - - block = json.loads(view.mcp_client_config("https://x.example.com/demo/mcp", "demo")) - assert block["mcpServers"]["demo"]["url"] == "https://x.example.com/demo/mcp" - assert block["mcpServers"]["demo"]["type"] == "http" - - -def test_connection_block_appends_the_mcp_path(): - import json - - block = json.loads(view.mcp_client_config("https://x.example.com/demo")) - assert block["mcpServers"]["open-index"]["url"].endswith("/mcp") - - -# -- one UI process serving many brains ---------------------------------------- - - -@pytest.fixture -def multi(tmp_path, monkeypatch): - from open_index.brain import Brain - - root = tmp_path / "brains" - root.mkdir() - for name in ("alpha", "beta"): - shutil.copytree(EXAMPLE, root / name) - Brain.open(root / name).index() - monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) - monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) - return AppTest.from_file(str(APP), default_timeout=180).run() - - -def test_an_empty_brains_root_explains_itself(tmp_path, monkeypatch): - empty = tmp_path / "none" - empty.mkdir() - monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) - monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(empty)) - at = AppTest.from_file(str(APP), default_timeout=180).run() - assert any("No brains found" in e.value for e in at.error) - - -def test_single_brain_mode_shows_no_picker(populated): - """OPEN_INDEX_DIR alone must behave exactly as before.""" - assert not any(s.label.startswith("Index") for s in populated.sidebar.selectbox) - - -def test_many_brains_render_without_error(multi): - assert not multi.exception, [e.value for e in multi.exception] - - -def test_no_index_switcher_widget_anywhere(multi): - """The URL is the only selector; a dropdown would let the address bar and - the screen disagree about which index you are looking at.""" - assert not multi.sidebar.selectbox diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index 28ff901..2852fdb 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -185,49 +185,6 @@ def test_color_for_unknown_doc_type_is_the_default(brain): # centred because the canvas width was an invalid CSS value. -def test_graph_width_is_an_int_not_a_css_string(): - """streamlit-agraph does f"{width}px", so "100%" becomes "100%px" and the - canvas never sizes — the cause of the off-centre map.""" - assert isinstance(view.GRAPH_WIDTH, int) - assert isinstance(view.GRAPH_HEIGHT, int) - # Reproduce the library's formatting and check it yields valid CSS. - for value in (view.GRAPH_WIDTH, view.GRAPH_HEIGHT): - rendered = f"{value}px" - assert re.fullmatch(r"\d+px", rendered), f"invalid CSS length: {rendered}" - - -def test_dark_theme_labels_are_light(): - dark = view.graph_theme("dark") - light = view.graph_theme("light") - assert dark["node_label"] != light["node_label"] - # A light label on a dark canvas: high channel values. - assert int(dark["node_label"].lstrip("#")[:2], 16) > 0x80 - assert int(light["node_label"].lstrip("#")[:2], 16) < 0x80 - - -def test_label_halo_is_disabled_in_both_themes(): - """vis's default white stroke turns every label into outlined text.""" - for theme in ("dark", "light"): - assert view.graph_theme(theme)["stroke_width"] == 0 - - -def test_unknown_or_missing_theme_falls_back_to_light(): - """Streamlit reports None when the viewer follows their browser setting.""" - assert view.graph_theme(None) == view.graph_theme("light") - assert view.graph_theme("") == view.graph_theme("light") - assert view.graph_theme("solarized") == view.graph_theme("light") - - -def test_theme_lookup_is_case_insensitive(): - assert view.graph_theme("Dark") == view.graph_theme("dark") - - -def test_every_theme_defines_the_full_palette(): - keys = {"node_label", "edge_label", "edge", "stroke_width"} - for theme in ("dark", "light"): - assert keys <= set(view.graph_theme(theme)) - - # -- Schema tab ---------------------------------------------------------------- @@ -297,44 +254,6 @@ def test_help_tab_is_first_in_the_guide(): # unreadable; the full text moves to the hover tooltip. -def test_nodes_carry_no_label(brain): - """Entity names are long and arbitrary; drawn beside every dot they overlap - each other. Identity comes from the tooltip instead.""" - from open_index.graph import build_overview_graph - - specs = view.graph_node_specs(build_overview_graph(brain)) - assert specs - assert all(s["label"] == "" for s in specs) - # Empty string, not None: None serialises to null and vis draws that. - assert all(s["label"] is not None for s in specs) - - -def test_every_node_carries_its_identity_on_hover(brain): - from open_index.graph import build_overview_graph - - specs = {s["id"]: s for s in view.graph_node_specs(build_overview_graph(brain))} - spec = specs["product:checkout"] - assert "Checkout" in spec["title"] - assert "product:checkout" in spec["title"] - - -def test_edges_carry_no_label_but_name_the_relationship(brain): - from open_index.graph import build_overview_graph - - specs = view.graph_edge_specs(build_overview_graph(brain), "#ccc") - assert specs - assert all(s["label"] == "" for s in specs) - assert any("has common issue" in s["title"] for s in specs) - - -def test_node_colour_comes_from_the_doc_type(brain): - from open_index.graph import build_overview_graph - - graph = build_overview_graph(brain) - by_id = {n.id: n.color for n in graph.nodes} - assert all(s["color"] == by_id[s["id"]] for s in view.graph_node_specs(graph)) - - def test_node_tooltip_carries_the_full_name_and_type(): tip = view.node_tooltip("issue:x", "A very long name that got truncated", "issue", {"severity": "high"}) @@ -376,10 +295,6 @@ def test_legend_is_ordered_by_count(brain): assert [r["count"] for r in rows] == sorted([r["count"] for r in rows], reverse=True) -def test_graph_width_fits_beside_the_legend(): - assert view.GRAPH_WIDTH <= 1000 - - # -- the model explanation on the help tab ------------------------------------ @@ -407,67 +322,3 @@ def test_model_guide_distinguishes_schema_from_data(): assert "schema" in text -# -- one UI process, many brains: the URL is the selector ---------------------- - - -def test_the_url_path_selects_the_brain(): - assert view.brain_from_url("https://h/sales-index", - ["support-index", "sales-index"]) == "sales-index" - - -def test_a_deeper_path_selects_on_the_first_segment(): - assert view.brain_from_url("https://h/sales-index/x", - ["alpha", "sales-index"]) == "sales-index" - - -def test_the_root_path_falls_back_to_the_first_brain(): - assert view.brain_from_url("https://h/", ["alpha", "beta"]) == "alpha" - - -def test_an_unknown_path_falls_back_rather_than_erroring(): - """A stale link should land somewhere useful.""" - assert view.brain_from_url("https://h/deleted", ["alpha", "beta"]) == "alpha" - - -def test_a_missing_url_falls_back(): - assert view.brain_from_url(None, ["alpha", "beta"]) == "alpha" - - -def test_a_query_string_does_not_affect_selection(): - assert view.brain_from_url("https://h/beta?x=1", ["alpha", "beta"]) == "beta" - - -def test_no_brains_selects_nothing(): - assert view.brain_from_url("https://h/x", []) is None - - -# -- the endpoint shown on the help tab ---------------------------------------- -# -# A shared explorer cannot be handed one correct MCP URL as configuration: the -# answer depends on which index you are looking at. It is derived from the page -# URL instead, which also keeps it right behind any proxy or hostname. - - -def test_endpoint_is_derived_from_the_page_url(): - assert view.mcp_url_for("https://brain.acme.com/sales-index", "sales-index") \ - == "https://brain.acme.com/sales-index/mcp" - - -def test_endpoint_ignores_the_rest_of_the_path_and_query(): - assert view.mcp_url_for("https://h:8443/sales-index/x?q=1", "sales-index") \ - == "https://h:8443/sales-index/mcp" - - -def test_endpoint_uses_the_directory_name_not_the_configured_brain_name(): - """The URL segment is the directory; brain.yaml's name may differ.""" - assert view.mcp_url_for("https://h/dir-name", "dir-name").endswith("/dir-name/mcp") - - -def test_endpoint_keeps_the_scheme(): - assert view.mcp_url_for("http://localhost:8501/a", "a").startswith("http://") - - -def test_no_endpoint_without_a_url_or_a_name(): - assert view.mcp_url_for(None, "a") is None - assert view.mcp_url_for("https://h/a", None) is None - assert view.mcp_url_for("/relative/path", "a") is None diff --git a/tests/test_ui_web.py b/tests/test_ui_web.py new file mode 100644 index 0000000..5d6de21 --- /dev/null +++ b/tests/test_ui_web.py @@ -0,0 +1,270 @@ +"""The explorer as served — real requests against the real app. + +These replace the Streamlit AppTest suite. They assert what a visitor receives, +which is the thing that broke before: under Streamlit every path rendered +whichever brain sorted first, and a status-code check could not see it because +the shell was returned for any path. Here the index is a route parameter, so the +same question is answerable directly from the response body. +""" + +import shutil +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +from open_index.brain import Brain + +EXAMPLE = Path(__file__).resolve().parent.parent / "examples" / "support-brain" + + +def _fresh_app(): + """Build the app with the module caches cleared. + + open_brain and discover are lru_cached for the life of the process, so a + test that changes the environment would otherwise be served another test's + brain. + """ + from open_index.ui import web + + web.open_brain.cache_clear() + web.discover.cache_clear() + return web.build_app() + + +@pytest.fixture +def single(tmp_path, monkeypatch): + """One brain, served at the root.""" + d = tmp_path / "support-brain" + shutil.copytree(EXAMPLE, d) + Brain.open(d).index() + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.delenv("OPEN_INDEX_READ_ONLY", raising=False) + return TestClient(_fresh_app()) + + +@pytest.fixture +def many(tmp_path, monkeypatch): + """Two brains, each at its own path.""" + root = tmp_path / "brains" + root.mkdir() + for name in ("alpha", "beta"): + shutil.copytree(EXAMPLE, root / name) + Brain.open(root / name).index() + monkeypatch.setenv("OPEN_INDEX_BRAINS_ROOT", str(root)) + monkeypatch.delenv("OPEN_INDEX_DIR", raising=False) + monkeypatch.delenv("OPEN_INDEX_READ_ONLY", raising=False) + return TestClient(_fresh_app()) + + +def an_entity(brain_dir) -> str: + return Brain.open(brain_dir).backend.all_entities()[0].id + + +# -- every page renders ------------------------------------------------------- + + +@pytest.mark.parametrize( + "path", ["/", "/schema", "/explore", "/map", "/analytics", "/jobs"]) +def test_every_tab_renders(single, path): + r = single.get(path) + assert r.status_code == 200 + assert ":`, but a slug containing / must not 404 the route.""" + assert single.get("/entity/issue:a/b").status_code in (200, 404) + + +# -- the map ------------------------------------------------------------------ + + +def test_the_map_page_loads_without_the_graph(single): + """The page is HTML; the graph arrives separately, so a slow graph cannot + block the page from rendering.""" + body = single.get("/map").text + assert 'id="cy"' in body + assert "/api/graph" in body + + +def test_graph_json_has_nodes_edges_and_a_legend(single): + data = single.get("/api/graph").json() + assert data["nodes"] and "legend" in data and "edges" in data + assert all("tooltip" in n for n in data["nodes"]) + + +def test_graph_nodes_carry_no_canvas_label(single): + """Labels are the thing that made the old map unreadable.""" + data = single.get("/api/graph").json() + assert all("label" in n for n in data["nodes"]) # for the tooltip + assert all(n["color"] for n in data["nodes"]) + + +def test_focus_narrows_the_graph(single, tmp_path): + eid = an_entity(tmp_path / "support-brain") + whole = single.get("/api/graph").json() + focused = single.get("/api/graph", params={"focus": eid}).json() + assert len(focused["nodes"]) <= len(whole["nodes"]) + + +def test_graph_json_for_an_unknown_index_is_404(many): + assert many.get("/zzz/api/graph").status_code == 404 + + +# -- read-only ---------------------------------------------------------------- + + +def test_read_only_says_the_write_tools_are_absent(tmp_path, monkeypatch): + d = tmp_path / "b" + shutil.copytree(EXAMPLE, d) + Brain.open(d).index() + monkeypatch.setenv("OPEN_INDEX_DIR", str(d)) + monkeypatch.delenv("OPEN_INDEX_BRAINS_ROOT", raising=False) + monkeypatch.setenv("OPEN_INDEX_READ_ONLY", "1") + body = TestClient(_fresh_app()).get("/").text + assert "read-only" in body + + +# -- escaping ----------------------------------------------------------------- + + +def test_the_inline_markdown_filter_escapes_before_converting(): + from open_index.ui.web import inline_markdown + + out = str(inline_markdown(" **bold** `code`")) + assert " - {% endif %} {% endblock %} diff --git a/open_index/ui/web.py b/open_index/ui/web.py index fd31ecc..e2bba32 100644 --- a/open_index/ui/web.py +++ b/open_index/ui/web.py @@ -91,6 +91,22 @@ def is_read_only() -> bool: return os.environ.get("OPEN_INDEX_READ_ONLY", "").lower() in ("1", "true", "yes") +def directory_hidden() -> bool: + """Whether this host refuses to enumerate the indexes it serves. + + Default off: a self-hosted instance wants a home page listing its indexes. + Set OPEN_INDEX_HIDE_DIRECTORY=1 when the index names are themselves + sensitive — several unrelated tenants or prospects on one host, where + knowing that /acme-index exists is the leak, not its contents. Then only + someone already holding a name can reach it: / is a 404, an unknown name is + a 404 that names nothing, and no page links to a sibling. + + This closes the UI only. A reverse proxy that publishes its own directory + (a generated /indexes.json, say) has to be dealt with there too. + """ + return os.environ.get("OPEN_INDEX_HIDE_DIRECTORY", "").lower() in ("1", "true", "yes") + + def mcp_url_for_request(request, name: str) -> str: """This index's MCP endpoint, from the URL the browser actually used. @@ -120,7 +136,9 @@ def _base_context(request, name: str, brain: Brain, active: str) -> dict[str, An "active": active, "tabs": TABS, "base": f"/{name}" if name else "", - "multi": len(available_brains()) > 1 or bool(name), + # Drives the "all indexes" link, which must not appear on a host that + # refuses to enumerate them. + "multi": (not directory_hidden()) and (len(available_brains()) > 1 or bool(name)), "read_only": is_read_only(), } @@ -338,10 +356,13 @@ def render(request, template: str, ctx: dict): return templates.TemplateResponse(request, template, ctx) def not_found(request, wanted: str): - brains = available_brains() + # The listing is the whole point of hiding the directory: a 404 that + # helpfully names every other index would hand over exactly what the + # flag exists to withhold. + brains = [] if directory_hidden() else sorted( + n for n in available_brains() if n) return templates.TemplateResponse( - request, "missing.html", - {"wanted": wanted, "brains": sorted(n for n in brains if n)}, + request, "missing.html", {"wanted": wanted, "brains": brains}, status_code=404, ) @@ -357,6 +378,10 @@ def root(request): brains = available_brains() if "" in brains: # single brain: serve it here return make(page_help, "help.html")(request) + if directory_hidden(): + # Not a redirect even when there is only one: on a hidden host the + # root must not reveal which index that is. + return not_found(request, "") if len(brains) == 1: return RedirectResponse(f"/{next(iter(brains))}") return templates.TemplateResponse( diff --git a/tests/test_ui_web.py b/tests/test_ui_web.py index 5d6de21..6f858f4 100644 --- a/tests/test_ui_web.py +++ b/tests/test_ui_web.py @@ -268,3 +268,57 @@ def test_entity_content_is_escaped(single, tmp_path): body = single.get("/entity/issue:xss").text assert "