diff --git a/.github/workflows/browser-desktop.yml b/.github/workflows/browser-desktop.yml new file mode 100644 index 0000000..edd8fd2 --- /dev/null +++ b/.github/workflows/browser-desktop.yml @@ -0,0 +1,178 @@ +name: Browser desktop program + +# Builds cfabric-browser as a standalone program (PyInstaller, pywebview) on +# Windows, macOS and Linux, smoke-tests each build, and tests the Windows build +# once more on a runner where Python is out of reach. A tag desktop-v* (such as +# desktop-v0.1.0-test.1) also publishes the builds as a GitHub pre-release, with +# libs/explorer/packaging/RELEASE_NOTES.md as its text. Not browser-v*: that tag +# publishes the package on PyPI (publish.yml). + +on: + push: + branches: [master, main, 'browser-*', 'desktop-*'] + tags: ['desktop-v*'] + paths: + - 'libs/explorer/**' + - 'libs/core/cfabric/**' + - '.github/workflows/browser-desktop.yml' + pull_request: + branches: [master, main] + paths: + - 'libs/explorer/**' + - 'libs/core/cfabric/**' + - '.github/workflows/browser-desktop.yml' + workflow_dispatch: + +jobs: + build: + name: Build on ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - { name: windows, os: windows-latest } + - { name: macos-arm64, os: macos-latest } + - { name: macos-x64, os: macos-15-intel } + - { name: linux, os: ubuntu-latest } + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + working-directory: libs/explorer + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: libs/explorer/frontend/package-lock.json + - name: Build the interface + working-directory: libs/explorer/frontend + run: | + npm ci + npm run build + # the Rust core from PyPI (searches several times faster), not editable: + # PyInstaller does not see editable installs + - name: Install + run: | + pip install "context-fabric==0.6.0rc1" ../mcp ".[build]" + python -c "import importlib.metadata as m; print('context-fabric', m.version('context-fabric'))" + - name: Build the program + run: pyinstaller packaging/cfabric-browser.spec --noconfirm + - name: Smoke test + run: | + program=dist/cfabric-browser/cfabric-browser + [ -f "$program.exe" ] && program=$program.exe + bash packaging/smoke-test.sh "$program" tests/fixtures/display_corpus + - name: Add a translation (downloads the JPS 1917 from Sefaria) + run: | + program=dist/cfabric-browser/cfabric-browser + [ -f "$program.exe" ] && program=$program.exe + # on macOS the program as people start it, inside the app + [ -d "dist/CACCHT Explorer.app" ] && program="dist/CACCHT Explorer.app/Contents/MacOS/cfabric-browser" + bash packaging/translation-test.sh "$program" + - name: Download a corpus (the SP, from its GitHub release) + run: | + program=dist/cfabric-browser/cfabric-browser + [ -f "$program.exe" ] && program=$program.exe + [ -d "dist/CACCHT Explorer.app" ] && program="dist/CACCHT Explorer.app/Contents/MacOS/cfabric-browser" + bash packaging/download-test.sh "$program" + - name: Add a dictionary (BDB, from the release of Context-Fabric/bdb) + run: | + program=dist/cfabric-browser/cfabric-browser + [ -f "$program.exe" ] && program=$program.exe + [ -d "dist/CACCHT Explorer.app" ] && program="dist/CACCHT Explorer.app/Contents/MacOS/cfabric-browser" + bash packaging/dictionary-test.sh "$program" + - name: Serve an AI app (MCP over HTTP and over stdio) + run: | + program=dist/cfabric-browser/cfabric-browser + [ -f "$program.exe" ] && program=$program.exe + [ -d "dist/CACCHT Explorer.app" ] && program="dist/CACCHT Explorer.app/Contents/MacOS/cfabric-browser" + bash packaging/mcp-test.sh "$program" + - name: Size + run: du -sh dist/* + - name: Pack (keeps file permissions) + run: | + mkdir -p ../../out + case "${{ matrix.name }}" in + windows) 7z a -tzip ../../out/CACCHT-Explorer-windows.zip ./dist/cfabric-browser > /dev/null ;; + macos-*) ditto -c -k --keepParent "dist/CACCHT Explorer.app" ../../out/CACCHT-Explorer-${{ matrix.name }}.zip ;; + linux) tar -czf ../../out/CACCHT-Explorer-linux.tar.gz -C dist cfabric-browser ;; + esac + ls -l ../../out + - uses: actions/upload-artifact@v4 + with: + name: cfabric-browser-${{ matrix.name }} + path: out/* + retention-days: 14 + + clean-windows: + name: Run on Windows without Python + needs: build + runs-on: windows-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + libs/explorer/packaging + libs/explorer/tests/fixtures + - uses: actions/download-artifact@v4 + with: + name: cfabric-browser-windows + - name: Unpack + run: 7z x CACCHT-Explorer-windows.zip -oprogram > /dev/null + - name: Smoke test with Python out of reach + run: | + # take every folder with a Python off the path (the py launcher lives + # in C:\Windows), until none is left + export PATH=$(echo "$PATH" | tr ':' '\n' | grep -viE "python|windowsapps" | paste -sd: -) + for name in python python3 py; do + tries=0 + while found=$(command -v "$name") && [ $((tries += 1)) -le 10 ]; do + echo "taking $(dirname "$found") off the path ($found)" + export PATH=$(echo "$PATH" | tr ':' '\n' | grep -vixF "$(dirname "$found")" | paste -sd: -) + done + done + if command -v python python3 py; then + echo "::error title=Clean Windows::a Python is still reachable"; exit 1 + fi + bash libs/explorer/packaging/smoke-test.sh \ + program/cfabric-browser/cfabric-browser.exe \ + libs/explorer/tests/fixtures/display_corpus + + release: + name: Publish a pre-release + if: startsWith(github.ref, 'refs/tags/desktop-v') + needs: [build, clean-windows] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + libs/explorer/packaging/RELEASE_NOTES.md + libs/explorer/packaging/mcpb + sparse-checkout-cone-mode: false + - uses: actions/download-artifact@v4 + with: + path: out + merge-multiple: true + # the Claude Desktop extension, which reaches the running program + - run: python3 libs/explorer/packaging/mcpb/pack.py out/CACCHT-Explorer.mcpb + - run: ls -l out + # desktop-v0.1.0-test.2 is released as "CACCHT Explorer 0.1.0-test.2" + - id: version + run: echo "version=${GITHUB_REF_NAME#desktop-v}" >> "$GITHUB_OUTPUT" + - uses: softprops/action-gh-release@v2 + with: + name: CACCHT Explorer ${{ steps.version.outputs.version }} + body_path: libs/explorer/packaging/RELEASE_NOTES.md + prerelease: true + files: out/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad79dd1..5d640cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,46 @@ jobs: run: pip install -e ".[dev]" - name: Run tests run: pytest tests/ -v + + test-browser: + # with the Python core of this repository, and with the Rust core (0.6) + # from PyPI: the browser uses only the public API of either + name: Test cfabric-browser (${{ matrix.core }} core) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + core: [python, rust] + defaults: + run: + working-directory: libs/explorer + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: libs/explorer/frontend/package-lock.json + - name: Type-check, test and build the interface + working-directory: libs/explorer/frontend + run: | + npm ci + npm test + npm run build + - name: Install the core, and the MCP package (the browser shares its tools) + run: | + if [ "${{ matrix.core }}" = rust ]; then + pip install "context-fabric==0.6.0rc1" + else + pip install -e ../core + fi + pip install -e ../mcp + - name: Install browser package with dev dependencies + run: pip install -e ".[dev]" + - name: Run tests + run: | + python -c "import cfabric, importlib.metadata as m; print('context-fabric', m.version('context-fabric'))" + pytest tests/ -v diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d2a337b..0c357fd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,6 +6,7 @@ on: - 'core-v*' - 'mcp-v*' - 'benchmarks-v*' + - 'browser-v*' jobs: publish-core: @@ -87,3 +88,40 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: twine upload --skip-existing dist/* + + publish-browser: + name: Publish cfabric-browser + if: startsWith(github.ref, 'refs/tags/browser-v') + runs-on: ubuntu-latest + defaults: + run: + working-directory: libs/explorer + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Build the interface into the package + working-directory: libs/explorer/frontend + run: | + npm ci + npm run build + - name: Install dependencies + run: | + pip install build twine + pip install -e ../core + pip install -e ".[dev]" + - name: Run tests + run: pytest tests/ + - name: Build package + run: python -m build + - name: Check that the interface is in the wheel + run: python -c "import zipfile, glob; names = zipfile.ZipFile(glob.glob('dist/*.whl')[0]).namelist(); assert 'cfabric_browser/static/index.html' in names, 'interface missing from wheel'" + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: twine upload --skip-existing dist/* diff --git a/.gitignore b/.gitignore index 0a99881..b144a72 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,8 @@ dist/ # TF-specific .tf/ .cfm/ +# the Rust core's compiled cache (context-fabric 0.6) +.cfr/ .ipynb_checkpoints/ # IDE / Editor @@ -76,3 +78,6 @@ htmlcov/ # Others .private .corpora + +# Built interface (npm run build in libs/explorer/frontend) +libs/explorer/cfabric_browser/static/ diff --git a/libs/core/CHANGELOG.md b/libs/core/CHANGELOG.md index bfd88d3..f6ffb03 100644 --- a/libs/core/CHANGELOG.md +++ b/libs/core/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- `T.sectionFeatureTypes` was `str` for every level on the first load of a corpus (from `.tf` files, before the `.cfm` cache exists), so numeric chapters and verses could not be looked up from references like `Genesis 1:1` in that session +- `S.search` / `S.fetch` docstrings had the generator-vs-tuple behaviour of `limit` the wrong way round + ## [0.5.8] - 2026-10-14 ### Added diff --git a/libs/core/cfabric/navigation/text.py b/libs/core/cfabric/navigation/text.py index 2b2aa74..b43a9eb 100644 --- a/libs/core/cfabric/navigation/text.py +++ b/libs/core/cfabric/navigation/text.py @@ -471,7 +471,13 @@ def __init__(self, api: Api) -> None: # Get dataType from api.F feature or CF.features fFeature = getattr(api.F, fName, None) if fFeature is not None and hasattr(fFeature, 'meta'): - dataType = fFeature.meta.get('value_type', 'str') + # .cfm metadata says `value_type`; metadata read from .tf + # files (the first load, before compiling) says `valueType` + dataType = ( + fFeature.meta.get('value_type') + or fFeature.meta.get('valueType') + or 'str' + ) else: fObj = api.CF.features.get(fName) dataType = fObj.dataType if fObj else 'str' diff --git a/libs/core/cfabric/search/search.py b/libs/core/cfabric/search/search.py index 94ec882..3bc5b6d 100644 --- a/libs/core/cfabric/search/search.py +++ b/libs/core/cfabric/search/search.py @@ -264,7 +264,7 @@ def search( Each result is a tuple of nodes, where each node corresponds to an *atom*-line in your search template (see `tf.about.searchusage`). - If `limit` is not `None`, a *generator* is returned, + If `limit` is `None`, a *generator* is returned, which yields the results one by one. Otherwise, the results will be fetched up till `limit` @@ -415,7 +415,7 @@ def fetch( Each result is a tuple of nodes, where each node corresponds to an *atom*-line in your search template (see `tf.about.searchusage`). - If `limit` is not `None`, a *generator* is returned, + If `limit` is `None`, a *generator* is returned, which yields the results one by one. Otherwise, the results will be fetched up till `limit` diff --git a/libs/explorer/CHANGELOG.md b/libs/explorer/CHANGELOG.md new file mode 100644 index 0000000..4f96bbf --- /dev/null +++ b/libs/explorer/CHANGELOG.md @@ -0,0 +1,140 @@ +# Changelog + +All notable changes to cfabric-browser will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- The command `caccht-explorer`, the same as `cfabric-browser` under the app's name. + +### Changed +- The extra `desktop` is split in two: `window` (pywebview and certifi, for `--window`) and `build` (also PyInstaller, for building the standalone program), so that installing the Explorer with its window doesn't bring PyInstaller. +- Cleaner code (from the code review, round 3), with the same behaviour: + - The API is in `cfabric_browser/routes/`, one module per area (`library`, `reading`, `parallel`, `search`, `proposals`, `lexicon`) on a shared `Context`, instead of one function of 1,700 lines in `api.py`; the OpenAPI schema is unchanged. + - `net.py` (certificates, requests, checksums, the checked download) and `words.py` (word type, the words of a node, the original-script format, `BookSpans`) replace code that was written two or three times. + - `CorpusRegistry` keeps only what the browser and `cfabric-mcp`'s tools use (`register`, `get`, `get_api`, `list_corpora`, `current`). + - Interface: one hook for state in the URL (`shared/urlState.ts`, five copies before; the history is no longer written from inside a state update), one for the server's description (`shared/health.ts`, four copies), one for closing menus on a click outside or Escape (`shared/popover.ts`, six copies); corpus cards are fetched once per corpus and shared by all interfaces and by switching (`corpusCard`, one cache in `queryClient.ts`); unused code removed. + - More tests of the search pool: cancelling queued and running searches, and which finished searches are kept. +- Faster (from the code review, round 2): + - The interface loads Panes and the template editor (CodeMirror) only when they are first shown: the code loaded at the start is 278 kB instead of 792 kB. + - Hovering over a word no longer draws the whole text again (Reader, Interlinear, Parallel, Panes, Concordance): only the card follows the hovered word (`HoverState`, `Hovering`). + - What was fetched of a corpus stays good while the program runs (`staleTime`), so opening a view, pane or panel again doesn't fetch it again. Searches, the library, downloads, translations and dictionaries are still asked again when shown. + - The Concordance adds the new results of a running search instead of starting again with every batch, and is built anew only once they are sorted; the eight kept are the last used. + - The innermost nodes of a result (the hit of the Concordance, the highlights of Parallel's search results) are found from each node's first and last slot instead of sets of all its slots: for `book` / `word` 3,000 results take 0.2 s instead of 110 s (SP). + - The three-way comparison aligns each scroll once per verse instead of once per place. + - The DSS book index asks for each scroll's name once instead of for each word (1.3 s instead of 7.7 s). + - A corpus' display settings, book list, book index and lexicon are built once when several requests ask for them together. +- No more secret: `~/.cfabric/running.json` holds only the address of each running Explorer, and `POST /api/proposals` needs no `X-Cfabric-Secret`. The secret protected only that route (not `/mcp`, contrary to the documentation); other web sites are now kept out of all routes by the check of origin and host, and a secret that changes at every start can't be given to apps that connect by address (Claude Code, Codex). +- `/where/ … /have/` was explained wrongly in the BHSA guide for models, in the user guide (*Searching*) and in `cfabric-mcp`'s syntax guide: an object under `/have/` written as a new line (`word sp=nmpr`) is another object, so *subjects that are only proper names* found the subjects with at least one (10,484). The guides now name the object (`w:word`, then `w sp=nmpr`; 5,893), give `/without/ word sp#nmpr` as the shorter form, and say that `/where/` also lets through what has no such object. In the question set, *noun phrases that consist only of nouns* had the same mistake (now 22,447), and *verses with >HB[ and JHWH/* counted pairs of words instead of verses (now 60); three requests got other readings that count as right too (`also`). +- Asking for a search from ChatGPT and Codex (groundwork; `chatgpt-plan.md`, phase 2): `cfabric-browser mcp` (and the Claude Desktop extension's proxy) answers `initialize`, `ping` and the lists of tools, prompts and resources itself, from the same server definition, and finds or starts the Explorer in the background, so an app that gives a server only seconds to start (Codex: 10) no longer gives up while the Explorer starts; only a tool call waits for it. All tools but `propose_search` are marked read-only (MCP tool annotations), so apps that ask before tools that change something ask only for the proposal. A proposal without `model` is named after the app that sent it (its `clientInfo.name`, passed on by the helper in `X-Cfabric-Client`, or its User-Agent): *Proposed by Codex*, *ChatGPT*, *Claude*. A trial search that runs out of its 20 seconds says so and how to narrow the template, in `try_search` and `propose_search`. +- The menu no longer names the design the Parallel interface follows. +- The app is called CACCHT Explorer: next to its mark, in page and window titles, the macOS app (`CACCHT Explorer.app`), the downloads (`CACCHT-Explorer-*.zip`, `CACCHT-Explorer.mcpb`), the release titles, the MCP server's name, the citation and the messages. The package, command and module stay `cfabric-browser` / `cfabric_browser`; Context-Fabric stays the name of the core. The code moved from `libs/browser` to `libs/explorer`. +- Parallel's *Search* opens on *Template*, which now comes first, instead of *Search terms*; adding a search term from the word panel still switches to the terms. +- The browser works with either core, the Python one (`context-fabric` 0.5) and the Rust one (0.6.0rc1), and uses only their public API; CI tests it with both. Its displays (`cfabric_browser.display`), corpus registry (`cfabric_browser.registry`) and search cancel token (`cfabric_browser.cancel`) moved here from the core, where they were only on the browser's branches, with their tests and fixtures (`tests/fixtures`). Searches run through `S.search`: the worker checks the cancel token and the timeout between results, and ends a worker that doesn't stop in time. Templates are checked by the Python engine's cheap phases, or by `S.study` on the Rust core; the features per atom (export columns, statistics) come from a small parser of templates of the browser's own (`queries.template_atoms`), identical to the Python engine's on 81 templates. +- The standalone program is built on the Rust core (`context-fabric` 0.6.0rc1 from PyPI): searches are several times faster, and the first opening of a corpus prepares it in less time (the BHSA in 73 s instead of 186 s, here). It prepares a `.cfr` cache next to the corpus; a `.cfm` cache of the Python core is left alone. + +### Fixed + +- Other web sites could send the server requests (start a download or a build, and with a name pointing at 127.0.0.1 read the API). The server now refuses requests from another origin, and, when it listens only on this computer, requests under other host names (`cfabric_browser/guard.py`, `create_app(hosts=…)`). +- Searching: the watchdog could give the next queued search to the worker it was about to end (the search then failed with "The search process stopped unexpectedly"), and could mark a search that had just finished as timed out and end a healthy worker. A corpus that a worker couldn't load made the pool start new workers without end; now its searches fail with the reason, and workers that keep dying before they are ready are not started again after three tries. +- `/api/corpora` waited for a corpus that was still being prepared (minutes for the BHSA) to get its title; the title now comes from its app config. +- A translation build that failed at its last step lost the earlier translation, and a build could replace a folder of the same name that it hadn't made; the half-built folders (`.jps1917.building`) showed up in the library. Cancelling after the work was put in place reported it as cancelled. +- A download that was cut off (not retried) or cancelled left its partial file behind. +- `cfabric-browser mcp` started the Explorer again for every message after a start had failed; it now waits five minutes before trying again (a running Explorer is still found at once). +- The lexicon's caches are filled under a lock (a gloss could go missing, now and then, when two requests came at once). +- The Reader showed the wrong chapter after › then ‹ (or the keys, or Back) when a verse had been selected; Reader, Interlinear and Concordance stayed at *Opening…* when the URL named a corpus that isn't open (an old link), and now open the reference corpus; the Interlinear jumped back to the URL's verse whenever a setting changed; the Concordance and Parallel's search results first fetched the old page after a change of order, filter or search; Panes and Parallel's search kept the recent searches and problems of another corpus, and closing a pane gave its state and notes to the next one; a failed switch of interface did nothing visible. + +- The guide's setup for ChatGPT (since test.15) asked for the program's path and `mcp` as an argument in the desktop app, whose form may have no field for arguments, and on a Mac the path has a space. It now gives the address (*Streamable HTTP*, `http://127.0.0.1:8107/mcp`) for the desktop app and `codex mcp add --url` for the terminal; the full `config.toml` block (`command`, `args`, `tool_timeout_sec`) for letting ChatGPT start the Explorer, with the Mac path and how to open the hidden `~/.codex`; and that the desktop app may offer it only when working with Codex. + +- In the Reader, the *Workspaces*, *Display* and *Cite* menus opened behind the panel on wide screens. +- Hovering over a word of the SP showed a single letter (`sign`) unless the user had a local display file saying that words are its base type. The browser now has display settings of its own (`cfabric_browser/display_settings/`), layered under the local ones in `~/.cfabric/display`: for the SP, words as units, and for 7.2.4 the phrase labels and hidden atoms. +- `pyyaml` is a dependency of the browser's own: it was only there because the Python core brought it along, and the Rust core doesn't. + +### Added +- The guide's chapter *Asking Claude* is now *Asking Claude or ChatGPT*, with how to connect the ChatGPT desktop app and Codex (not yet tried with ChatGPT itself); the release notes for test.15. +- `scripts/ask_questions.py`: asks an AI app (`claude -p` or `codex exec`, with the user's own subscription) each request of the question set through a running Explorer, runs each proposal in full and scores it (proposed, right count, trials, seconds); not in CI. Claude Code (its default model, 2026-10-21): 30 of 30 right after the corrections above (25 before, all five misses due to the question set). +- The guide's chapters *Getting started*, *The datasets*, *The five ways of working*, *Reading and comparing*, *Words*, *Searching* (a short course in templates with counts from the BHSA guide), *The Concordance*, *Asking Claude* (the setup, moved from the release notes), *Adding texts*, *Notes, workspaces and citing* and *Questions and problems*; the release notes are shortened to what is new, the downloads and how to start, with links to the guide; with screenshots made by `frontend/scripts/screenshots.mjs` from the running program. `npm test` checks that every label the guide names (in italics) is in the program, and the README has a checklist for the guide, screenshots, release notes and caccht.github.io before each desktop release. +- A user guide (chapters still being written): Markdown in `guide/`, made into pages by `frontend/scripts/guide.mjs` during `npm run build`, served at `/guide/` (offline), with *Guide to …* in the interface menu; `--site DIR` writes the same pages for caccht.github.io. The packaging smoke test checks that the program serves it. +- Searching in plain language, the BHSA guide (`assistant/guides/bhsa.md`, given by `corpus_guide`): object types, the main features and their values in plain words, conventions (ETCBC transliteration, `[` and `/` endings, `=` for homonyms, Latin book names), templates in short, 42 worked examples with their counts, and common mistakes. A question set of 30 requests with correct templates and counts (`tests/data/bhsa-questions.yaml`); `tests/test_assistant_questions.py` checks both against the real BHSA with `CFABRIC_REAL_CORPORA=1`. +- Searching in plain language, connecting: the browser is an MCP server at `/mcp` (Streamable HTTP, stateless, JSON; only on 127.0.0.1, no web page as origin) with the tools `list_corpora`, `corpus_guide`, `describe_corpus`, `list_features`, `describe_feature`, `search_syntax_guide`, `check_template`, `try_search` and `propose_search`. `cfabric-browser mcp` is the same over stdio for apps that start their own servers (Claude Desktop): it passes each message on to the running browser and starts one if none runs; the standalone program takes `mcp` too. A Claude Desktop extension (`packaging/mcpb`, `Context-Fabric.mcpb` in the desktop release) does the same in Node.js. README: how to connect Claude Code, Claude Desktop and Antigravity CLI. +- Searching in plain language, proposals: a search proposed by a language model (`propose_search`, or `POST /api/proposals` with the browser's secret) appears as a card above the search form in Panes, Parallel and the Concordance, with the user's request, the template, the explanation and the trial's count, and *Run* / *Edit* / *Dismiss*; nothing runs by itself. The page follows proposals live (`/api/proposals/events`). Where a search came from (request, model, time, the proposed template) stays with it in the URL (`a0`, `ask`), in workspaces, when switching interfaces and in the citation, marked *(edited)* when the template was changed. The browser writes its address and a new secret to `~/.cfabric/running.json` at start, for a model's app to reach it. Monospace text no longer joins `=>` into an arrow. +- Searching in plain language, groundwork (`cfabric_browser/assistant/tools.py`): the tools a language model gets on the browser's open corpora: describing corpora and features and the template language (shared with `cfabric-mcp`), `check_template`, and `try_search`, a trial through the search pool (count up to 10,000 within 20 s, a few examples with their passages, and for an empty result the `feature=value` conditions whose value the feature never has). The browser now depends on `cfabric-mcp`. +- Linking the BHSA's lexemes to entries of BDB (`cfabric_browser/dictionaries/link.py`), the groundwork for BDB in the lexicon: the words of each verse are aligned with the Open Scriptures Hebrew Bible by their consonants, and a lexeme gets the BDB entries its words' lemmas lead to through Open Scriptures' lexical index. +- A lexeme's page for a frequent word (הלך, 1,547 occurrences) opened after about 20 s, because each word's book was looked up with `L.u`; the books are now found by slot range (0.5 s). +- BDB: a long entry (הלך, דבר) shows its stems folded under their first words. +- BDB elsewhere: the lexicon's *Find* searches BDB's short definitions, its rows show them, and it can be ordered *By root (BDB)*; the hover card and the inspector (*BDB entry*) give a word's entry, DSS words through the BHSA's lexeme; the Reader's *Lexicon* tab shows the definitions; the Interlinear has a line *BDB*. `bdb` in `/api/corpora/{c}/nodes/{n}/summary` and `/info`. +- BDB in the lexicon: a lexeme's page has a tab *BDB* with its entries (the text with verses that open the passage, references to other entries, abbreviations explained, neighbours, root family, Strong's and TWOT, a link to Sefaria). Added and removed in the library under *Dictionaries* or with `cfabric-browser add bdb [FILE]` / `remove bdb`; `/api/dictionaries`, `/api/dictionaries/{d}/entries/{id}`, and `bdb` in `/api/corpora/{c}/lexicon/{n}`. The file is published as release `v1.0` of `Context-Fabric/bdb` (CC BY-NC 4.0) and checked by its checksum; `add bdb FILE` adds one built here. +- Building the BDB dictionary file (`cfabric_browser/dictionaries/build.py`, run once for publishing): the text of every entry from Sefaria's *BDB Dictionary* and *BDB Aramaic Dictionary*, fetched one entry at a time with a pause between requests and kept in a checksummed cache (`sefaria.py`); joined to Open Scriptures' ids, parts of speech and short definitions by BDB's order and headwords; HTML reduced to a few tags, with references to verses as OSIS ids and to other entries as their ids (`bdb.py`); the lexemes linked, with corrections by hand in `corrections.yaml`; BDB's abbreviations. Joins by hand in `joins.yaml` for entries spelled otherwise and for neighbours with the same consonants that the join takes for each other; entries whose text stops short at Sefaria are marked `incomplete`, and the two entries Sefaria lacks get Open Scriptures' headword and short definition. Links checked against the verses BDB cites (`by_citations`): another entry of the same language and word class replaces the linked one when it cites the lexeme's verses clearly better; cross references and roots don't replace others. +- Downloading corpora (`cfabric_browser/downloads.py`): the BHSA (2021), the SP (7.2.4) and the DSS (2.0.1) from their own releases on GitHub, the files Text-Fabric uses, pinned by checksum. They are unpacked into `~/text-fabric-data/github` in Text-Fabric's layout with its `__checkout__.txt` files; names outside the corpus' folder are refused, and data folders already on the computer are left alone. *Open a corpus* lists those that are missing under *Can be downloaded*, with their size, licence (CC BY-NC 4.0 for all three) and terms of use; *Download* shows the progress and opens the corpus when it is there. Also `cfabric-browser download [NAME ...]` and the endpoints `/api/downloads`. The background jobs of translations and downloads share `cfabric_browser/jobs.py`. +- `--ask-downloads` (used by the standalone program): a first screen, *Getting the corpora*, offers the missing BHSA, SP and DSS with one *Download* button (nothing is downloaded without it; *Not now* is remembered), shows each corpus being downloaded, unpacked and prepared, and then goes on into the interface. *Cancel* stops the downloads; without a connection it offers *Try again*. A downloaded corpus is opened and loaded once right away, which compiles it (minutes for the BHSA and the DSS), so that later starts take seconds. `DELETE /api/downloads/{org}/{repo}` cancels a download; `askDownloads` in `/api/health`. `packaging/download-test.sh` downloads the SP in the standalone program with an empty home folder; CI runs it on all four builds. + +### Changed +- Only two text formats: the BHSA's *Vocalized* (`text-orig-full`: vowels and accents, the qere where it differs) and *Unvocalized* (`text-orig-plain`: the ketiv's consonants); the other corpora only their full text, so *Display* offers a choice of text only for the BHSA. Transliterations, lexeme formats, the ketiv format and the DSS' word-level and source formats are no longer offered; a request for one of them (an old link, a saved workspace, `fmt=` in a download) gives the corpus' default text (`cfabric_browser/formats.py`). The comparison and the Concordance still choose their consonants from all of a corpus' formats. +- *Parallel* shows the reference corpus (the BHSA) as the first column when it chooses the texts itself, since the others are compared with the first. +- Translations as corpora, starting with the JPS 1917 (`cfabric_browser/translations/`): the builder downloads the 39 books from Sefaria's export, checks the title, the licence and the number of chapters and verses against the BHSA, removes the markup (a verse repeated in small type goes into `note`), and writes a corpus with words, verses, chapters and books (`text`, `trailer`, `lower`). Where the 1917 print divides verses differently (the Ten Commandments in Exodus 20 and Deuteronomy 5, and Numbers 25:19), the verses are divided as in the BHSA, and `printed` keeps the print's number. +- *Translations* in *Open a corpus*: *Add* downloads and builds a translation, shows the progress and opens it; *Remove* deletes it (at the next start when it is open). Also `cfabric-browser add [ID]` and `cfabric-browser remove ID`, and the endpoints `/api/translations`. The standalone program offers the JPS 1917. +- Translations in the interfaces: an English column in *Parallel*, *Panes* and the Reader, with the notes printed after verses and the print's verse numbers where it divides differently; a line per verse under the Interlinear (*Under*, heading *Translations*); English searches in the Concordance, sorted and collocated by `lower`; *Cite* names the edition. Translations are left out of the differences, and the hover card stays away for their words. Corpus cards and `/api/corpora` say whether a corpus is a translation (`translation`); `/passage` has `notes` and `printeds`, `/parallel` cells `note` and `printed`, and the Interlinear's `under` is a line (`translation`, `text`, `note`) for a translation. +- *Cite* in the Reader and the Interlinear (the chapter, or the verse chosen). +- *Concordance*, the fifth interface, after Sketch Engine and AntConc: + - search results as lines with the hit in context, sorted by the hit or the words around it; + - facets with counts that filter; + - one line per verse, random samples, hits per 10,000 words, download of the lines; + - collocations by logDice; + - statistics. + - New endpoints `/kwic`, `/kwic/facets` and `/collocations` under `/api/searches/{id}` (`cfabric_browser/concordance.py`). Values such as `NA` count as missing. +- *Interlinear*, the fourth interface, after STEPBible and BibleHub: + - word blocks with lines of choice and order (transliteration, lexeme, gloss, part of speech, morphology, any feature); + - another text word by word under them, with differences; + - phrases and clauses with their function and type; + - practice by hiding a line; + - colours by part of speech or stem. + - New endpoint `/api/corpora/{c}/interlinear` (`cfabric_browser/interlinear.py`), and `align_words` in `compare.py`, which pairs the words of two texts on their consonants. +- *Reader*, the third interface, after Sefaria: + - one text in a quiet column, line by line or as running text in paragraphs (petuhah, setumah), chapter after chapter as one scrolls; + - a panel about the clicked verse: *Parallels* (with differences), *Words*, *Lexicon*, *Search* (in *Parallel*) and *Notes* (the user's own notes and bookmarks, kept locally and in exported workspaces); + - on a phone the panel opens from the bottom. +- An interface menu instead of the two-way switch, ready for five interfaces (*Panes*, *Parallel*, and the planned *Reader*, *Interlinear* and *Concordance*). Switching goes through one shared description of what is shown (`common.ts`), so that what one interface cannot show is kept for the next. Interfaces can be left out of the menu (*Choose which to offer…*, or `ui: hide:` in `browser.yaml`); `--ui` takes all five names, with `panes` for the classic interface. `/api/health` gives `hiddenUis`. Frontend tests with Vitest (`npm test`, also in CI) +- A second interface, *Parallel*, after Parabible's layout (`/Genesis/1?texts=bhsa,sp,dss`): a top bar with the chapter, its neighbours (also ←/→) and a drawer with the books, and a column per corpus that can be turned on and off, ordered and added from the library. Chapters follow the BHSA's numbering, a row per verse with a cell per corpus (`/api/parallel`); the DSS shows each scroll's text of the verse. Hovering over a word shows its card; clicking opens a word panel on the left (the inspector, with *Lexicon entry* for BHSA words). *Lexicon* in the top bar shows the lexicon; its verses open their chapter with the verse marked. *Differences* marks the words that differ from the first column, and with MT, SP and DSS, which scrolls agree with which per place. *Search* builds a search from words (*Add as search term* in the word panel: tick the features that count, choose verse, sentence, clause or phrase, and books) or takes a template; the results show each verse in every text (`/api/searches/{id}/parallel`), with statistics, download and citing as in *Panes*. Section items in `/sections` now carry their feature and its own value (`book`, `Jesaia`). *Display*, *Workspaces* (saved with their interface; loading one switches to it) and *Cite* (a citation per text) in its top bar; on a phone the texts of a verse are stacked. The standalone program starts in *Parallel*. A switch in both top bars goes between *Panes* and *Parallel* with the same passage and corpora; the last one used is remembered, `--ui` (or `ui:` in `~/.cfabric/browser.yaml`) sets the first. Corpus cards now have an `id` (`ETCBC/bhsa`) and `/api/health` the default `ui`. +- Hovering over a word shows a card with its essentials: gloss, lexeme, transliteration, part of speech and morphology (stem, tense, person, gender, number, state, suffix), in words rather than codes, read from the first feature of each role with a value, so also in the SP and the DSS, where the ETCBC's analysis (`_etcbc` features) comes first and the DSS's own fills the gaps; DSS words get their gloss from the BHSA's lexicon by their ETCBC lexeme (`lex_etcbc`) when the BHSA is open (`/api/corpora/{c}/nodes/{n}/summary`, `cfabric_browser/summary.py`). Clicking still opens the inspector, which now starts with the same essentials and folds the other features away; empty values and the `g_` features are left out unless asked for +- Lexicon, for a corpus with lexeme nodes and glosses (BHSA), after SHEBANQ's *Words* page: a third view besides *Read* and *Search*, with the lexemes per language and first letter (vocalised form, transliteration, gloss, part of speech, frequency), finding across all letters by Hebrew, transliteration or gloss, a filter by part of speech and sorting by frequency. A lexeme shows every verse it occurs in with its words marked, its distribution over the books (choose one to see only its verses), a search for it, and a TSV download of its occurrences (`/api/corpora/{c}/lexicon`, `/lexicon/{n}`, `/lexicon/{n}/occurrences`, `/lexicon/{n}/occurrences.tsv`); the lexeme is kept in the URL (`lx0=`) +- `--only ORG/REPO` (and `library: only:` in `~/.cfabric/browser.yaml`): limit the library to a few corpora, which are then also the only ones that can be opened from the browser. The standalone test version offers only BHSA, SP and DSS this way +- Program icon for the standalone program (Windows `.ico`, macOS `.icns`): the Context-Fabric mark in gold on a dark tile, drawn from `favicon.svg` by `packaging/make_icons.py` +- Standalone program: output and errors go to `~/.cfabric/logs/cfabric-browser.log`, with a message on Windows when it cannot start; without a window toolkit it opens a web browser instead. `--no-window` runs it as a plain server. Built and smoke-tested on Windows, macOS and Linux by the `browser-desktop.yml` workflow (`packaging/smoke-test.sh`) +- `--window`: the browser in a window of its own (pywebview) instead of a web browser; closing it stops the server. The window keeps its web storage (settings, saved workspaces) in `~/.cfabric/window`, and takes the first free port from 8107 so that it keeps the same address. The standalone program opens this window, without a console +- Start without corpora: `cfabric-browser` on its own opens an empty browser that offers the corpora on this computer (*Open a corpus*) straight away; the first corpus opened goes into the first pane. With `--no-library`, a corpus is still required +- Standalone program: `packaging/cfabric-browser.spec` builds `cfabric-browser` with PyInstaller into a program that needs no Python installation (`pip install .[desktop]`); `python -m cfabric_browser` also starts the browser +- `cfabric-browser` command: serve one or more corpora given by name (`bhsa`), repository or data directory +- JSON API: corpora, features, section tree, passages and nodes as render trees, node inspector, template check, search with live status events, paging, results in context, TSV export +- Searches in worker processes: parallel, cancellable, with time limits and a result cap; results stream in and are put in canonical order when done +- TSV export of all results: features mentioned in the template become columns per result node (`sp*` adds a feature without narrowing the search, as in Text-Fabric); `?excel=true` gives a UTF-16 file with byte-order mark that Excel opens correctly +- Several corpora side by side: up to four panes, each with its own corpus, view, passage and search, all kept in the URL +- Synchronised reading: opening a passage in one pane opens the corresponding passage in the others, through a concordance of book names (`1_Samuel`, `1Sam`, `Samuel_1`); where a verse is missing in a corpus, its chapter is shown with a note; reading panes without a passage (a short link such as `?c0=bhsa&ref0=Genesis+1&c1=sp&sync=1`, or a new pane) open the passage of the first pane that has one +- Search all corpora: run a template in every open corpus and compare the counts; each opens in its own pane +- Choose a corpus version with `@`: `DT-UCPH/sp@7.2.4`, or `@latest` for the newest on this computer (by default the version named in the corpus' app config); the corpus card reports the version actually loaded +- Reading by biblical book in a corpus of manuscripts (DSS), whose sections are scroll, fragment and line: an index of the words' `book`/`chapter`/`verse` gives per verse the text of every scroll that contains it (`/books`, `/witnesses?ref=Genesis 1`); synchronised reading aligns it with MT and SP (`align?view=books`). Reconstructed text is flagged (`dim`), also in the scroll view, and the DSS placeholders for lost text (`ε`) and illegible letters (`#`, `?`) are shown as in the editions, `[…]` and `◦`, while the transcription's morpheme breaks (`׳`, as in `עמ׳י`) are left out; in comparisons, an illegible letter counts as unknown; text the DSS also lists as non-biblical (`biblical=2`) is labelled. With Show differences, each scroll is compared with the MT or SP pane: reconstructed letters don't count, nor does what is lost at the edges of a fragment or next to a reconstruction; the MT/SP pane marks a word where any scroll differs, with a tooltip naming the scrolls (`compare?bview=books`) +- Three-way comparison: with MT, SP and a DSS pane in book view open and Show differences on, each place where MT and SP differ shows which scrolls side with MT, which with SP and which with neither, read only where a scroll preserves the text; as a line per place under each verse in the DSS pane, and in tooltips on the marked words (`/api/agreement`) +- Show differences: where reading panes show the same passage of two Hebrew corpora (e.g. MT/BHSA and SP), the words in which the texts differ are marked, compared on their consonants verse by verse. Vowels, accents and word division don't count; a difference in matres lectionis only is marked as spelling, any other as a variant; a plus is marked in the text that has it. Kept in the URL (`diff=1`) +- Clicking a word in the text opens the word in the node inspector, also where slots are signs (SP, DSS): signs within a word are no longer inspected one by one, and words shown as one unit in the pretty view are clickable +- Local display settings in `~/.cfabric/display//[@].yaml`, layered over a corpus' app config +- Cite: a citation to copy for the passage being read or for a search (template and number of results), with the corpus' title, version and DOI and the software versions and date; `/api/health` reports the context-fabric version +- Query editor: completes feature values after `feature=` (also after `|`), most frequent first with their counts; hovering a feature shows its description, the object types it is on and its most frequent values; hovering an object type shows how many there are +- Saved workspaces: save the current panes, passages and templates under a name in this browser, open or delete them, and export or import them as files; importing a Text-Fabric browser job opens its query (and runs it) and its first passage +- A pane whose search the server no longer has (after a restart, or from an old link) searches again with its template +- Download dialog: choose extra feature columns per part of the results, besides those in the template, and TSV or TSV for Excel (`export.tsv?extra=3:lex,gloss`) +- Statistics on search results: how often each value of a feature occurs on one part of the results (by default a feature the template leaves open, such as `lex*`), and how the results spread over books or chapters; with a table view, and links to read each section +- Library list: copies of one corpus (same title, e.g. a fork) are listed once, keeping the newest data; each entry shows the version that opens; corpora can be hidden with `library: hide:` in `~/.cfabric/browser.yaml` or `--hide` +- Corpus library: open other corpora found on this computer while the browser runs (`--no-library` switches this off) +- Template problems as structured data (line, message, feature or node type), including problems inside quantifiers that the search engine only logs + +### Fixed +- Standalone program on Windows: it didn't start when unpacked with Explorer from the downloaded zip, because .NET Framework refuses assemblies marked as downloaded (`Zone.Identifier`) and the window needs one (pythonnet's `Python.Runtime.dll`). The program now takes that mark off its own libraries at startup. Any failure of the window now falls back to a web browser. +- Switching between interfaces: + - the Reader, the Interlinear and the Concordance passed on only their own corpus when no other texts were chosen; + - a search in the reference corpus' pane no longer pushes out the only pane with the passage (*Panes*); + - going through *Panes* no longer forgets the passage or the search of the interface before; + - an empty Concordance doesn't open the search panel in *Parallel*. +- The Concordance on a phone: the facets fold away behind *Filters*, and the page scrolls as a whole. +- Switching from *Panes* to *Parallel* could leave an empty page when the panes showed exactly the open corpora, in their order: the two interfaces shared a cache entry for the corpus cards diff --git a/libs/explorer/README.md b/libs/explorer/README.md new file mode 100644 index 0000000..c17871b --- /dev/null +++ b/libs/explorer/README.md @@ -0,0 +1,357 @@ +# CACCHT Explorer (cfabric-browser) + +CACCHT Explorer is a graphical browser for [Context-Fabric](https://context-fabric.ai) corpora, the successor of the Text-Fabric browser. It runs on your own computer and works offline. + +The package, its command and its module keep their technical names: `cfabric-browser` and `cfabric_browser`. The code is in `libs/explorer`. + +It works with either Context-Fabric core: the Python one (`context-fabric` 0.5) and the Rust one (0.6), which searches several times faster. It uses only their public API. + +> **Status: pre-alpha.** Reading, searching and inspecting work, also with several corpora side by side. + +**Using the program** is explained in the user guide: [Guide to CACCHT Explorer](https://caccht.github.io/explorer/) on caccht.github.io, also built into the program at `/guide/` (written in `guide/`; see *The user guide* below). Most users have the standalone program from the [desktop pre-releases](https://github.com/Context-Fabric/context-fabric/releases). This README is for installing it with pip, its options, the API and development; [context-fabric.ai](https://context-fabric.ai/docs/explorer) has an overview for programmers. + +## Usage + +```bash +pip install cfabric-browser +cfabric-browser bhsa +``` + +`cfabric-browser` is not on PyPI yet; until it is, install it from this repository: `pip install "context-fabric==0.6.0rc1" ./libs/mcp "./libs/explorer[window]"`. + +This opens the browser at . The command is also installed as `caccht-explorer`, under the app's name; both do the same. Give several corpora to compare them, e.g. `cfabric-browser bhsa CenterBLC/LXX ETCBC/peshitta`; others can be opened from the browser with *Open a corpus*. Without corpora (`cfabric-browser`), the browser starts empty and shows that list straight away. You can give a corpus in several ways: + +| Argument | Meaning | +|---|---| +| `bhsa`, `ETCBC/dss` | A corpus that Text-Fabric has downloaded or cloned, in `~/text-fabric-data/github` or `~/github` | +| `~/github/ETCBC/bhsa` | A repository in Text-Fabric layout (`tf//`); the version in its app config is used, otherwise the newest | +| `path/to/tf/2021` | A data directory with `.tf` files or a `.cfm` cache | +| `DT-UCPH/sp@7.2.4`, `sp@latest` | A specific version, or the newest on this computer (by default the version named in the corpus' app config) | +| `hebrew=…` | Any of the above, shown under the name `hebrew` | + +Options: +- `--port`: default 8107. +- `--workers`: how many searches can run at once (default 2). +- `--timeout`: time limit per search in seconds (default 60). +- `--max-results`: results kept per search (default 500,000). +- `--no-open`: don't open a web browser. +- `--window` (`--no-window`): show the browser in a window of its own instead of a web browser (needs pywebview: `pip install cfabric-browser[window]`). Closing the window stops the server. Without `--port` it uses the first free port from 8107, so several windows can run at once. The window keeps its settings, search history and saved workspaces in `~/.cfabric/window`. +- `--ui NAME`: the interface to start in until the user chooses one: `panes` (the default), `parallel`, `reader`, `interlinear` or `concordance`. Permanently, and to leave interfaces out of the menu, in `~/.cfabric/browser.yaml`: + + ```yaml + ui: + default: parallel + hide: [concordance] + ``` + + (`ui: parallel` alone also works.) See *Interfaces*. +- `--no-library`: don't let users open other corpora found on this computer. Then at least one corpus must be given. +- `--only ORG/REPO`: limit the *Open a corpus* list to these corpora (repeatable), e.g. for a test with a few corpora; others can't be opened from the browser either. Permanently: `library: only:` in `~/.cfabric/browser.yaml`. +- `--hide ORG/REPO`: leave a corpus out of the *Open a corpus* list (repeatable). To do this permanently, list them in `~/.cfabric/browser.yaml`: + + ```yaml + library: + hide: + - CLARIAH/descartes-tf + ``` + + The list also shows copies of one corpus (same title, such as a fork) only once, keeping the one with the newest data. + +The server listens on `127.0.0.1` only. Pass `--host` to change that, and only when you know what you're doing. + +## Downloading corpora + +The BHSA (2021), the Samaritan Pentateuch (7.2.4) and the Dead Sea Scrolls (2.0.1) can be downloaded from the browser when they are not on this computer: *Open a corpus* lists them under *Can be downloaded*, with their size; *Download* gets one and opens it. From the command line: + +```bash +cfabric-browser download # the corpora that can be downloaded, and which are here +cfabric-browser download bhsa sp dss +``` + +- They come from the corpora's own releases on GitHub, the same files Text-Fabric downloads, and each is checked against its checksum. +- They go where Text-Fabric keeps its downloads (`~/text-fabric-data/github`), in its layout and with its `__checkout__.txt` files, so Text-Fabric uses them too. A data folder that is already there is never overwritten. +- About 68 MB for the three (365 MB on disk). Right after downloading, each is opened and prepared once, which takes a few minutes for the BHSA and the DSS; after that it opens in a few seconds. +- With `--ask-downloads` (as the standalone program starts), the browser first offers the missing ones on a screen of their own, with one *Download* button; *Not now* is remembered. + +## Translations + +Translations can be added as texts of their own, shown next to the Hebrew ones. The first is the **JPS 1917**, *The Holy Scriptures: A New Translation* of the Jewish Publication Society (1917). In *Open a corpus*, under *Translations*, *Add* downloads it from Sefaria's export (about 5 MB), prepares it in a few seconds and opens it; *Remove* deletes it again. From the command line: + +```bash +cfabric-browser add # the translations that can be added +cfabric-browser add jps1917 +cfabric-browser remove jps1917 +``` + +- It is public domain in the United States, because it was published before 1931. Elsewhere that depends on the translators' dates; check the rules in your country. +- It becomes an ordinary corpus in `~/text-fabric-data/github/Context-Fabric/jps1917/tf/1917`, which Text-Fabric can also open: words (`text`, `trailer`, and `lower` for searching, as in `word lower=lord`), verses, chapters and books with the BHSA's English book names. +- Its verses follow the Hebrew numbering, as in the BHSA, so they line up verse by verse. In three places the 1917 print joins verses that the BHSA divides: the Ten Commandments in Exodus 20:13–16 and Deuteronomy 5:17–20, and Numbers 25:19 with 26:1. Those are divided as in the BHSA, and the feature `printed` on a verse keeps the print's number (for instance `20:14` on Exodus 20:17). +- A verse repeated in small type after the last verse of Malachi, Lamentations and Ecclesiastes, as read in the synagogue, is in the feature `note`. +- Building it checks the source: its title and licence, the number of chapters and verses per book, and that there is no markup other than those notes. When something differs, it stops and says where. +- The download is kept in `~/.cfabric/translations`, so adding it again needs no network. A translation that is open is removed when the browser starts again. + +In the interfaces: +- *Parallel*, *Panes* and the Reader (also under *Parallels*): a text like the others, read from left to right. A note printed after a verse is shown dimmed after it, and a verse divided differently in the print says so (`(20:13 in the print)`). +- *Interlinear*: a translation can't be paired word by word with the Hebrew, so under *Under* (heading *Translations*) it comes as a line under each verse. It is not offered as the text shown word by word. +- *Concordance*: search in English, e.g. `word lower=lord`; sorting and collocations go by the word in lower case. +- *Differences*: translations are left out of the comparison. +- The card shown when hovering over a word stays away for English words, which have no gloss or morphology; a search term made from one uses `lower`. +- *Cite* names the 1917 edition and Sefaria's export. + +## Dictionaries + +**BDB**, *A Hebrew and English Lexicon of the Old Testament* by Brown, Driver and Briggs (Oxford, 1906), can be added to the lexicon. A lexeme's page (BHSA) then has a tab *BDB* next to *Occurrences* with its entry: + +- the text as Sefaria has it, its verses as links that open the passage, its references to other entries as links to them, and BDB's abbreviations explained when the mouse is on them; +- the neighbouring entries, the root family (the root and the words BDB derives from it, with links to their lexemes), Strong's and TWOT numbers, and a link to the entry on Sefaria; +- when a lexeme is linked to more than one entry, all of them, the main one first, and a note when the link is less sure. + +Elsewhere: +- *Lexicon* list: *Find* also searches BDB's short definitions (*perish* finds אבד), each lexeme shows BDB's definition where it differs from the gloss, and the order *By root (BDB)* lists the lexemes under their root's letter in BDB's order, the words derived from a root indented under it. +- The card shown when hovering over a word and the inspector give BDB's entry of its lexeme; the inspector has *BDB entry*, which opens the lexicon at it. DSS words get it through the BHSA's lexeme, as they get their gloss. +- The Reader's *Lexicon* tab shows BDB's definitions; the Interlinear offers the line *BDB (its short definition)*. + +The links from the BHSA's lexemes to the entries are made when the file is built: the words of each verse are aligned with the Open Scriptures Hebrew Bible, whose lemmas lead to BDB's entries, and checked against the verses BDB cites. The file is built once and published (about 5 MB, in `~/.cfabric/dictionaries/bdb`); in *Open a corpus*, under *Dictionaries*, *Add* downloads it and *Remove* deletes it. From the command line: + +```bash +cfabric-browser add bdb # download it +cfabric-browser add bdb FILE # a file built on this computer +cfabric-browser remove bdb +python -m cfabric_browser.dictionaries.build OUT # build it (walks Sefaria's API once, about 12,000 requests) +``` + +The file is licensed under CC BY-NC 4.0, like the BHSA, the SP and the DSS. The text of BDB (1906) is in the public domain (Sefaria's is from a scan in the National Library of Israel), and Open Scriptures' data in it (ids, short definitions, Strong's and TWOT numbers, from its CC BY 4.0 lexicon) remains available under CC BY 4.0. The links to the BHSA's lexemes derive from the BHSA (CC BY-NC 4.0). + +## Interfaces + +The browser has several interfaces, chosen in the menu at the right of the top bar (it shows the name of the one in use). Switching keeps the passage, the corpora, the search, the lexicon and the settings; what an interface cannot show is kept for the next one. The interface used last is remembered by the web browser (or the window); `--ui` only sets the one for a first start. *Choose which to offer…* in the menu leaves interfaces out of it, though never the one in use, nor both *Panes* and *Parallel*. + +All five are built: *Panes*, *Parallel*, *Reader*, *Interlinear* and *Concordance*. + +### Reader + +*Reader* (`/read/bhsa/Genesis/1?v=3&panel=parallels`) is after Sefaria. It shows one text in a quiet, wide column. + +- **Reading:** + - *Verses* gives a line per verse; *Running text* runs the verses on, with a new paragraph after each petuhah (פ) or setumah (ס) in the text. + - Scrolling goes on into the next chapters, and the address follows the chapter at the top. + - `‹ ›`, the arrow keys and the chapter name (for the books and chapters) work as in *Parallel*. The menu next to it chooses the text to read. +- **The panel:** clicking a verse marks it and opens the panel, on the right (on a phone, from the bottom). Clicking a word opens the panel at that word. Its tabs: + - *Parallels*: the verse in the other texts, chosen with *Texts*; the DSS per scroll; *Show differences* marks them, in the text read as well. + - *Words*: the words of the verse with gloss and morphology; click one for everything about it. + - *Lexicon*: the verse's lexemes with gloss and frequency. *All* opens the lexicon in *Parallel*. + - *Search*: a search for words like one of the verse's, opened in *Parallel*. + - *Notes*: your own note and bookmark (★) on the verse, and a list of all your notes. They are kept in this browser (or the program's window), never sent anywhere, and are included in exported workspaces. +- *Cite*, next to the chapter, cites the chapter, or the verse chosen. + +- *Panes* (`/?c0=bhsa&ref0=Genesis+1&...`): up to four panes side by side, each with its own corpus, passage and view. +- *Parallel* (`/Genesis/1?texts=bhsa,sp,dss`), laid out after [Parabible](https://parabible.com): a slim top bar with the chapter (`‹ ›` and ←/→ for the neighbours, the chapter name for a drawer with the books and their chapters), and a column per corpus. *Texts* turns open corpora on and off, orders the columns, and adds more from the corpora on this computer; `–` on a column removes it. The chapters follow the BHSA's numbering, also when the BHSA is not shown (it is opened in the background when it is on this computer); without it, the first column's. Each row is a verse: verses are matched by book (through the book concordance), chapter and verse number, there is a row for every verse number in any of the texts, and a text that lacks a verse has an empty cell. A corpus read by biblical book (DSS) shows in each cell the text of every scroll that has the verse, labelled with the scroll. A corpus without the chapter keeps a narrow column marked *not in this corpus*. + +Hovering over a word shows the same card as in *Panes*; clicking it opens the word panel on the left, with the same contents as the inspector and, for a BHSA word, *Lexicon entry*. *Lexicon* in the top bar (when a corpus with a lexicon is open) shows the lexicon instead of the chapter; a verse in it opens its chapter with that verse marked (`&v=26`; the lexicon is `&lexicon=bhsa&lx=…`). Its *Search* goes to the search in *Panes* for now. + +*Differences* in the top bar (`&diff=1`) marks the words where the texts differ from the first column, with the same rules and colours as *Show differences* in *Panes*; each column header says how many words differ, and hovering over a marked word says from what. With MT, SP and the DSS shown, a DSS cell lists, for each place in that verse where MT and SP differ and a scroll preserves the text, which scrolls side with which. + +*Search* in the top bar searches one of the shown texts (`&search=1&sc=bhsa&q=…&s=…`), in either of two ways: + +- *Template* (the default): the query editor of *Panes*, starting from the template the terms make, if there are any. +- *Search terms*, as on Parabible: click a word, then *Add as search term* in the word panel (adding one switches to *Search terms*). A term finds words with the features you tick (its transliterated lexeme, or else its lexeme, to start with; also part of speech, stem, tense, person, gender, number, state). Choose whether all terms must be in one verse, sentence, clause or phrase, and, for corpora with books, which books (by the book feature's own values: the BHSA's `book` says `Jesaia` where the list says Isaiah). The search template this makes is shown below. + +Results are rows: the passage, then the verse the result lies in, in the searched text with the result's innermost nodes highlighted, and the same verse in the other texts (the DSS: per scroll); 25 per page. *Statistics*, *Download…* and *Cite* are as in *Panes*; clicking a passage opens its chapter at that verse. A shared link to a search the server no longer has runs it again. + +The top bar also has *Display* (for the BHSA *Vocalized* or *Unvocalized* text, and what is shown; shared with *Panes*), *Workspaces* (a saved workspace remembers its interface: loading a parallel one from *Panes* switches over, and the other way round), *Cite* next to the chapter (a citation per shown text), and light or dark colours. On a phone the texts of a verse are shown below each other, each with its name. + + +### Interlinear + +*Interlinear* (`/interlinear/bhsa/Genesis/1?under=sp&struct=1`) is after STEPBible, BibleHub and the Logos interlinears. It opens with the BHSA, and shows each word as a small block with lines under it. The blocks run from right to left, so a verse reads as a sentence. + +- **Lines** chooses the lines and their order. They are read by role, as on the hover card, so the SP and the DSS work too: + - the word; + - the transliteration of the lexeme (`BR>[`); + - the lexeme; + - the gloss; + - the part of speech; + - the morphology, in words; + - and any feature of the corpus (*Add a feature*). +- **Under** shows another text word by word under the words, with its differences marked. The SP under the BHSA, for example: words are matched on their consonants, as in the comparison, so a different word division still lines up. +- **Structure** groups the words in phrases, underlined with their function (`Pred`, `Subj`, `Objc`, …), and in clauses with their type (`WayX`, `xQtX`, …). +- **Practise** hides a line; click a word's **?** to see it, or use *Show all* and *Hide again*. +- **Colour** colours the words by part of speech, or the verbs by stem, with a legend. +- Clicking a word opens the word panel on the right. +- *Cite*, next to the chapter, cites the chapter, or the verse chosen. + +A transliteration of the word as it sounds is not available yet: the BHSA's `phono` is a module of its own. + +### Concordance + +*Concordance* (`/concordance/bhsa?q=…`) is after Sketch Engine and AntConc. It starts from a search: a template, or search terms made from words in the results (click a word, then *Add as search term*). The results come as lines (KWIC, key word in context): the hit in the middle and the words before and after it on either side, within the same book. Clicking a passage opens it in the *Reader*. + +- **Sort by** the hit or a word before or after it (up to the third), by its form (consonants), lexeme or part of speech. Patterns show at once, such as what comes before HLK[. +- **Facets** on the left: book, part of speech, stem, tense, person, gender, number, state, language, each with counts. A click filters the lines, and the facets follow. +- **Above the lines:** + - the number of lines and of hits per 10,000 words; + - *Context* (3, 6 or 10 words); + - *One per verse*; + - *Sample* (a random 50 to 500; *another* draws again); + - *Download lines*: a TSV table of passage, before, hit and after, in the order and with the filters shown, at most 20,000. +- **Collocations:** the lexemes (or forms, or parts of speech) within 1 to 5 words of the hits, with their count there, their frequency in the corpus and their logDice score. logDice measures how typical a word is near the hit (at most 14). +- **Statistics:** as in the other interfaces. + +*Cite* in the line with the number of results cites the search. On a phone the facets fold away behind *Filters*. + +## Queries + +Queries use Text-Fabric's search template language, run by Context-Fabric's search engine. Queries written for the TF browser work unchanged and give the same results. + +Problems in a template are reported per line. That includes problems inside quantifiers such as `/without/`, which the engine itself only logs, and then returns results as if the quantifier excluded nothing. + +## Asking in your own words + +You can ask your own AI app (with your own subscription) for a search in plain language, for instance *all cases of >LHJM/ in the BHSA*. The model writes a search template, checks it with a small trial search, and sends it to the browser. There it appears above the search form, in Panes, Parallel and the Concordance, with your request, the template, the model's explanation and the trial's count: *Run* runs it, *Edit* puts it in the editor first, *Dismiss* drops it. Nothing runs by itself, and the results always come from the browser's own search, never from the model's text. Where a search came from stays with it: in its link, the workspace and the citation ("Proposed by Claude, 16 October 2026, for: …", with *(edited)* when you changed the template). + +The browser serves its tools over MCP; the app connects to it while the browser runs: + +- **Claude Code:** `claude mcp add --transport http context-fabric http://127.0.0.1:8107/mcp` (8107 is the browser's usual port; with `--port`, use that one). Or, whatever the port: `claude mcp add context-fabric -- cfabric-browser mcp`. +- **Claude Desktop:** install the extension `CACCHT-Explorer.mcpb` (from the release, or `python packaging/mcpb/pack.py`) by opening it, or in *Settings → Extensions*. Or add to `claude_desktop_config.json`: `"mcpServers": {"context-fabric": {"command": "cfabric-browser", "args": ["mcp"]}}` (the standalone program works too: its path, with `mcp`). +- **Antigravity CLI** (Google's successor of Gemini CLI): in `~/.gemini/config/mcp_config.json`, `"mcpServers": {"context-fabric": {"command": "cfabric-browser", "args": ["mcp"]}}`. +- **The ChatGPT desktop app and Codex** (Codex's app, CLI and IDE extension; they share `~/.codex/config.toml`; in the ChatGPT app perhaps only when working with Codex): in the desktop app *Settings → MCP servers → Add server*, *Streamable HTTP*, `http://127.0.0.1:8107/mcp`; or `codex mcp add context-fabric --url http://127.0.0.1:8107/mcp`. To have it start the browser, in `config.toml` under `[mcp_servers.context-fabric]`: `command = "cfabric-browser"` and `args = ["mcp"]` (the form may have no field for arguments, and the Mac program's path has a space). The first tool call may wait for the corpora to open: `tool_timeout_sec = 300` there (the default is 60). ChatGPT on the web and on phones can't reach a program on your computer. + +`cfabric-browser mcp` passes each message on to the running browser (found in `~/.cfabric/running.json`, which the browser writes when it starts) and starts one if none runs. The tools: `list_corpora`, `corpus_guide`, `describe_corpus`, `list_features`, `describe_feature`, `search_syntax_guide`, `check_template`, `try_search` (the count up to 10,000 within 20 s and a few examples, with hints when there are none) and `propose_search`. `/mcp` only answers on 127.0.0.1 and refuses requests from web pages, so other sites in your web browser can't use it. + +What you ask goes to the AI company of your app; the corpora stay on your computer, apart from the few examples the model sees in its trial searches. + +How well an app does is measured with the 30 requests of `tests/data/bhsa-questions.yaml`: `python scripts/ask_questions.py claude` (or `codex`) sends each to the app without its interactive screen (`claude -p`, `codex exec`, on your own subscription), connected to a running browser with the BHSA open (`--url`, default port 8107), runs each proposal in full and compares its count with the question's. It prints a summary (proposed, right count, trials per request) and writes the rows as JSON lines. `--only 1,5` asks a few, `--model` chooses the model. Not in CI. + +## Words + +Hovering over a word shows a card with its essentials: gloss, lexeme, transliteration, part of speech and morphology (stem, tense, person, gender, number, state, suffix), in words rather than codes, and, with *Show differences*, what it differs from. Each is read from the first feature of its role that has a value; values such as `NA` and `absent` are left out (`cfabric_browser/summary.py`). The DSS has its own analysis and the ETCBC's (features ending in `_etcbc`, as in the BHSA): the ETCBC's comes first (e.g. `sp_etcbc`, which tells prepositions and proper nouns apart), the DSS's own where the ETCBC has nothing. The DSS has no glosses: they come from the lexicon of another open corpus (the BHSA), by the word's ETCBC lexeme and language (`lex_etcbc`, `lang_etcbc`), marked `· bhsa`. Clicking a word opens the inspector with the same essentials, all other features (folded away; without empty values, and without the `g_` features unless asked for), its relations and what it lies in. + +## Lexicon + +A corpus with lexeme nodes (`lex`) that have glosses, such as the BHSA, gets a third view besides *Read* and *Search*: *Lexicon*, after SHEBANQ's *Words* page. + +- Lexemes per language (Hebrew, Aramaic) and first letter, in Hebrew alphabetical order (sin and shin as one letter), with their vocalised form, transliteration, gloss, part of speech and frequency. +- *Find* searches all letters at once, by Hebrew (with or without vowels), transliteration or gloss; lexemes can also be limited to a part of speech, and sorted by frequency. +- A lexeme shows every verse it occurs in, as running text with its words marked (a verse with several occurrences once, with `×2`), how often it occurs per book (choose a book to see only its verses), and links to search for it (`word lex=BR>[ language=Hebrew`) and to download its occurrences as TSV. Clicking a verse reads it; with *Sync passages*, the other panes follow. + +The features used are, per role, the first one the corpus has: `lex`, `voc_lex_utf8` or `lex_utf8`, `gloss`, `sp`, `language`, `freq_lex` (otherwise the words are counted), `rank_lex` and `nametype` (`cfabric_browser/lexicon.py`). + +## Display settings + +How a corpus is displayed comes from its Text-Fabric app configuration (`/app/config.yaml`). To adjust it without editing downloaded files, put the settings that differ in `~/.cfabric/display//.yaml`, or, for one version only, `@.yaml`. These files use the same format as the app configuration and are layered over it. For example, `~/.cfabric/display/DT-UCPH/sp@7.2.4.yaml`: + +```yaml +typeDisplay: + word: + base: true + featuresBare: gloss + phrase: + label: '{typ} {function}' + phrase_atom: + hidden: true +``` + +The browser comes with such files of its own, in `cfabric_browser/display_settings/`, for corpora whose app configuration lacks something it needs: for the SP, that words are the units to show, hover over and click (its slots are signs). Your own files in `~/.cfabric/display` are layered over those. + +## API + +All functions of the interface are available as a JSON API; interactive documentation is at `/docs` while the browser runs. Where they overlap, the endpoints mirror the tools of `cfabric-mcp`, so that people and language models use the same operations. + +| Endpoint | Purpose | +|---|---| +| `GET /api/corpora`, `GET /api/corpora/{c}` | Corpora and everything needed to work with one (types, sections, display settings) | +| `GET /api/corpora/{c}/features[/{f}]` | Features, with values and counts | +| `GET /api/corpora/{c}/sections?ref=` | The section tree, one level at a time | +| `GET /api/corpora/{c}/passage?ref=Genesis 1:1` | Render trees of a passage, with its section path and the previous and next passages | +| `GET /api/corpora/{c}/align?from=bhsa&ref=Genesis 1:1` | The corresponding passage in this corpus; `exact` is false when only the enclosing chapter exists | +| `GET /api/corpora/{c}/books` | For a corpus of manuscripts (DSS): the biblical books in it, with chapters, verses and scrolls | +| `GET /api/corpora/{c}/witnesses?ref=Genesis 1` | A chapter or verse in the book view: per verse, the text of each scroll that contains it | +| `GET /api/agreement?a=bhsa&aref=…&b=sp&bref=…&c=dss&cref=…` | At each place where `a` and `b` differ, which scrolls of `c` (book view) agree with `a`, with `b`, or with neither | +| `GET /api/compare?a=bhsa&aref=Genesis 1&b=sp&bref=Genesis 1` | Words that differ between two corresponding Hebrew passages, compared on their consonants verse by verse: `matres` (only matres lectionis) or `variant` | +| `GET /api/searches/{id}/parallel?offset=&limit=&texts=` | A page of results as parallel rows: each result's verse in the searched corpus, highlighted, and in `texts` | +| `GET /api/corpora/{c}/interlinear?ref=&features=&under=` | A passage word by word: per word its lines in words, the extra features, and its phrase and clause (with function and type); with `under`, another corpus' words aligned under them, with their differences | +| `GET /api/searches/{id}/kwic?offset=&limit=&left=&right=&sort=&filter=&per=&sample=&seed=&features=` | A page of the results as concordance lines: the hit and its context, sorted (`L1.text`, `hit.lex`, …), filtered by features of the hit (`vs:qal,book:Genesis`), one per verse, or sampled | +| `GET /api/searches/{id}/kwic/facets?feature=&filter=&per=` | How often each value of a feature of the hit occurs in the filtered results (`book`: in book order) | +| `GET /api/searches/{id}/collocations?feature=&window=&filter=&per=&min=&top=` | The values of a feature near the hits, with count, corpus frequency and logDice | +| `GET /api/parallel?lead=&ref=&texts=` | A chapter of `lead` (`Genesis 1`) in the corpora `texts`, as rows of verses with a cell per corpus (the DSS: per scroll), for the parallel interface | +| `GET /api/corpora/{c}/lexicon` | All lexemes of a corpus with a lexicon (BHSA), in alphabetical order per language; with BDB, `bdbShort`, `bdbOrder`, `bdbLetter`, `bdbRoot` per lexeme | +| `GET /api/corpora/{c}/lexicon/{n}` | One lexeme, with its number of verses, how often it occurs per book, and its entries in BDB (`bdb`, null when BDB hasn't been added) | +| `GET /api/corpora/{c}/lexicon/{n}/occurrences?offset=&limit=&book=` | A page of the verses it occurs in, as plain render trees with its words highlighted | +| `GET /api/corpora/{c}/lexicon/{n}/occurrences.tsv` | Every occurrence: section, word and verse text; `excel=true` for Excel | +| `GET /api/library`, `POST /api/corpora` | Corpora found on this computer; open one by its id (e.g. `ETCBC/dss`) | +| `GET /api/downloads[/{org}/{repo}]`, `POST`/`DELETE /api/downloads/{org}/{repo}` | Corpora that can be downloaded, whether they are here, and the progress of a download (`job`: download, unpack, prepare); start or cancel one | +| `GET /api/translations[/{t}]` | Translations that can be added, whether they are, and the progress of adding one (`job`) | +| `POST /api/translations/{t}`, `DELETE /api/translations/{t}` | Add a translation (download and build it, in the background), or remove it | +| `GET /api/proposals`, `GET /api/proposals/events` | Searches proposed by a language model, newest first; server-sent events whenever they change | +| `POST /api/proposals`, `POST /api/proposals/{id}` | Propose a search; mark one `run` or `dismissed` | +| `POST /mcp` | The tools for a language model, as an MCP server (Streamable HTTP, stateless); only on 127.0.0.1, not from web pages | +| `GET /api/dictionaries[/{d}]` | Dictionaries that can be added (BDB), whether they are, and the progress of adding one; one of them with its metadata and abbreviations | +| `POST /api/dictionaries/{d}`, `DELETE /api/dictionaries/{d}` | Download a dictionary (in the background), or remove it | +| `GET /api/dictionaries/{d}/entries/{id}?corpus=` | An entry: its text, neighbours and root family, with the lexemes of `corpus` linked to them and its verses as references there | +| `GET /api/corpora/{c}/nodes/{n}[/info]` | Render tree of a node; all its features, edges and surroundings | +| `GET /api/corpora/{c}/nodes/{n}/summary` | The essentials of a word, as on the card shown when hovering over it; `bdb`: its entries in BDB, by its lexeme (in the BHSA, also for the DSS) | +| `POST /api/corpora/{c}/check` | Check a template without searching | +| `POST /api/corpora/{c}/search` | Start a search (`{"template": …, "limit": …, "timeout": …}`) | +| `GET /api/searches/{id}[/events]` | Status, or a live event stream of it | +| `GET /api/searches/{id}/results` | A page of results (optionally with plain renderings) | +| `GET /api/searches/{id}/results/{i}` | One result in its context, highlighted | +| `GET /api/searches/{id}/stats/features?position=` | Features with values on one part of the results; which the template mentions, and which it leaves open | +| `GET /api/searches/{id}/stats/values?position=&feature=` | How often each value occurs there (distinct nodes) | +| `GET /api/searches/{id}/stats/sections?level=` | Results per book (0), chapter (1), ... in text order | +| `GET /api/searches/{id}/export.tsv` | All results, one row per result; features in the template become columns (write `sp*` to add one without narrowing the search); `extra=3:lex,gloss` adds columns for the third node; `excel=true` for Excel | +| `DELETE /api/searches/{id}` | Stop a search | + +Display options are query parameters: `mode` (`pretty`/`plain`), `fmt` (only `text-orig-full` and, where the corpus has it, `text-orig-plain`; another format gives the default), `full`, `hide_types`, `hidden_types`, `base_types`, `condense_type`, `extra_features`, `lang`. + +## How it works + +- Searches run in worker processes, so the interface stays responsive and searches can run in parallel. Because corpora are memory-mapped, the workers share most of their memory. A search that ignores its time limit or a stop request is ended by restarting its worker. +- Results stream in while the search runs. Once it has finished, they are put in canonical order. +- Displays are built by `cfabric_browser.display` (a port of Text-Fabric's displays without HTML) as structured render trees, which the interface draws. The corpus' Text-Fabric app configuration (`app/config.yaml`) is used as it is. + +## Development + +The interface is a Vite + React + TypeScript app in `frontend/`. It is built into `cfabric_browser/static/`, which is not in git; the release workflow builds it before packaging. + +```bash +cd libs/explorer/frontend +npm install +npm run build # type-check and build into cfabric_browser/static +npm run dev # live-reloading interface on :5173, using a running cfabric-browser on :8107 +``` + +Fonts are bundled from npm (`@fontsource`), so the browser needs no internet. For Hebrew, Ezra SIL or SBL Hebrew are used when installed on the computer, otherwise the bundled Noto Serif Hebrew. + +Tests: `pytest tests/` in `libs/explorer` (they start real search workers on a small fixture corpus). + +### The user guide + +The guide for users is written in Markdown in `guide/`, one file per chapter (the order is `PAGES` in `frontend/scripts/guide.mjs`). `npm run build` makes it into pages in `cfabric_browser/static/guide/`, which the program serves at `/guide/`, so it works offline; *Guide to …* in the interface menu opens the page for the interface in use (`frontend/src/guide.ts`). `node scripts/guide.mjs --site DIR` also writes the pages for caccht.github.io into `DIR/explorer/`, in the style of that site's own pages. Links between chapters are written to the Markdown files (`words.md#bdb`); `npm test` checks that every link and every interface's page exist. The screenshots in `guide/images/` are made by `npm run screenshots -- http://127.0.0.1:8799` (`frontend/scripts/screenshots.mjs`, with playwright-core and the Chrome installed on the computer) from a running browser on the real corpora, started with a home folder that has only the corpora and BDB, so that they show what users see; the list of shots is `SHOTS` in that script, and `npm test` checks that the guide uses each of them. `npm test` also checks the guide's names of things in the program: every phrase in italics must be a text of the frontend, the guide's own pages or the server's messages (`…` stands for a value, `→` separates a menu from its item), unless it is in `NOT_LABELS` in `frontend/scripts/guide.test.mjs` (other programs' buttons, examples, glosses); so a renamed button fails the test until the guide follows. The plan: `../make_browser_context_fabric/documentation-plan.md` (outside the repository). + +**Before each desktop release** (a `desktop-v*` tag): + +1. **The guide.** For every change users notice, update its chapter in `guide/` (a new button, a changed default, a new corpus or text), and `npm test` (names in the program, links, pictures). +2. **The screenshots**, when the interface changed: start the browser on the real corpora with a home folder that holds only the corpora and BDB (a junction or link to `text-fabric-data`, and `.cfabric/dictionaries`), with `--only` for the corpora of `CORPORA` in `packaging/cfabric-browser.py`, then `npm run screenshots -- http://127.0.0.1:`. Look at each picture before committing. +3. **The release notes** (`packaging/RELEASE_NOTES.md`): what is new since the last release at the top, the downloads and how to start; the rest belongs in the guide. +4. **caccht.github.io:** `node scripts/guide.mjs --site `, a pull request with the changed pages (and new chapters in its `sitemap.xml`), merged before the release, because the release notes link to it. +5. **The checks of `browser-desktop.yml`** on all four builds, then the tag. + + +### Standalone program + +`packaging/cfabric-browser.spec` builds a program that runs without a Python installation (with PyInstaller; about 65 MB on Windows). It shows the browser in a window of its own (`--window`), without a console. Started without corpora, for example by double-clicking it, it offers the corpora on this computer, and first offers to download the BHSA, the SP and the DSS when they are missing (`--ask-downloads`). Build it in a separate virtual environment with normal, not editable, installs: PyInstaller does not find editable installs. On Windows, make that environment with a Python from python.org: PyInstaller cannot bundle the Microsoft Store Python. + +```bash +cd libs/explorer +(cd frontend && npm run build) +python -m venv build/env +build/env/Scripts/python -m pip install ../core ".[build]" # Linux/macOS: build/env/bin/python +build/env/Scripts/pyinstaller packaging/cfabric-browser.spec --noconfirm +dist/cfabric-browser/cfabric-browser bhsa +``` + +Its entry point is `packaging/cfabric-browser.py`, which adds `--window` (`--no-window` overrides it), `--ui parallel` (the parallel interface on a first start), `--ask-downloads` and, for the test version, `--only` for the corpora in its `CORPORA` list (BHSA, SP, DSS and the JPS 1917); empty that list to offer all corpora. As it has no console, it writes its output and errors to `~/.cfabric/logs/cfabric-browser.log`, and on Windows shows a message when it cannot start. If no window toolkit is available (on Linux, pywebview needs GTK or Qt), it opens a web browser instead. + +The program icons in `packaging/icons/` are drawn from the Context-Fabric mark (`frontend/public/favicon.svg`) by `packaging/make_icons.py` (needs Pillow); run it again when the mark changes. + +`packaging/smoke-test.sh` starts a built program as a server with a small corpus and checks that the interface is served and that a search runs; it needs only bash and curl. `packaging/translation-test.sh` and `packaging/download-test.sh` start it with an empty home folder and add the JPS 1917 or download the SP (both over the network). The workflow `browser-desktop.yml` builds the program on Windows, macOS (Apple silicon and Intel, as `CACCHT Explorer.app`) and Linux, runs these three tests on each build (on macOS inside the app), runs the Windows build once more on a runner where no Python can be reached, and keeps the builds as artifacts for 14 days. The builds are not signed, so Windows SmartScreen and macOS Gatekeeper warn about them. Like `cfabric_browser/__main__.py` (`python -m cfabric_browser`), it calls `multiprocessing.freeze_support()` so that search workers start correctly in the frozen program. diff --git a/libs/explorer/cfabric_browser/__init__.py b/libs/explorer/cfabric_browser/__init__.py new file mode 100644 index 0000000..3431561 --- /dev/null +++ b/libs/explorer/cfabric_browser/__init__.py @@ -0,0 +1,3 @@ +"""CACCHT Explorer, the Context-Fabric browser.""" + +__version__ = "0.1.0" diff --git a/libs/explorer/cfabric_browser/__main__.py b/libs/explorer/cfabric_browser/__main__.py new file mode 100644 index 0000000..2281124 --- /dev/null +++ b/libs/explorer/cfabric_browser/__main__.py @@ -0,0 +1,12 @@ +"""`python -m cfabric_browser`, and the entry point of the standalone program.""" + +import multiprocessing +import sys + +from cfabric_browser.cli import main + +if __name__ == "__main__": + # In a frozen program (PyInstaller), each search worker starts the program + # again; this turns that start into the worker instead of a second server. + multiprocessing.freeze_support() + sys.exit(main()) diff --git a/libs/explorer/cfabric_browser/app.py b/libs/explorer/cfabric_browser/app.py new file mode 100644 index 0000000..388ae0c --- /dev/null +++ b/libs/explorer/cfabric_browser/app.py @@ -0,0 +1,178 @@ +"""The browser's web application: the JSON API plus the single-page interface.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from importlib import metadata +from pathlib import Path + +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from mcp.server.fastmcp.server import StreamableHTTPASGIApp +from starlette.routing import Route + +from cfabric_browser import __version__ +from cfabric_browser.assistant.mcp import build_server as build_mcp +from cfabric_browser.assistant.mcp import local +from cfabric_browser.assistant.tools import Tools +from cfabric_browser.corpora import CorpusService +from cfabric_browser.dictionaries.manage import DictionaryJobs +from cfabric_browser.downloads import DownloadJobs +from cfabric_browser.guard import Guard +from cfabric_browser.routes import build_router +from cfabric_browser.translations.manage import Jobs +from cfabric_browser.workers import SearchPool + +STATIC = Path(__file__).parent / "static" + +NOT_BUILT = """ +CACCHT Explorer + +

CACCHT Explorer

+

The server is running, but the interface has not been built.

+

Build it with npm install and npm run build in +libs/explorer/frontend, or use the API at /docs.

+ +""" + + +def create_app( + service: CorpusService, + pool: SearchPool, + static_dir: Path = STATIC, + manage_pool: bool = True, + library: bool = True, + ui: str = "classic", + hidden_uis: list[str] | None = None, + translations: Jobs | None = None, + downloads: DownloadJobs | None = None, + ask_downloads: bool = False, + dictionaries: DictionaryJobs | None = None, + assistant: Tools | None = None, + hosts: list[str] | None = None, +) -> FastAPI: + """Build the application. + + Parameters + ---------- + service: CorpusService + The corpora to serve. + pool: SearchPool + The search workers. + static_dir: Path + Directory with the built interface (`index.html` and `assets/`). + manage_pool: bool + Start the pool when the app starts and stop it when it stops. + library: bool + Let people open other corpora found on this computer. Switch this off + when the browser is reachable by others. + ui: str + The interface to start in when the user has not chosen one yet: + `classic` (panes), `parallel`, `reader`, `interlinear` or + `concordance` (see `cfabric_browser.interfaces`). + hidden_uis: list[str] | None + Interfaces to leave out of the menu until the user chooses otherwise. + translations: Jobs | None + Where translations are built and how (`cfabric_browser.translations`); + by default into the first place the library looks. + downloads: DownloadJobs | None + Where corpora are downloaded to (`cfabric_browser.downloads`); by + default the first place the library looks. + ask_downloads: bool + At the start, offer to download the corpora that can be downloaded and + are missing (the standalone program). + dictionaries: DictionaryJobs | None + Where dictionaries (BDB) are kept (`cfabric_browser.dictionaries`); by + default `~/.cfabric/dictionaries`. + assistant: Tools | None + The tools of searching in plain language and their proposals + (`cfabric_browser.assistant`); by default on these corpora and pool. + hosts: list[str] | None + The host names the server answers to (`cfabric_browser.guard`), such + as `guard.LOCAL_HOSTS` when it listens only on this computer; `None` + for any. Requests from other web sites are refused either way. + """ + + # searching in plain language: the tools, also as an MCP server at /mcp + tools = assistant if assistant is not None else Tools(service, pool) + mcpServer = build_mcp(local(tools)) + mcpServer.streamable_http_app() # makes its session manager + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + if manage_pool: + pool.start(wait=False) + try: + async with mcpServer.session_manager.run(): + yield + finally: + if manage_pool: + pool.shutdown() + + app = FastAPI( + title="CACCHT Explorer", + version=__version__, + lifespan=lifespan, + ) + app.add_middleware(Guard, hosts=hosts) + app.include_router( + build_router( + service, + pool, + library=library, + translations=translations, + downloads=downloads, + dictionaries=dictionaries, + assistant=tools, + ) + ) + # before the interface's catch-all route + app.router.routes.insert( + 0, Route("/mcp", endpoint=StreamableHTTPASGIApp(mcpServer.session_manager)) + ) + + try: + coreVersion = metadata.version("context-fabric") + except metadata.PackageNotFoundError: + coreVersion = None + + @app.get("/api/health") + def health() -> dict: + return { + "ok": True, + "searchReady": pool.ready, + "version": __version__, + "coreVersion": coreVersion, + "ui": ui, + "hiddenUis": list(hidden_uis or []), + # offer the missing corpora at the start (/api/downloads) + "askDownloads": ask_downloads and library, + } + + index = static_dir / "index.html" + assets = static_dir / "assets" + if assets.is_dir(): + app.mount("/assets", StaticFiles(directory=assets), name="assets") + + @app.get("/{path:path}", include_in_schema=False) + def spa(path: str): + # the interface does its own routing: every non-API path gets the page + if path.startswith("api/"): + raise HTTPException(404, "Not found") + candidate = (static_dir / path).resolve() + inside = bool(path) and candidate.is_relative_to(static_dir.resolve()) + if inside and candidate.is_file(): + return FileResponse(candidate) + # a folder of pages (the guide, /guide/): its index.html, at an address + # ending in / so that the pages' relative links work + if inside and (candidate / "index.html").is_file(): + if not path.endswith("/"): + return RedirectResponse(f"/{path}/") + return FileResponse(candidate / "index.html") + if index.is_file(): + return FileResponse(index) + return HTMLResponse(NOT_BUILT) + + return app diff --git a/libs/explorer/cfabric_browser/assistant/__init__.py b/libs/explorer/cfabric_browser/assistant/__init__.py new file mode 100644 index 0000000..aeb19e7 --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/__init__.py @@ -0,0 +1,8 @@ +"""Searching in plain language: the tools a language model gets to turn a +request ("all cases of >LHJM/ in the BHSA") into a search template that the +browser shows and runs. + +`tools.py` has the tools; the model reaches them through MCP (the user's own +AI app, with their own subscription). The results always come from the +browser's search, never from the model's text. +""" diff --git a/libs/explorer/cfabric_browser/assistant/guides/bhsa.md b/libs/explorer/cfabric_browser/assistant/guides/bhsa.md new file mode 100644 index 0000000..7c99656 --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/guides/bhsa.md @@ -0,0 +1,123 @@ +# Searching the BHSA + +The BHSA (Biblia Hebraica Stuttgartensia Amstelodamensis, ETCBC, version 2021) is the Hebrew Bible with linguistic annotations on words, phrases, clauses and sentences. A search template describes objects, one per line; indentation means "inside". The browser runs it; you only write it. + +## Object types (how many), from large to small + +book (39) > chapter (929) > verse (23,213) > half_verse (45,179) > sentence (63,717) > clause (88,131) > phrase (253,203) > word (426,590). + +Also: `sentence_atom`, `clause_atom`, `phrase_atom`, `subphrase` (finer divisions), and `lex` (9,230 lexemes). A `lex` node contains all the words of that lexeme, so `lex gloss=create` with `word` indented under it finds the words of lexemes glossed "create". + +## The main features + +**Words** (`word`): + +- `lex`: the lexeme in ETCBC transliteration (see *Conventions*): `>LHJM/`, `HLK[`, `JHWH/`. +- `sp`: part of speech: `subs` noun, `verb`, `prep` preposition, `conj` conjunction, `nmpr` proper noun, `art` article, `adjv` adjective, `nega` negation, `prps` personal pronoun, `advb` adverb, `prde` demonstrative pronoun, `intj` interjection, `inrg` interrogative particle, `prin` interrogative pronoun. +- `vs`: verbal stem: `qal`, `nif` niphal, `piel`, `pual`, `hit` hitpael, `hif` hiphil, `hof` hophal, and rarer ones (`hsht`, `poel`, ...); Aramaic `peal`, `pael`, `haf`, `afel`, `peil`, `htpe`, `htpa`, ... `NA` for words that aren't verbs. +- `vt`: verbal tense: `perf` perfect (qatal), `impf` imperfect (yiqtol), `wayq` wayyiqtol, `impv` imperative, `infc` infinitive construct, `infa` infinitive absolute, `ptca` active participle, `ptcp` passive participle. +- `ps` person (`p1`, `p2`, `p3`), `gn` gender (`m`, `f`), `nu` number (`sg`, `pl`, `du`); `unknown` where it can't be told (common gender, for instance), `NA` where it doesn't apply. +- `st`: state: `a` absolute, `c` construct, `e` emphatic (Aramaic). +- Pronominal suffix: `prs` its consonants (`W`, `K`, `HM`, ...; `absent` without one), with `prs_ps`, `prs_gn`, `prs_nu`. +- `language`: `Hebrew` or `Aramaic` (6,106 Aramaic words, in Daniel, Ezra and a few verses elsewhere). +- `nametype` of proper nouns: `pers` person, `topo` place, `gens` people, `ppde`, `mens`, and combinations such as `pers,gens,topo`: use `nametype~topo` to include those. +- `ls`: lexical set: `card` cardinal, `ordn` ordinal, `gntl` gentilic, `quot` quotation verb (אמר), `ques`, `nmdi` distributive noun, ... +- Text: `g_cons` (consonants in transliteration, without vowels: `>LHJM`), `g_word_utf8` (the word in Hebrew with vowels), `g_lex_utf8`. +- `gloss` (English) is on `lex` nodes, not on words. + +**Phrases** (`phrase`): + +- `function`: what the phrase does in its clause: `Pred` predicate, `Subj` subject, `Objc` object, `Cmpl` complement, `Adju` adjunct, `Time`, `Loca` location, `Modi` modifier, `Conj` conjunction, `Nega` negation, `Ques` question, `Rela` relative, `Voct` vocative, `Intj`, `PreC` predicate complement, `PreO` predicate with object suffix, `PreS` predicate with subject suffix, `Frnt` fronted element, `Supp` supplementary constituent, ... +- `typ`: phrase type: `VP` verbal, `NP` nominal, `PP` prepositional, `CP` conjunctive, `PrNP` proper-noun phrase, `AdvP`, `NegP`, `InrP`, `AdjP`, `PPrP` personal pronoun, `DPrP` demonstrative pronoun, ... +- `det`: determination: `det`, `und` (undetermined). + +**Clauses** (`clause`): + +- `typ`: clause type, by the form of its verb and what comes before it: `Way0` wayyiqtol first, `WayX` wayyiqtol after something, `WQt0` weqatal first, `xQt0` qatal after a word, `xQtX` qatal after a constituent, `XQtl` qatal after a non-subject, `ZQt0`, `ZYq0` (no conjunction), `xYq0`, `WYq0` (yiqtol), `ZIm0` imperative, `NmCl` nominal clause, `AjCl` adjectival, `InfC` infinitive construct clause, `Ptcp` participle clause, `Ellp` ellipsis, `Voct`, `Defc` defective, `CPen` casus pendens, `MSyn` macrosyntactic sign, ... +- `rela`: relation to another clause: `Objc` object clause, `Subj` subject clause, `Attr` attributive (relative), `Adju` adjunct, `Cmpl`, `PreC`, `RgRc` regens/rectum, `Resu` resumptive, `Coor` coordinated, `Spec`, `Appo`, `Link`, ...; `NA` for main clauses. +- `kind`: `VC` verbal clause, `NC` nominal clause, `WP` without predicate. +- `domain`: `N` narrative, `Q` quotation (direct speech), `D` discursive, `?` unknown. +- `txt`: text type by levels of embedding (`N`, `Q`, `NQ`, `NQQ`, ...). + +**Sections**: `book book=Genesis`, `chapter chapter=1`, `verse verse=3`, indented in that order. + +## Conventions + +- **Lexemes** are written in ETCBC transliteration: `>` א, `B` ב, `G` ג, `D` ד, `H` ה, `W` ו, `Z` ז, `X` ח, `V` ט, `J` י, `K` כ, `L` ל, `M` מ, `N` נ, `S` ס, `<` ע, `P` פ, `Y` צ, `Q` ק, `R` ר, `F` שׂ, `C` שׁ, `T` ת (final forms the same). +- A verb's lexeme ends in `[` (`HLK[` to go, `BR>[` to create), a noun's or adjective's in `/` (`>LHJM/` god(s), `MLK/` king, `VWB/` good), particles have neither (`L` to, `>CR` which, `KJ` that, `W` and, `H` the article, `H=` the interrogative he). +- `=` marks homonyms: `DBR[` speak and `DBR/` word; `>MR[` say; `H` the article but `H=` the interrogative; `QW=/`, `LHJM/ | `word lex=>LHJM/` | 2,601 | +| the verb HLK[ in all its forms | `word lex=HLK[` | 1,554 | +| HLK[ in the qal | `word lex=HLK[ vs=qal` | 1,412 | +| infinitives construct of HLK[ | `word lex=HLK[ vt=infc` | 144 | +| active participles of >MR[ | `word lex=>MR[ vt=ptca` | 138 | +| the niphal of DBR[ | `word lex=DBR[ vs=nif` | 4 | +| hiphil imperfects | `word vs=hif vt=impf` | 2,264 | +| perfects, second person feminine singular | `word vt=perf ps=p2 gn=f nu=sg` | 222 | +| the name YHWH | `word lex=JHWH/` | 6,828 | +| Aramaic words | `word language=Aramaic` | 6,106 | +| place names | `word sp=nmpr nametype=topo` | 6,371 | +| place names, also those that are also personal names | `word sp=nmpr nametype~topo` | 10,916 | +| feminine plural nouns | `word sp=subs gn=f nu=pl` | 5,992 | +| nouns in the construct state | `word sp=subs st=c` | 36,679 | +| verbs with a suffix of the first person | `word sp=verb prs_ps=p1` | 1,649 | +| the interrogative he | `word lex=H=` | 743 | +| words with the consonants >LHJM, of any lexeme | `word g_cons=>LHJM` | 1,177 | +| the words of lexemes glossed "create" | `lex gloss=create` / ` word` | 54 | +| the words of lexemes whose gloss starts with "king" | `lex gloss~^king` / ` word` | 3,001 | +| wayyiqtol forms in Genesis | `book book=Genesis` / ` word vt=wayq` | 2,106 | +| imperatives in the Psalms | `book book=Psalmi` / ` word vt=impv` | 701 | +| >LHJM/ in Genesis 1 | `book book=Genesis` / ` chapter chapter=1` / ` word lex=>LHJM/` | 32 | +| phrases that are the subject | `phrase function=Subj` | 31,979 | +| determined noun phrases | `phrase typ=NP det=det` | 20,476 | +| clauses with a predicate and a subject | `clause` / ` phrase function=Pred` / ` phrase function=Subj` | 18,966 | +| clauses with the predicate before the subject | `clause` / ` p1:phrase function=Pred` / ` p2:phrase function=Subj` / `p1 < p2` | 13,938 | +| clauses with the subject before the predicate | `clause` / ` p1:phrase function=Subj` / ` p2:phrase function=Pred` / `p1 < p2` | 5,028 | +| clauses whose subject is >LHJM/, with a verb as predicate | `clause` / ` phrase function=Pred` / ` word sp=verb` / ` phrase function=Subj` / ` word lex=>LHJM/` | 614 | +| clauses where >LHJM/ is the subject of >MR[ | `clause` / ` phrase function=Pred` / ` word lex=>MR[` / ` phrase function=Subj` / ` word lex=>LHJM/` | 147 | +| clauses with HJH[ as predicate | `clause` / ` phrase function=Pred` / ` word lex=HJH[` | 3,521 | +| nominal clauses | `clause typ=NmCl` | 11,377 | +| wayyiqtol clauses with nothing before the verb | `clause typ=Way0` | 9,038 | +| qatal after a constituent (x-qatal-X) | `clause typ=xQtX` | 2,797 | +| object clauses | `clause rela=Objc` | 1,327 | +| relative clauses with >CR | `clause` / ` phrase function=Rela` / ` word lex=>CR` | 4,738 | +| negated clauses | `clause` / ` phrase function=Nega` | 6,044 | +| clauses in direct speech | `clause domain=Q` | 52,937 | +| an infinitive construct right after the preposition L | `phrase` / ` w1:word lex=L` / ` w2:word vt=infc` / `w1 <: w2` | 4,572 | +| a noun followed right away by an adjective | `phrase typ=NP` / ` w1:word sp=subs` / ` w2:word sp=adjv` / `w1 <: w2` | 1,579 | +| >MR[ followed in its clause by L with a suffix | `clause` / ` w1:word lex=>MR[` / ` w2:word lex=L prs#absent` / `w1 < w2` | 347 | +| verses with both >LHJM/ and BR>[ (21 combinations of a verse and the two words, in 12 verses) | `verse` / ` word lex=>LHJM/` / ` word lex=BR>[` | 21 | +| verses without a verb | `verse` / `/without/` / ` word sp=verb` / `/-/` | 1,607 | +| subjects that are only proper names | `phrase function=Subj` / `/where/` / ` w:word` / `/have/` / ` w sp=nmpr` / `/-/` | 5,893 | +| the same, shorter | `phrase function=Subj` / `/without/` / ` word sp#nmpr` / `/-/` | 5,893 | +| subjects with at least one proper name | `phrase function=Subj` / `/with/` / ` word sp=nmpr` / `/-/` | 10,484 | +| verses with both >HB[ and JHWH/, each verse once | `verse` / `/with/` / ` word lex=>HB[` / `/-/` / `/with/` / ` word lex=JHWH/` / `/-/` | 60 | + +(`/` separates the lines of a template here; write each on its own line, with the indentation shown.) + +## Common mistakes + +- English or Hebrew values: `sp=noun`, `vt=infinitive`, `vs=hiphil` find nothing; use the codes above (`subs`, `infc`, `hif`). +- A lexeme without its ending: `lex=HLK` finds nothing; it is `HLK[`. `try_search` names a value that a feature never has. +- English book names: `book=Psalms` finds nothing; it is `Psalmi`. +- `gloss` on a word: it is on `lex` nodes (`lex gloss=create`, with `word` under it). +- `function` on a word or a clause: it is a feature of phrases. +- The article `H` for the interrogative `H=`, `DBR[` (speak) for `DBR/` (word). +- `/where/` with a new object under `/have/` (`/where/` / ` word` / `/have/` / ` word sp=nmpr`): that finds subjects with at least one proper name (10,484), not subjects of proper names only (5,893). Name the object and refer to it, or use `/without/`. +- Counting: `verse` with two words under it gives a result for every pair of matching words in the verse, not one per verse. To count verses, keep the verse alone with its conditions inside a quantifier, or tell the user that the count is of combinations. diff --git a/libs/explorer/cfabric_browser/assistant/mcp.py b/libs/explorer/cfabric_browser/assistant/mcp.py new file mode 100644 index 0000000..2ae18e6 --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/mcp.py @@ -0,0 +1,225 @@ +"""The tools as an MCP server, for the user's own AI app (Claude Code, +Claude Desktop, the ChatGPT desktop app, Codex, ...). + +They are served at `/mcp` on the browser (Streamable HTTP, stateless, with +JSON answers), for apps that reach a local HTTP server (Claude Code, Codex). It +accepts only 127.0.0.1/localhost as host and no web page as origin, so +another tab in a web browser can't use it. Apps that start their own server +over stdio (Claude Desktop, the ChatGPT desktop app) go through a small proxy that passes each message +on to `/mcp`: `cfabric-browser mcp` (`stdio.py`), or the Desktop Extension's +own (`packaging/mcpb/`). + +All tools but `propose_search` are marked read-only, so that apps which ask +before each tool that changes something (Codex) ask only for the proposal. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from anyio import to_thread +from mcp.server.fastmcp import Context, FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from mcp.types import ToolAnnotations + +# calls a tool by name with its arguments; returns what the tool returns +Call = Callable[[str, dict[str, Any]], Awaitable[Any]] + +INSTRUCTIONS = """\ +You are helping a user search corpora (the Hebrew Bible as the BHSA, and +others) that are open in CACCHT Explorer. The user says in their +own words what they want to find; you write a search template for it. + +The results always come from the browser's search, never from your own +knowledge: don't answer with counts or lists of passages yourself. + +1. list_corpora, then corpus_guide for the corpus (it explains the features + and conventions, with examples); describe_corpus, list_features and + describe_feature for more. +2. Write a template (search_syntax_guide explains the language) and check it + with check_template. +3. Run it as a trial with try_search: look at the count and the examples. If + there are 0 results, read the hints; if far too many, narrow it. Repeat. +4. propose_search with the template, the user's request in their own words + and a short explanation of each line. The browser shows it to the user, + who runs it with Run. Tell the user it is there, with the trial's count. +""" + +# the header in which `cfabric-browser mcp` passes on the app's name, from +# its `initialize` (`/mcp` is stateless, so it never sees that itself) +CLIENT_HEADER = "X-Cfabric-Client" + +# the names apps give themselves (`clientInfo.name`, or in their User-Agent), +# as the user knows them; the first that occurs in it +CLIENTS = ( + ("chatgpt", "ChatGPT"), + ("codex", "Codex"), + ("claude", "Claude"), + ("antigravity", "Gemini"), + ("gemini", "Gemini"), +) + +READS = ToolAnnotations(readOnlyHint=True, openWorldHint=False) +# it shows something in the browser, and changes nothing else +PROPOSES = ToolAnnotations( + readOnlyHint=False, destructiveHint=False, idempotentHint=False, openWorldHint=False +) + +# the tools, by name +TOOLS = ( + "list_corpora", + "corpus_guide", + "describe_corpus", + "list_features", + "describe_feature", + "search_syntax_guide", + "check_template", + "try_search", + "propose_search", +) + + +def build_server(call: Call, name: str = "CACCHT Explorer") -> FastMCP: + server = FastMCP( + name, + instructions=INSTRUCTIONS, + stateless_http=True, + json_response=True, + streamable_http_path="/mcp", + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=["127.0.0.1:*", "localhost:*", "[::1]:*"], + # no web page may call it, not even the browser's own + allowed_origins=[], + ), + ) + + @server.tool(annotations=READS) + async def list_corpora() -> dict[str, Any]: + """The corpora open in the browser: their names (to pass to the other + tools), titles, ids, and section types (book, chapter, verse).""" + return await call("list_corpora", {}) + + @server.tool(annotations=READS) + async def corpus_guide(corpus: str) -> dict[str, Any]: + """A guide to searching this corpus: its object types, its main + features in plain words, conventions (such as how lexemes are + written), worked examples and common mistakes. Read it first.""" + return await call("corpus_guide", {"corpus": corpus}) + + @server.tool(annotations=READS) + async def describe_corpus(corpus: str) -> dict[str, Any]: + """The corpus' object types (word, phrase, clause, verse, ...) with + their numbers, and its sections.""" + return await call("describe_corpus", {"corpus": corpus}) + + @server.tool(annotations=READS) + async def list_features( + corpus: str, node_types: list[str] | None = None + ) -> dict[str, Any]: + """The features of the corpus, with descriptions; only those of + `node_types` (such as ["word"]) if given.""" + return await call("list_features", {"corpus": corpus, "node_types": node_types}) + + @server.tool(annotations=READS) + async def describe_feature(corpus: str, features: list[str]) -> dict[str, Any]: + """Features in detail: their most frequent values with counts, so + that a template uses values the corpus has.""" + return await call("describe_feature", {"corpus": corpus, "features": features}) + + @server.tool(annotations=READS) + async def search_syntax_guide(section: str | None = None) -> dict[str, Any]: + """The search template language: objects on lines, indentation for + what lies inside what, features, relations and quantifiers. Without + `section`, an overview with the sections there are.""" + return await call("search_syntax_guide", {"section": section}) + + @server.tool(annotations=READS) + async def check_template(corpus: str, template: str) -> dict[str, Any]: + """Check a template without searching: its problems (syntax, unknown + object types or features), each with its line; `ok` when none.""" + return await call("check_template", {"corpus": corpus, "template": template}) + + @server.tool(annotations=READS) + async def try_search( + corpus: str, template: str, examples: int = 5 + ) -> dict[str, Any]: + """Run a template as a trial: the number of results (`atLeast` when + there are more than the trial counts), a few examples with their + passage and text, and for 0 results hints such as a value a feature + never has. For checking your template, not as the answer.""" + return await call( + "try_search", {"corpus": corpus, "template": template, "examples": examples} + ) + + @server.tool(annotations=PROPOSES) + async def propose_search( + corpus: str, + template: str, + request: str, + explanation: str, + model: str | None = None, + ctx: Context | None = None, + ) -> dict[str, Any]: + """Show the search in the user's browser, where they run it with Run, + edit it, or dismiss it. `request`: what the user asked, in their own + words; `explanation`: what each line of the template does, briefly; + `model`: the name of the AI app or model, as the user should see it + (such as "Claude" or "ChatGPT"). + A template with problems is not shown: they come back to be fixed.""" + return await call( + "propose_search", + { + "corpus": corpus, + "template": template, + "request": request, + "explanation": explanation, + "model": model or client_name(ctx), + }, + ) + + return server + + +def readable(name: str) -> str | None: + """The name of an app as the user knows it ("codex-mcp-client" is + Codex), None for one we don't know.""" + lowered = name.lower() + for part, known in CLIENTS: + if part in lowered: + return known + return None + + +def client_name(ctx: Context | None) -> str | None: + """The name of the app that called, for a proposal without `model`: from + the stdio helper's header, else the User-Agent.""" + try: + request = ctx.request_context.request if ctx is not None else None + except ValueError: # not in a request + return None + if request is None: + return None + headers = request.headers + return readable(headers.get(CLIENT_HEADER) or headers.get("user-agent") or "") + + +def local(tools: Any) -> Call: + """Calls on the browser's own tools (for `/mcp`), in a thread: a trial + search waits for the pool.""" + from cfabric_browser.assistant.tools import ToolError + + async def call(name: str, args: dict[str, Any]) -> Any: + if name not in TOOLS: + raise ValueError(f"No tool {name}") + + def run() -> Any: + try: + return getattr(tools, name)(**args) + except ToolError as e: + return {"ok": False, "error": str(e)} + + return await to_thread.run_sync(run) + + return call diff --git a/libs/explorer/cfabric_browser/assistant/proposals.py b/libs/explorer/cfabric_browser/assistant/proposals.py new file mode 100644 index 0000000..5299fb7 --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/proposals.py @@ -0,0 +1,99 @@ +"""Searches proposed by a language model, kept until the user runs or +dismisses them. + +A proposal has the user's request in their own words, the template, the +model's explanation, which model made it and when, and the trial's count. +The open page follows the proposals live (`/api/proposals/events`); nothing +runs until the user presses *Run*. +""" + +from __future__ import annotations + +import datetime +import threading +import uuid +from dataclasses import asdict, dataclass, field +from typing import Any + +NEW = "new" +RUN = "run" +DISMISSED = "dismissed" +STATES = (NEW, RUN, DISMISSED) +KEEP = 50 # proposals kept, the newest + + +@dataclass +class Proposal: + id: str + corpus: str + template: str + request: str + explanation: str + model: str | None + created: str + # the trial before proposing: its count, and whether that is a lower bound + count: int | None = None + atLeast: bool = False + state: str = NEW + + +@dataclass +class ProposalStore: + proposals: list[Proposal] = field(default_factory=list) + version: int = 0 + _cond: threading.Condition = field(default_factory=threading.Condition, repr=False) + + def add( + self, + corpus: str, + template: str, + request: str, + explanation: str, + model: str | None = None, + count: int | None = None, + atLeast: bool = False, + ) -> dict[str, Any]: + proposal = Proposal( + id=uuid.uuid4().hex[:12], + corpus=corpus, + template=template, + request=request, + explanation=explanation, + model=model, + created=datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), + count=count, + atLeast=atLeast, + ) + with self._cond: + self.proposals.append(proposal) + del self.proposals[:-KEEP] + self.version += 1 + self._cond.notify_all() + return asdict(proposal) + + def set_state(self, pid: str, state: str) -> dict[str, Any] | None: + if state not in STATES: + raise ValueError(f"Unknown state {state!r}") + with self._cond: + for p in self.proposals: + if p.id == pid: + p.state = state + self.version += 1 + self._cond.notify_all() + return asdict(p) + return None + + def snapshot(self) -> dict[str, Any]: + """All proposals, the newest first, and the version they are at.""" + with self._cond: + return { + "version": self.version, + "proposals": [asdict(p) for p in reversed(self.proposals)], + } + + def wait_change(self, version: int, timeout: float) -> int: + """Block until the proposals change after `version`, or the timeout.""" + with self._cond: + if self.version == version: + self._cond.wait(timeout) + return self.version diff --git a/libs/explorer/cfabric_browser/assistant/running.py b/libs/explorer/cfabric_browser/assistant/running.py new file mode 100644 index 0000000..1eccd1a --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/running.py @@ -0,0 +1,69 @@ +"""Where a running browser can be reached, for a language model's tools. + +At its start the browser writes its address to `~/.cfabric/running.json`; +the stdio helper (`cfabric-browser mcp`) reads it to forward a model's tool +calls. (Other web pages are kept out by `cfabric_browser.guard`, and `/mcp` +by its own checks of host and origin.) Several browsers can run at once +(windows on other ports): each has an entry, the newest last, and removes it +when it stops. An entry of a browser that was killed stays; readers try the +newest that answers. +""" + +from __future__ import annotations + +import datetime +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +RUNNING = "~/.cfabric/running.json" + + +def read(path: str | Path = RUNNING) -> list[dict[str, Any]]: + """The browsers that said they run, the newest last.""" + try: + data = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + found = data.get("browsers") if isinstance(data, dict) else None + return [b for b in found or [] if isinstance(b, dict) and "url" in b] + + +def _write(path: Path, browsers: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + # written next to it and renamed, readable by this user only + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".running-", suffix=".json") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump({"browsers": browsers}, f, indent=1) + os.chmod(tmp, 0o600) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +def announce(url: str, path: str | Path = RUNNING) -> dict[str, Any]: + """Add this browser (its process and address).""" + entry = { + "pid": os.getpid(), + "url": url, + "started": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), + } + place = Path(path).expanduser() + others = [b for b in read(place) if b.get("url") != url] + _write(place, [*others, entry]) + return entry + + +def withdraw(url: str, path: str | Path = RUNNING) -> None: + """Remove this browser's entry, when it stops.""" + place = Path(path).expanduser() + found = read(place) + left = [ + b for b in found if not (b.get("url") == url and b.get("pid") == os.getpid()) + ] + if len(left) != len(found): + _write(place, left) diff --git a/libs/explorer/cfabric_browser/assistant/stdio.py b/libs/explorer/cfabric_browser/assistant/stdio.py new file mode 100644 index 0000000..cd9fdee --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/stdio.py @@ -0,0 +1,276 @@ +"""`cfabric-browser mcp`: the browser's MCP server over stdio, for AI apps +that start their own servers (Claude Desktop, the ChatGPT desktop app). + +It is a small proxy: each JSON-RPC message from the app (one per line on +stdin) goes to the running browser's `/mcp` as an HTTP request, and the answer +comes back as a line on stdout. The browser's `/mcp` is stateless and answers +with JSON, so nothing else is needed. The browser is found in +`~/.cfabric/running.json`; when none answers, one is started. + +What doesn't need the browser is answered here: `initialize`, `ping`, the +lists of tools, prompts and resources (from the same server definition, +`mcp.build_server`), and notifications. Apps give a server only seconds to +start (Codex: 10), and starting the browser takes longer, so it is looked +for, or started, in the background from the first message, and only a tool +call waits for it. The app's name from `initialize` goes with each tool call +(`mcp.CLIENT_HEADER`), for proposals that don't name their model. + +Nothing but those messages may be written to stdout. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from collections.abc import Awaitable, Callable, Sequence +from pathlib import Path +from typing import Any, BinaryIO + +import anyio +from mcp.server.fastmcp import FastMCP +from mcp.shared.version import SUPPORTED_PROTOCOL_VERSIONS +from mcp.types import LATEST_PROTOCOL_VERSION + +from cfabric_browser.assistant import mcp, running + +START_TIMEOUT = 120.0 # seconds to wait for a browser that was started +RETRY_START = 300.0 # seconds before trying again to start one that failed + + +def find_browser(path: str | Path = running.RUNNING) -> str | None: + """The address of the newest running browser that answers.""" + for entry in reversed(running.read(path)): + try: + with urllib.request.urlopen(entry["url"] + "api/health", timeout=3): + return str(entry["url"]) + except (OSError, ValueError): + continue + return None + + +def start_browser(path: str | Path = running.RUNNING) -> str | None: + """Start the browser in the background (the program itself when frozen), + and wait until it announces itself.""" + if getattr(sys, "frozen", False): + command = [sys.executable] + else: + command = [sys.executable, "-m", "cfabric_browser.cli"] + options: dict[str, Any] = { + "stdin": subprocess.DEVNULL, + "stdout": subprocess.DEVNULL, + "stderr": subprocess.DEVNULL, + } + if os.name == "nt": + options["creationflags"] = ( + subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP + ) + else: + options["start_new_session"] = True + subprocess.Popen(command, **options) + deadline = time.monotonic() + START_TIMEOUT + while time.monotonic() < deadline: + found = find_browser(path) + if found: + return found + time.sleep(1) + return None + + +class Browser: + """The running browser's address, looked for (and the browser started if + need be) in a thread of its own, from the first time it is asked for.""" + + def __init__(self, path: str | Path, start: bool) -> None: + self.path = path + self.start = start + self._url: str | None = None + self._looking: threading.Thread | None = None + # when starting it last failed: not tried again for a while, so that + # every message doesn't start (and wait for) another one + self._failedStart: float | None = None + + def look(self) -> None: + """Start looking, unless it is found or being looked for.""" + if self._url is None and self._looking is None: + self._looking = threading.Thread(target=self._find, daemon=True) + self._looking.start() + + def _find(self) -> None: + self._url = find_browser(self.path) + if self._url is not None or not self.start: + return + failed = self._failedStart + if failed is not None and time.monotonic() - failed < RETRY_START: + return + self._url = start_browser(self.path) + self._failedStart = None if self._url else time.monotonic() + + def url(self) -> str | None: + """The address, after waiting for the looking; None if there is no + browser.""" + self.look() + if self._looking is not None: + self._looking.join() + self._looking = None + return self._url + + def lost(self) -> None: + """It stopped, or restarted on another port: look again next time.""" + self._url = None + + +def _not_here(name: str, args: dict[str, Any]) -> Any: + raise RuntimeError("tools run in the browser") + + +class Local: + """The answers that don't need the browser, from the server definition + the browser itself serves.""" + + def __init__(self) -> None: + self.server: FastMCP = mcp.build_server(_not_here) + + def answer(self, message: dict[str, Any]) -> dict[str, Any] | None: + """The result for a message answered here, None for one that goes to + the browser.""" + method = message.get("method") + if method == "initialize": + return self.initialize(message.get("params") or {}) + if method == "ping": + return {} + lists: dict[str, tuple[str, Callable[[], Awaitable[Sequence[Any]]]]] = { + "tools/list": ("tools", self.server.list_tools), + "prompts/list": ("prompts", self.server.list_prompts), + "resources/list": ("resources", self.server.list_resources), + "resources/templates/list": ( + "resourceTemplates", + self.server.list_resource_templates, + ), + } + if method in lists: + key, get = lists[method] + items = anyio.run(get) + return {key: [dump(item) for item in items]} + return None + + def initialize(self, params: dict[str, Any]) -> dict[str, Any]: + """As the SDK answers it: the version the app asks for if we know it, + else our latest.""" + options = self.server._mcp_server.create_initialization_options() + asked = params.get("protocolVersion") + result = { + "protocolVersion": asked + if asked in SUPPORTED_PROTOCOL_VERSIONS + else LATEST_PROTOCOL_VERSION, + "capabilities": dump(options.capabilities), + "serverInfo": { + "name": options.server_name, + "version": options.server_version, + }, + } + if options.instructions: + result["instructions"] = options.instructions + return result + + +def dump(item: Any) -> Any: + """A pydantic object of the SDK as JSON, as it sends them.""" + return item.model_dump(by_alias=True, mode="json", exclude_none=True) + + +def forward(url: str, message: bytes, client: str | None = None) -> bytes | None: + """One message to the browser's /mcp; its answer, None for none (a + notification). `client`: the app's name, from its `initialize`.""" + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if client: + headers[mcp.CLIENT_HEADER] = client + request = urllib.request.Request(url + "mcp", data=message, headers=headers) + with urllib.request.urlopen(request, timeout=180) as response: + body = response.read() + return body if body.strip() else None + + +def error(message: dict[str, Any], text: str) -> bytes | None: + """A JSON-RPC error for a request (none for a notification).""" + if "id" not in message: + return None + answer = { + "jsonrpc": "2.0", + "id": message["id"], + "error": {"code": -32603, "message": text}, + } + return json.dumps(answer).encode() + + +def serve( + stdin: BinaryIO | None = None, + stdout: BinaryIO | None = None, + path: str | Path = running.RUNNING, + start: bool = True, +) -> int: + source = stdin or sys.stdin.buffer + sink = stdout or sys.stdout.buffer + browser = Browser(path, start) + local = Local() + client: str | None = None + for line in source: + if not line.strip(): + continue + try: + message = json.loads(line) + except ValueError: + continue + if not isinstance(message, dict): + continue + browser.look() + if message.get("method") == "initialize": + info = (message.get("params") or {}).get("clientInfo") or {} + client = str(info.get("name") or "") or None + if "id" not in message: + # a notification: the stateless /mcp does nothing with one + continue + answer: bytes | None + result = local.answer(message) + if result is not None: + answer = json.dumps( + {"jsonrpc": "2.0", "id": message["id"], "result": result} + ).encode() + else: + answer = call_browser(browser, message, line, client) + if answer is not None: + sink.write(answer.rstrip() + b"\n") + sink.flush() + return 0 + + +def call_browser( + browser: Browser, message: dict[str, Any], line: bytes, client: str | None +) -> bytes | None: + """A message for the browser, with a second try when it was lost.""" + answer: bytes | None = None + for _ in range(2): + url = browser.url() + if url is None: + return error( + message, + "CACCHT Explorer is not running and could not be " + "started. Start it, then try again.", + ) + try: + return forward(url, line, client) + except urllib.error.HTTPError as e: + return error(message, f"The browser answered {e.code}") + except OSError: + # it stopped, or restarted on another port: look again + browser.lost() + answer = error(message, "Lost the connection to the browser") + return answer diff --git a/libs/explorer/cfabric_browser/assistant/tools.py b/libs/explorer/cfabric_browser/assistant/tools.py new file mode 100644 index 0000000..cb7932d --- /dev/null +++ b/libs/explorer/cfabric_browser/assistant/tools.py @@ -0,0 +1,243 @@ +"""The tools a language model gets, on the browser's open corpora. + +Describing corpora and features and the template language are `cfabric-mcp`'s +tools, run on the browser's registry (`cfabric_mcp.tools.using`), so the two +don't drift apart. Checking a template is the browser's own check, and trial +searches go through the browser's search pool, with its result cap, timeout +and cancelling, as any typed search does. +""" + +from __future__ import annotations + +import re +import time +from importlib import resources +from typing import Any + +from cfabric_mcp import tools as shared + +from cfabric_browser import formats +from cfabric_browser.assistant.proposals import ProposalStore +from cfabric_browser.corpora import CorpusService +from cfabric_browser.display import section_ref_for +from cfabric_browser.queries import check_template +from cfabric_browser.workers import DONE, FINISHED, SearchPool + +# a trial: the count up to this many results, within this many seconds, and +# a few examples +TRIAL_LIMIT = 10_000 +TRIAL_TIMEOUT = 20.0 +EXAMPLES = 5 +# the text shown of a node in an example +EXAMPLE_TEXT = 120 +# for a trial that ran out of time: what to do, rather than the app cutting +# off a tool that seems to hang (Codex does after 60 seconds) +TOO_SLOW = ( + f"The trial stopped after {TRIAL_TIMEOUT:.0f} seconds: the template is " + "too slow to try. Narrow it: put the most selective feature on the first " + "object, or add a condition, and try again." +) + + +# `feature=value` in a template (not `feature~regex`, `feature None: + self.service = service + self.pool = pool + self.proposals = proposals if proposals is not None else ProposalStore() + # per (corpus, feature) the values it has + self._values: dict[tuple[str, str], set[str]] = {} + + def _corpus(self, corpus: str) -> str: + if not self.service.has(corpus): + open_ = ", ".join(self.service.names()) or "none" + raise ToolError(f"No corpus {corpus!r} is open; open: {open_}") + return corpus + + # describing + + def list_corpora(self) -> dict[str, Any]: + """The open corpora: name, title, id and whether it is a translation.""" + found = [] + for name in self.service.names(): + card = self.service.card(name) + found.append( + { + "name": name, + "title": card["title"], + "id": card["id"], + "translation": bool(card["translation"]), + "sectionTypes": card["sectionTypes"], + } + ) + return {"corpora": found} + + def describe_corpus(self, corpus: str) -> dict[str, Any]: + with shared.using(self.service.registry): + return shared.describe_corpus(self._corpus(corpus)) + + def list_features( + self, corpus: str, node_types: list[str] | None = None + ) -> dict[str, Any]: + with shared.using(self.service.registry): + return shared.list_features( + node_types=node_types, corpus=self._corpus(corpus) + ) + + def describe_feature(self, corpus: str, features: list[str]) -> dict[str, Any]: + with shared.using(self.service.registry): + return shared.describe_features(features, corpus=self._corpus(corpus)) + + def search_syntax_guide(self, section: str | None = None) -> dict[str, Any]: + return shared.search_syntax_guide(section) + + def corpus_guide(self, corpus: str) -> dict[str, Any]: + """The hand-written guide to searching this corpus, if there is one + (`guides/.md`, by the corpus' id `org/repo`).""" + card = self.service.card(self._corpus(corpus)) + repo = (card["id"] or "").split("/")[-1].lower() + guide = resources.files("cfabric_browser.assistant").joinpath( + "guides", f"{repo}.md" + ) + if repo and guide.is_file(): + return {"corpus": corpus, "guide": guide.read_text(encoding="utf-8")} + return { + "corpus": corpus, + "guide": None, + "note": "No guide for this corpus yet: use describe_corpus, " + "list_features and describe_feature.", + } + + # checking and trying + + def check_template(self, corpus: str, template: str) -> dict[str, Any]: + """The template's problems (syntax, unknown types, features and values) + without searching; `ok` when there are none.""" + api = self.service.api(self._corpus(corpus)) + problems = check_template(api, template) + return {"ok": not problems, "problems": problems} + + def try_search( + self, corpus: str, template: str, examples: int = EXAMPLES + ) -> dict[str, Any]: + """Run a template as a trial: how many results (up to `TRIAL_LIMIT`, + within `TRIAL_TIMEOUT` seconds) and the first few, with their passage + and text, for the model to judge its template by.""" + corpus = self._corpus(corpus) + checked = self.check_template(corpus, template) + if not checked["ok"]: + return {"ok": False, "problems": checked["problems"]} + job = self.pool.submit( + corpus, template, limit=TRIAL_LIMIT, timeout=TRIAL_TIMEOUT + ) + version = -1 + deadline = time.monotonic() + TRIAL_TIMEOUT + 10 + while job.state not in FINISHED and time.monotonic() < deadline: + version = job.wait_change(version, 1.0) + if job.state not in FINISHED: + self.pool.cancel(job.id) + found = job.snapshot() + result: dict[str, Any] = { + "ok": found["state"] == DONE and not found["problems"], + "state": found["state"], + "count": found["count"], + # more results than the trial counts: `count` is a lower bound + "atLeast": bool(found["truncated"]) or found["state"] != DONE, + "seconds": found["elapsed"], + "problems": found["problems"], + "examples": [ + self._example(corpus, result) + for result in job.page(0, max(0, min(examples, 20))) + ], + } + if found["state"] == DONE and not found["count"]: + result["hints"] = self.unknown_values(corpus, template) + elif found["state"] != DONE and not found["problems"]: + result["note"] = TOO_SLOW + return result + + def unknown_values(self, corpus: str, template: str) -> list[str]: + """For an empty result: the `feature=value` conditions whose value the + feature never has (a misspelled lexeme such as `>LHM/`).""" + api = self.service.api(corpus) + hints = [] + for feature, value in EQUALS.findall(template): + if feature not in api.Fall(): + continue + key = (corpus, feature) + if key not in self._values: + self._values[key] = {str(v) for (v, _) in api.Fs(feature).freqList()} + if value not in self._values[key]: + hints.append(f"{feature} never has the value {value}") + return hints + + def _example(self, corpus: str, nodes: tuple[int, ...]) -> dict[str, Any]: + api = self.service.api(corpus) + config = self.service.config(corpus) + fmt = formats.default(config) + F, T = api.F, api.T + return { + "ref": section_ref_for(api, config, nodes[0]), + "nodes": [ + { + "node": int(n), + "type": F.otype.v(n), + "text": T.text(n, fmt=fmt).strip()[:EXAMPLE_TEXT], + } + for n in nodes + ], + } + + # proposing: the only tool that changes what the user sees + + def propose_search( + self, + corpus: str, + template: str, + request: str, + explanation: str, + model: str | None = None, + ) -> dict[str, Any]: + """Show a search in the browser, for the user to run, edit or dismiss: + the template, the user's request in their words, and an explanation + of the template. A template with problems is not shown; they come + back to be fixed. Nothing runs until the user presses *Run*.""" + corpus = self._corpus(corpus) + trial = self.try_search(corpus, template, examples=0) + if trial.get("problems") or trial.get("state") not in (DONE, None): + refused = { + "ok": False, + "problems": trial.get("problems"), + "state": trial.get("state"), + } + if "note" in trial: + refused["note"] = trial["note"] + return refused + proposal = self.proposals.add( + corpus, + template, + request.strip(), + explanation.strip(), + model, + count=trial["count"], + atLeast=trial["atLeast"], + ) + return { + "ok": True, + "proposal": proposal["id"], + "count": trial["count"], + "atLeast": trial["atLeast"], + "note": "Shown in the browser; the user runs it with Run.", + } diff --git a/libs/explorer/cfabric_browser/books.py b/libs/explorer/cfabric_browser/books.py new file mode 100644 index 0000000..3ba2203 --- /dev/null +++ b/libs/explorer/cfabric_browser/books.py @@ -0,0 +1,107 @@ +"""A concordance of biblical book names across corpora. + +Corpora name their books differently: `1_Samuel` (BHSA), `1Sam` (LXX), +`Samuel_1` (Peshitta). To show the same passage in several corpora, book +names are mapped to one canonical id (OSIS book ids). Only mappings that are +safe are listed: for instance the LXX's `2Esdr` (Ezra and Nehemiah together) +is not mapped to Ezra, because its chapters do not correspond. + +Names are compared after normalisation: lower case, without spaces, +underscores and punctuation, so `Song_of_songs`, `Song of Songs` and +`songofsongs` are the same name. +""" + +from __future__ import annotations + +import re + +# canonical id: other names (normalised automatically) +BOOKS: dict[str, tuple[str, ...]] = { + "Gen": ("Genesis", "Ge", "Gn"), + "Exod": ("Exodus", "Ex", "Exo"), + "Lev": ("Leviticus", "Lv"), + "Num": ("Numbers", "Nu", "Nm"), + "Deut": ("Deuteronomy", "Dt", "Deu"), + "Josh": ("Joshua", "Jos"), + "Judg": ("Judges", "Jdg", "Jdgs"), + "Ruth": ("Ru", "Rut"), + "1Sam": ("1 Samuel", "Samuel 1", "1Sa", "I Samuel"), + "2Sam": ("2 Samuel", "Samuel 2", "2Sa", "II Samuel"), + "1Kgs": ("1 Kings", "Kings 1", "1Ki", "I Kings"), + "2Kgs": ("2 Kings", "Kings 2", "2Ki", "II Kings"), + "1Chr": ("1 Chronicles", "Chronicles 1", "1Ch", "I Chronicles"), + "2Chr": ("2 Chronicles", "Chronicles 2", "2Ch", "II Chronicles"), + "Ezra": ("Ezr",), + "Neh": ("Nehemiah", "Nehemia"), + "Esth": ("Esther", "Est"), + "Job": ("Jb",), + "Ps": ("Psalms", "Psalm", "Psa", "Pss"), + "Prov": ("Proverbs", "Pr", "Prv"), + "Eccl": ("Ecclesiastes", "Qoh", "Qohelet", "Ec", "Ecc"), + "Song": ("Song of Songs", "Song of Solomon", "Cant", "Canticles", "SoS"), + "Isa": ("Isaiah", "Is"), + "Jer": ("Jeremiah", "Je"), + "Lam": ("Lamentations", "La"), + "Ezek": ("Ezekiel", "Eze", "Ezk"), + "Dan": ("Daniel", "Da", "Dn"), + "Hos": ("Hosea", "Ho"), + "Joel": ("Jl",), + "Amos": ("Am",), + "Obad": ("Obadiah", "Ob"), + "Jonah": ("Jon",), + "Mic": ("Micah", "Mi"), + "Nah": ("Nahum", "Na"), + "Hab": ("Habakkuk", "Hb"), + "Zeph": ("Zephaniah", "Zep"), + "Hag": ("Haggai", "Hg"), + "Zech": ("Zechariah", "Zec"), + "Mal": ("Malachi", "Ml"), + # books outside the Hebrew canon + # where a corpus has two forms of a text, both point to the book; the + # first form found in a corpus is used (see book_index) + "Tob": ("Tobit", "Tb", "TobBA", "TobS", "Tobit A", "Tobit B"), + "Jdt": ("Judith", "Jth"), + "Wis": ("Wisdom", "Wisdom of Solomon", "Ws"), + "Sir": ("Sirach", "Ecclesiasticus", "Ben Sira"), + "Bar": ("Baruch",), + "EpJer": ("Letter of Jeremiah", "Epistle of Jeremiah"), + "Sus": ("Susanna",), + "Bel": ("Bel and the Dragon",), + "1Macc": ("1 Maccabees", "1Mac", "Maccabees 1", "Maccabees 1 A", "Maccabees 1 B"), + "2Macc": ("2 Maccabees", "2Mac", "Maccabees 2"), + "3Macc": ("3 Maccabees", "3Mac", "Maccabees 3"), + "4Macc": ("4 Maccabees", "4Mac", "Maccabees 4"), + "1Esd": ("1 Esdras", "1Esdr"), + "PrMan": ("Prayer of Manasseh", "Prayer of Manasseh A", "Prayer of Manasseh B"), + "Odes": ("Od", "Ode"), + "PssSol": ("Psalms of Solomon", "PsSol"), +} + + +def normalise(name: str) -> str: + return re.sub(r"[^0-9a-z]", "", str(name).lower()) + + +ALIASES: dict[str, str] = {} +for canonical, others in BOOKS.items(): + for name in (canonical, *others): + ALIASES.setdefault(normalise(name), canonical) + + +def canonical_book(name: str) -> str | None: + """The canonical id of a book name, or None if it is not in the concordance.""" + return ALIASES.get(normalise(name)) + + +def book_index(names: list[str]) -> dict[str, str]: + """Map canonical ids to a corpus' own book names. + + When a corpus has several books for one id (the LXX has `Dan` and + `DanTh`), the first one, normally the main text, is used. + """ + index: dict[str, str] = {} + for name in names: + canonical = canonical_book(name) + if canonical is not None: + index.setdefault(canonical, name) + return index diff --git a/libs/explorer/cfabric_browser/cancel.py b/libs/explorer/cfabric_browser/cancel.py new file mode 100644 index 0000000..708f1c2 --- /dev/null +++ b/libs/explorer/cfabric_browser/cancel.py @@ -0,0 +1,84 @@ +""" +# Search cancellation + +Cooperative cancellation and timeouts for searches. + +The browser's search workers (`workers.py`) check a `CancelToken` between the +results they get from the core's `S.search`; when it has been cancelled, or +its deadline has passed, `check()` raises `SearchCancelled` (or +`SearchTimeout`). Neither core checks it while it is still matching, so a +search that doesn't get to its results in time is stopped by ending its +worker process (the pool's watchdog). Moved here from the core (it was only +on the browser's branches), so that the browser works with any core. +""" + +from __future__ import annotations + +import threading +import time + + +class SearchCancelled(Exception): + """Raised when a search is stopped through its `CancelToken`.""" + + +class SearchTimeout(SearchCancelled): + """Raised when a search exceeds the timeout of its `CancelToken`.""" + + +class CancelToken: + """A thread-safe handle to stop a running search. + + Parameters + ---------- + timeout: float, optional None + Number of seconds after creation of the token after which the search + is stopped. `None` means no timeout. + + Examples + -------- + >>> token = CancelToken(timeout=10) + >>> results = api.S.search(template, cancel=token) + + From another thread: + + >>> token.cancel() + """ + + __slots__ = ("_deadline", "_event", "timeout") + + def __init__(self, timeout: float | None = None) -> None: + self._event = threading.Event() + self.timeout = timeout + self._deadline = ( + None if timeout is None else time.monotonic() + max(0.0, timeout) + ) + + def cancel(self) -> None: + """Request the search to stop at its next check.""" + self._event.set() + + @property + def cancelled(self) -> bool: + """Whether `cancel()` has been called.""" + return self._event.is_set() + + @property + def expired(self) -> bool: + """Whether the timeout has passed.""" + return self._deadline is not None and time.monotonic() > self._deadline + + def check(self) -> None: + """Raise if the search should stop. + + Raises + ------ + SearchCancelled + If `cancel()` has been called. + SearchTimeout + If the timeout has passed. + """ + if self._event.is_set(): + raise SearchCancelled("search cancelled") + if self._deadline is not None and time.monotonic() > self._deadline: + raise SearchTimeout(f"search exceeded timeout of {self.timeout} seconds") diff --git a/libs/explorer/cfabric_browser/cli.py b/libs/explorer/cfabric_browser/cli.py new file mode 100644 index 0000000..1857f36 --- /dev/null +++ b/libs/explorer/cfabric_browser/cli.py @@ -0,0 +1,400 @@ +"""The `cfabric-browser` command, also installed as `caccht-explorer`.""" + +from __future__ import annotations + +import argparse +import atexit +import logging +import sys +import threading +import time +import webbrowser +from pathlib import Path +from typing import TYPE_CHECKING + +from cfabric_browser import __version__ + +if TYPE_CHECKING: + from cfabric_browser.dictionaries.manage import Catalogued + +DEFAULT_PORT = 8107 + + +def _prog() -> str: + """The command's name as it was called, for help and usage.""" + called = Path(sys.argv[0]).stem if sys.argv and sys.argv[0] else "" + return "caccht-explorer" if called == "caccht-explorer" else "cfabric-browser" + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog=_prog(), + description="CACCHT Explorer: browse, search and compare Context-Fabric corpora, offline.", + epilog=( + "Corpora can be given as a data directory, a repository in " + "Text-Fabric layout, or a name such as 'bhsa' or 'ETCBC/dss' that " + "is looked up in ~/text-fabric-data/github and ~/github. " + "Prefix with 'name=' to choose the name shown in the browser. " + "Without corpora, the browser starts empty and offers the corpora on " + "this computer to open. 'cfabric-browser add' lists the translations " + "and dictionaries that can be added (the JPS 1917, BDB); " + "'cfabric-browser add ID' adds one ('add bdb FILE' from a file built " + "here) and 'cfabric-browser remove ID' removes it. 'cfabric-browser " + "download' lists the corpora that can be downloaded (the BHSA, the SP " + "and the DSS), and 'cfabric-browser download bhsa sp dss' downloads " + "them." + ), + ) + parser.add_argument("corpora", nargs="*", metavar="CORPUS") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument( + "--port", + type=int, + default=None, + help=f"default {DEFAULT_PORT}; with --window the first free one from there", + ) + parser.add_argument("--workers", type=int, default=2, help="parallel searches") + parser.add_argument( + "--timeout", type=float, default=60.0, help="time limit per search (seconds)" + ) + parser.add_argument( + "--max-results", type=int, default=500_000, help="results kept per search" + ) + parser.add_argument( + "--no-open", action="store_true", help="do not open a web browser" + ) + parser.add_argument( + "--window", + action=argparse.BooleanOptionalAction, + default=False, + help="show the browser in a window of its own (needs pywebview);" + " closing it stops the server", + ) + parser.add_argument( + "--hide", + action="append", + default=[], + metavar="ORG/REPO", + help="leave this corpus out of the library (can be repeated); see also" + " library: hide: in ~/.cfabric/browser.yaml", + ) + parser.add_argument( + "--only", + action="append", + default=[], + metavar="ORG/REPO", + help="limit the library to this corpus (can be repeated), e.g. for a test" + " with a few corpora; see also library: only: in ~/.cfabric/browser.yaml", + ) + parser.add_argument( + "--no-library", + action="store_true", + help="do not let users open other corpora found on this computer", + ) + parser.add_argument( + "--ui", + choices=[ + "panes", + "classic", + "parallel", + "reader", + "interlinear", + "concordance", + ], + default=None, + help="the interface to start in until the user chooses one: panes, parallel," + " reader, interlinear or concordance; see also ui: in ~/.cfabric/browser.yaml", + ) + parser.add_argument( + "--ask-downloads", + action="store_true", + help="at the start, offer to download the BHSA, the SP and the DSS when they" + " are not on this computer (the standalone program does this)", + ) + parser.add_argument("--version", action="version", version=__version__) + return parser.parse_args(argv) + + +def download_command(names: list[str]) -> int: + """`cfabric-browser download [NAME ...]`: download corpora without starting + the browser; without names, list those that can be downloaded.""" + from cfabric_browser.corpora import SEARCH_ROOTS + from cfabric_browser.downloads import ( + DOWNLOADS, + DownloadError, + find, + get, + is_present, + ) + + root = SEARCH_ROOTS[0] + if not names: + print("Corpora that can be downloaded (cfabric-browser download NAME ...):") + for d in DOWNLOADS.values(): + here = any(is_present(d, r) for r in SEARCH_ROOTS) + state = "on this computer" if here else f"{d.size / 1e6:.0f} MB" + print(f" {d.name:<6} {d.id} {d.version}: {d.title} ({state})") + return 0 + wanted = [(name, find(name)) for name in names] + unknown = [name for (name, d) in wanted if d is None] + if unknown: + known = ", ".join(d.name for d in DOWNLOADS.values()) + print( + f"cfabric-browser: cannot download {', '.join(unknown)}; known: {known}", + file=sys.stderr, + ) + return 2 + + status = 0 + for _, d in wanted: + assert d is not None + if any(is_present(d, r) for r in SEARCH_ROOTS): + print(f"{d.id} {d.version} is already on this computer.") + continue + print( + f"Downloading {d.id} {d.version} ({d.size / 1e6:.0f} MB); terms of use: {d.terms}" + ) + + def progress(step: str, done: int, total: int) -> None: + if step == "download": + print( + f"\r {done / 1e6:.1f} of {total / 1e6:.1f} MB", end="", flush=True + ) + elif done == total: + print(f"\n unpacked {total} files", flush=True) + + try: + data = get(d, root=root, progress=progress) + except (DownloadError, OSError) as e: + print(f"\ncfabric-browser: {e}", file=sys.stderr) + status = 1 + continue + print(f" done: {data}") + return status + + +def dictionary_command(command: str, catalogued: Catalogued, rest: list[str]) -> int: + """`cfabric-browser add bdb [FILE]` and `cfabric-browser remove bdb`: add a + dictionary (downloaded, or from a file on this computer) or remove it.""" + from cfabric_browser.dictionaries.manage import add, add_file, remove + from cfabric_browser.dictionaries.store import DictionaryError + from cfabric_browser.downloads import DownloadError + + if command == "remove": + if not remove(catalogued): + print(f"{catalogued.name} has not been added.") + else: + print(f"Removed {catalogued.name}.") + return 0 + print(f"Adding {catalogued.name}") + print(f" {catalogued.licenceNote}") + + def progress(step: str, done: int, total: int) -> None: + print(f"\r downloading {done // 1000}/{total // 1000} kB", end="", flush=True) + + try: + place = ( + add_file(catalogued, rest[0]) + if rest + else add(catalogued, progress=progress) + ) + except (DictionaryError, DownloadError, OSError) as e: + print(f"\ncfabric-browser: {e}", file=sys.stderr) + return 1 + print(f"\n done: {place}") + print("It is shown on the pages of the lexicon (BHSA).") + return 0 + + +def translation_command(argv: list[str]) -> int: + """`cfabric-browser add [ID]` and `cfabric-browser remove ID`: add or + remove a translation without starting the browser.""" + from cfabric_browser.corpora import SEARCH_ROOTS + from cfabric_browser.dictionaries.manage import DICTIONARIES, is_added + from cfabric_browser.translations import TRANSLATIONS, BuildError, build + from cfabric_browser.translations.manage import is_built, remove + + root = SEARCH_ROOTS[0] + (command, *rest) = argv + if not rest: + if command == "remove": + print("cfabric-browser: say which translation to remove", file=sys.stderr) + return 2 + print("Translations that can be added (cfabric-browser add ID):") + for t in TRANSLATIONS.values(): + state = "added" if is_built(t, root) else "not added" + print(f" {t.id:<10} {t.name} ({state})") + print(f" {'':<10} {t.licence_note}") + print("Dictionaries that can be added (cfabric-browser add ID [FILE]):") + for d in DICTIONARIES.values(): + state = "added" if is_added(d) else "not added" + print(f" {d.id:<10} {d.name} ({state})") + print(f" {'':<10} {d.licenceNote}") + return 0 + if rest[0].lower() in DICTIONARIES: + return dictionary_command(command, DICTIONARIES[rest[0].lower()], rest[1:]) + translation = TRANSLATIONS.get(rest[0].lower()) + if translation is None: + known = ", ".join(TRANSLATIONS) + print( + f"cfabric-browser: no translation {rest[0]!r}; known: {known}", + file=sys.stderr, + ) + return 2 + + if command == "remove": + if not remove(translation, root): + print(f"{translation.title} has not been added.") + return 0 + print(f"Removed {translation.title}.") + return 0 + + print(f"Adding {translation.name}") + print(f" {translation.licence_note}") + shown = {"step": ""} + + def progress(step: str, done: int, total: int) -> None: + if step == "download": + print(f"\r downloading {done}/{total}", end="", flush=True) + elif step != shown["step"]: + print("\n building ...", flush=True) + shown["step"] = step + + try: + data = build(translation, root=root, progress=progress) + except (BuildError, OSError) as e: + print(f"\ncfabric-browser: {e}", file=sys.stderr) + return 1 + print(f" done: {data}") + print(f"Open it from the library, or: cfabric-browser bhsa {translation.id}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if argv and argv[0] in ("add", "remove"): + return translation_command(argv) + if argv and argv[0] == "download": + return download_command(argv[1:]) + if argv and argv[0] == "mcp": + # for an AI app (Claude Desktop): MCP over stdio, to the running browser + from cfabric_browser.assistant.stdio import serve + + return serve() + args = parse_args(argv) + if not args.corpora and args.no_library: + # nothing to show and no way to open anything + print( + "cfabric-browser: give at least one corpus, or leave out --no-library", + file=sys.stderr, + ) + return 2 + logging.basicConfig(level=logging.WARNING, format="%(message)s") + + import uvicorn + + from cfabric_browser.app import create_app + from cfabric_browser.corpora import CorpusService, load_settings, resolve_corpus + from cfabric_browser.guard import LOCAL_HOSTS + from cfabric_browser.interfaces import read_ui_settings, ui_id + from cfabric_browser.workers import SearchPool + + try: + specs = [resolve_corpus(c) for c in args.corpora] + except FileNotFoundError as e: + print(f"cfabric-browser: {e}", file=sys.stderr) + return 2 + names = [s.name for s in specs] + if len(set(names)) != len(names): + print( + "cfabric-browser: two corpora have the same name; use name=... to" + " tell them apart", + file=sys.stderr, + ) + return 2 + + # translations removed while they were open + from cfabric_browser.corpora import SEARCH_ROOTS + from cfabric_browser.translations.manage import remove_pending + + for removed in remove_pending(SEARCH_ROOTS[0]): + print(f"Removed the translation {removed}.", flush=True) + + settings = load_settings() + librarySettings = settings.get("library") or {} + hide = [*(librarySettings.get("hide") or ()), *args.hide] + only = [*(librarySettings.get("only") or ()), *args.only] + service = CorpusService(specs, hide=hide, only=only) + for spec in specs: + print(f"Loading {spec.name} from {spec.path} ...", flush=True) + t = time.perf_counter() + service.config(spec.name) + print(f" ready in {time.perf_counter() - t:.1f}s", flush=True) + if not specs: + print("No corpora given: open them in the browser.", flush=True) + + pool = SearchPool( + [(s.name, str(s.path)) for s in specs], + sort_key=lambda corpus: service.api(corpus).N.sortKeyTuple, + workers=args.workers, + max_results=args.max_results, + timeout=args.timeout, + ) + try: + (default, hidden) = read_ui_settings(settings.get("ui")) + ui = ui_id(args.ui) if args.ui else (default or "classic") + except ValueError as e: + print(f"cfabric-browser: {e} (ui: in browser.yaml)", file=sys.stderr) + return 2 + from cfabric_browser.assistant import running + + host = args.host + app = create_app( + service, + pool, + library=not args.no_library, + ui=ui, + hidden_uis=hidden, + ask_downloads=args.ask_downloads, + # reachable from other machines: under names we can't know + hosts=list(LOCAL_HOSTS) if host in ("127.0.0.1", "localhost", "::1") else None, + ) + + port = args.port + if port is None: + if args.window: + # a second window can run next to the first + from cfabric_browser.window import free_port + + port = free_port(host, DEFAULT_PORT) + else: + port = DEFAULT_PORT + url = f"http://{'localhost' if host in ('127.0.0.1', '0.0.0.0') else host}:{port}/" + if host not in ("127.0.0.1", "localhost", "::1"): + print( + "Warning: the browser is reachable from other machines on this network.", + file=sys.stderr, + ) + # for the stdio helper of a language model's app (cfabric-browser mcp) + local = f"http://127.0.0.1:{port}/" if host in ("127.0.0.1", "0.0.0.0") else url + try: + running.announce(local) + atexit.register(running.withdraw, local) + except OSError as e: + print(f"Note: could not write {running.RUNNING}: {e}", file=sys.stderr) + if args.window: + from cfabric_browser.window import run_window + + print(f"CACCHT Explorer at {url}, in its own window", flush=True) + return run_window(app, host, port, url) + + print(f"CACCHT Explorer at {url} (Ctrl+C to stop)", flush=True) + if not args.no_open: + threading.Timer(1.0, webbrowser.open, args=(url,)).start() + + uvicorn.run(app, host=host, port=port, log_level="warning") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/libs/explorer/cfabric_browser/compare.py b/libs/explorer/cfabric_browser/compare.py new file mode 100644 index 0000000..8dfa569 --- /dev/null +++ b/libs/explorer/cfabric_browser/compare.py @@ -0,0 +1,644 @@ +"""Textual differences between two corpora at the same passage. + +Two Hebrew texts (e.g. Genesis 1 in the MT/BHSA and in the SP) are compared +verse by verse on their consonants: + +* vowels, accents and other marks don't count; +* matres lectionis do count, but a difference made only of matres + (plene against defective spelling) is reported as `matres`, every other + difference as `variant`; +* word division doesn't count: the consonants of a verse are compared + without word boundaries, and the differences are mapped back to the + words that contain them; +* a plus of whole words is marked in the text that has it; the other text + has nothing to mark at that place. + +The consonants of a word come from `g_cons_utf8` when the corpus has it, and +are otherwise stripped from the word's text in the original script. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import Any + +from cfabric_browser.books import canonical_book +from cfabric_browser.corpora import CorpusService +from cfabric_browser.display import DisplayConfig, node_from_section_ref +from cfabric_browser.witnesses import ( + PLACEHOLDER_TYPES, + RECONSTRUCTED, + SIGN_TYPE, + format_ref, + parse_ref, + siglum_key, +) +from cfabric_browser.words import orig_format, word_type, words_of + +CONSONANT_FEATURE = "g_cons_utf8" +HEBREW_LETTER = re.compile("[א-ת]") +# final forms depend on word division, which doesn't count +FINALS = str.maketrans("ךםןףץ", "כמנפצ") +TO_FINAL = {normal: final for (final, normal) in zip("ךםןףץ", "כמנפצ")} + + +def with_finals(word: str) -> str: + """A word's consonants for people: with its last letter in final form.""" + return word[:-1] + TO_FINAL.get(word[-1], word[-1]) if word else word + + +MATRES = frozenset("אהוי") +UNKNOWN = "◦" # a letter that is there but cannot be read + +MATRES_ONLY = "matres" +VARIANT = "variant" + +Words = list[tuple[int, str]] +Marks = dict[int, str] + + +def consonants(text: str) -> str: + """The Hebrew letters of a text, with final forms made normal.""" + # decomposed, a precomposed shin with dot (U+FB2A, as in SP) is a plain shin + text = unicodedata.normalize("NFD", text) + return "".join(HEBREW_LETTER.findall(text.translate(FINALS))) + + +def verse_words(api, config: DisplayConfig, verse: int) -> Words: + """The words of a verse with their consonants.""" + wType = word_type(api, config) + feature = api.Fs(CONSONANT_FEATURE) if CONSONANT_FEATURE in api.Fall() else None + fmt = orig_format(config) + words = words_of(api, wType, verse) + result: Words = [] + for w in words: + w = int(w) + value = feature.v(w) if feature else None + if value is None: + value = api.T.text(w, fmt=fmt) + result.append((w, consonants(str(value)))) + return result + + +def _mark(marks: Marks, node: int, kind: str) -> None: + if marks.get(node) != VARIANT: + marks[node] = kind + + +@dataclass +class Unit: + """A place where two texts differ: letters `a[i1:i2]` against `b[j1:j2]`.""" + + kind: str + i1: int + i2: int + j1: int + j2: int + aWords: list[int] = field(default_factory=list) + bWords: list[int] = field(default_factory=list) + + +def diff_words( + a: Words, b: Words, known: list[bool] | None = None +) -> tuple[Marks, Marks]: + """The words of `a` and `b` that differ, each with the kind of difference. + + See `diff_units`. + """ + marksA: Marks = {} + marksB: Marks = {} + for unit in diff_units(a, b, known): + for w in unit.aWords: + _mark(marksA, w, unit.kind) + for w in unit.bWords: + _mark(marksB, w, unit.kind) + return marksA, marksB + + +def align_words(a: Words, b: Words) -> list[list[int]]: + """For each word of `a`, the words of `b` that correspond to it. + + As in the comparison, the consonants of each verse are matched without + word boundaries; a word of `b` goes with the word of `a` that its first + matching letter matches. A word of `b` without matching letters (a plus, + or a word written quite differently) goes with the word of `a` that the + word of `b` before it goes with (the first word of `a` at the start). + """ + textA = "".join(c for (_, c) in a) + textB = "".join(c for (_, c) in b) + ownerA = [i for (i, (_, c)) in enumerate(a) for _ in c] + ownerB = [j for (j, (_, c)) in enumerate(b) for _ in c] + assigned: dict[int, int] = {} + matcher = SequenceMatcher(None, textA, textB, autojunk=False) + for i, j, n in matcher.get_matching_blocks(): + for k in range(n): + assigned.setdefault(ownerB[j + k], ownerA[i + k]) + under: list[list[int]] = [[] for _ in a] + if not a: + return under + current = 0 + for j, (node, _) in enumerate(b): + current = assigned.get(j, current) + under[current].append(node) + return under + + +def diff_units(a: Words, b: Words, known: list[bool] | None = None) -> list[Unit]: + """The places where `a` and `b` differ, with the words that are marked there. + + The consonants are compared without word boundaries. Where letters are + present in one text only, the other text marks the word they fall into, + if they are attached to a word there; a plus of whole words leaves the + other text unmarked. + + `known` flags, per letter of `b`, whether it is preserved (a manuscript) + rather than reconstructed by an editor. Reconstructed letters keep the + texts aligned but are never marked, and neither is what `a` has in their + place. Letters of `a` that `b` lacks count only between two preserved + letters: at the edge of what survives, or next to a reconstruction, they + may just be lost. + """ + textA = "".join(c for (_, c) in a) + textB = "".join(c for (_, c) in b) + ownerA = [w for (w, c) in a for _ in c] + ownerB = [w for (w, c) in b for _ in c] + units: list[Unit] = [] + + def attached(owner: list[int], i: int, j: int) -> bool: + # letters i and j (j = i + 1) are in the same word + return 0 <= i and j < len(owner) and owner[i] == owner[j] + + def whole_words(owner: list[int], i1: int, i2: int) -> bool: + # letters i1..i2 are one or more complete words + return not attached(owner, i1 - 1, i1) and not attached(owner, i2 - 1, i2) + + def slide( + text: str, owner: list[int], lo: int, hi: int, left: int, right: int + ) -> int: + # Letters present in one text only can often sit in several places + # (בצלמנו ו כדמותנו: the extra ו is the word ו or the end of צלמנו). + # Shift them within the equal stretches around them (`left`, `right` + # letters) to the first place where they are whole words, if any. + shifts = [0] + s = 0 + while s < right and hi + s < len(text) and text[lo + s] == text[hi + s]: + s += 1 + shifts.append(s) + s = 0 + while s < left and lo - s > 0 and text[lo - s - 1] == text[hi - s - 1]: + s += 1 + shifts.append(-s) + for s in sorted(shifts): + if whole_words(owner, lo + s, hi + s): + return s + return 0 + + matcher = SequenceMatcher(None, textA, textB, autojunk=False) + opcodes = matcher.get_opcodes() + for k, (tag, i1, i2, j1, j2) in enumerate(opcodes): + if tag == "equal": + continue + if tag in ("insert", "delete"): + prev = opcodes[k - 1] if k > 0 and opcodes[k - 1][0] == "equal" else None + nxt = ( + opcodes[k + 1] + if k + 1 < len(opcodes) and opcodes[k + 1][0] == "equal" + else None + ) + left = prev[2] - prev[1] if prev else 0 + right = nxt[2] - nxt[1] if nxt else 0 + if tag == "insert": + s = slide(textB, ownerB, j1, j2, left, right) + else: + s = slide(textA, ownerA, i1, i2, left, right) + (i1, i2, j1, j2) = (i1 + s, i2 + s, j1 + s, j2 + s) + kept = [j for j in range(j1, j2) if known is None or known[j]] + if known is not None: + if tag != "delete" and not kept: + continue + if tag == "delete" and not ( + 0 < j1 < len(known) and known[j1 - 1] and known[j1] + ): + continue + # at the edge of what survives, `a` goes on where `b` is lost: + # only as many letters of `a` as `b` has stand against them + if tag == "replace" and j2 == len(known): + i2 = min(i2, i1 + (j2 - j1)) + if tag == "replace" and j1 == 0: + i1 = max(i1, i2 - (j2 - j1)) + # spelling: matres added or left out within a word; a word of its + # own (the article ה, the conjunction ו) or another letter is a variant + if tag == "insert": + spelling = {textB[j] for j in kept} <= MATRES and not whole_words( + ownerB, j1, j2 + ) + elif tag == "delete": + spelling = set(textA[i1:i2]) <= MATRES and not whole_words(ownerA, i1, i2) + else: + spelling = False + unit = Unit(MATRES_ONLY if spelling else VARIANT, i1, i2, j1, j2) + aWords = [ownerA[i] for i in range(i1, i2)] + bWords = [ownerB[j] for j in kept] + # letters only in b: mark the words of a they are attached to + if i1 == i2: + if attached(ownerB, j1 - 1, j1) and i1 > 0: + aWords.append(ownerA[i1 - 1]) + if attached(ownerB, j2 - 1, j2) and i1 < len(ownerA): + aWords.append(ownerA[i1]) + # letters only in a: likewise for b + if j1 == j2: + if attached(ownerA, i1 - 1, i1) and j1 > 0: + bWords.append(ownerB[j1 - 1]) + if attached(ownerA, i2 - 1, i2) and j1 < len(ownerB): + bWords.append(ownerB[j1]) + unit.aWords = sorted(set(aWords)) + unit.bWords = sorted(set(bWords)) + units.append(unit) + return units + + +def _verses(api, node: int) -> list[int]: + verseType = api.T.sectionTypes[-1] + if api.F.otype.v(node) == verseType: + return [node] + return [int(v) for v in api.L.d(node, otype=verseType)] + + +def compare_passages( + service: CorpusService, a: str, aRef: str, b: str, bRef: str, lang: str = "en" +) -> dict[str, Any]: + """Compare the passage `aRef` of corpus `a` with `bRef` of corpus `b`. + + The passages must correspond (`bRef` is where `aRef` aligns to in `b`); + their verses are paired by chapter and verse number. + + Raises + ------ + ValueError + If a reference is not a passage of its corpus. + """ + aApi, bApi = service.api(a), service.api(b) + aConfig, bConfig = service.config(a), service.config(b) + aNode = node_from_section_ref( + aApi, aRef, aConfig.section_sep1, aConfig.section_sep2, lang + ) + bNode = node_from_section_ref( + bApi, bRef, bConfig.section_sep1, bConfig.section_sep2, lang + ) + + def incomparable(reason: str) -> dict[str, Any]: + return {"comparable": False, "reason": reason, "verses": 0, "a": {}, "b": {}} + + aligned = service.align(a, aRef, b, lang) + if aligned is None or aligned["node"] != bNode: + return incomparable("The panes show different passages.") + bBook = bApi.T.sectionFromNode(bNode, lang="en")[0] + + marksA: Marks = {} + marksB: Marks = {} + paired = 0 + letters = False + for aVerse in _verses(aApi, aNode): + section = aApi.T.sectionFromNode(aVerse, lang="en") + bVerse = bApi.T.nodeFromSection((bBook, *section[1:]), lang="en") + if bVerse is None: + continue + aWords = verse_words(aApi, aConfig, aVerse) + bWords = verse_words(bApi, bConfig, int(bVerse)) + if not any(c for (_, c) in aWords) or not any(c for (_, c) in bWords): + continue + letters = True + paired += 1 + (mA, mB) = diff_words(aWords, bWords) + marksA.update(mA) + marksB.update(mB) + + if not letters: + return incomparable("Only Hebrew texts can be compared so far.") + return { + "comparable": True, + "reason": None, + "verses": paired, + "a": {str(n): k for (n, k) in marksA.items()}, + "b": {str(n): k for (n, k) in marksB.items()}, + } + + +def witness_letters(api, words: list[int]) -> tuple[Words, list[bool]]: + """The consonants of the words of a manuscript, and which are preserved. + + A letter is preserved unless its slot (sign) is flagged as reconstructed. + A lost or illegible sign (`ε`, `#`, `?` in the DSS) stands for letters + that are there but unknown: it becomes a placeholder that is not preserved. + """ + rec = api.Fs(RECONSTRUCTED).v if RECONSTRUCTED in api.Fall() else None + signType = api.Fs(SIGN_TYPE).v if SIGN_TYPE in api.Fall() else None + # a format of the slots: text-orig-extra (DSS) is one of words + formats = api.T.formats + fmt = "text-orig-full" if "text-orig-full" in formats else None + result: Words = [] + known: list[bool] = [] + for w in words: + letters = "" + for s in api.E.oslots.s(w): + if signType and signType(int(s)) in PLACEHOLDER_TYPES: + letters += UNKNOWN + known.append(False) + continue + c = consonants(str(api.T.text([int(s)], fmt=fmt))) + letters += c + known.extend([not (rec and rec(int(s)))] * len(c)) + result.append((w, letters)) + return result, known + + +def compare_witnesses( + service: CorpusService, a: str, aRef: str, b: str, bRef: str, lang: str = "en" +) -> dict[str, Any]: + """Compare passage `aRef` of `a` (MT, SP) with the scrolls at `bRef` of `b`. + + `bRef` is a passage in the book view of `b` (DSS). Each scroll that + contains a verse (a witness) is compared with that verse of `a`, its + reconstructed letters not counting (see `diff_words`). `a` marks a word + when any witness differs there; `aWitnesses` names them. + + Raises + ------ + ValueError + If `aRef` is not a passage of `a`. + """ + aApi, bApi = service.api(a), service.api(b) + aConfig = service.config(a) + index = service.book_index(b) + aNode = node_from_section_ref( + aApi, aRef, aConfig.section_sep1, aConfig.section_sep2, lang + ) + + def incomparable(reason: str) -> dict[str, Any]: + return {"comparable": False, "reason": reason, "verses": 0, "a": {}, "b": {}} + + parsed = parse_ref(bRef) + if index is None or parsed is None: + return incomparable(f"{b} cannot be read by biblical book.") + aligned = service.align(a, aRef, b, lang, view="books") + if aligned is None or aligned["ref"] != format_ref(*parsed): + return incomparable("The panes show different passages.") + + marksA: Marks = {} + marksB: Marks = {} + who: dict[int, set[str]] = {} + verses = 0 + letters = False + for aVerse in _verses(aApi, aNode): + section = aApi.T.sectionFromNode(aVerse, lang="en") + book = canonical_book(str(section[0])) + if book is None or len(section) < 3: + continue + (chapter, verse) = (int(section[1]), int(section[2])) + if not index.has(book, chapter, verse): + continue + aWords = verse_words(aApi, aConfig, aVerse) + if not any(c for (_, c) in aWords): + continue + verses += 1 + for witness in index.books[book][chapter][verse]: + (bWords, known) = witness_letters(bApi, witness.words) + if not any(known): + continue + letters = True + (mA, mB) = diff_words(aWords, bWords, known) + for node, kind in mA.items(): + _mark(marksA, node, kind) + who.setdefault(node, set()).add(witness.scroll) + marksB.update(mB) + + if not letters: + return incomparable("Only Hebrew texts can be compared so far.") + return { + "comparable": True, + "reason": None, + "verses": verses, + "a": {str(n): k for (n, k) in marksA.items()}, + "b": {str(n): k for (n, k) in marksB.items()}, + "aWitnesses": { + str(n): sorted(scrolls, key=siglum_key) for (n, scrolls) in who.items() + }, + } + + +# three texts: which scrolls agree with MT and which with SP + + +def align_letters(textA: str, textW: str) -> dict[int, int]: + """Which letter of `textW` each letter of `textA` is aligned with.""" + matcher = SequenceMatcher(None, textA, textW, autojunk=False) + return { + i + k: j + k for (i, j, n) in matcher.get_matching_blocks() for k in range(n) + } + + +def witness_reading( + textA: str, + textW: str, + known: list[bool], + i1: int, + i2: int, + aligned: dict[int, int] | None = None, +) -> tuple[int, str] | None: + """What a manuscript has where `a` has `textA[i1:i2]`, or None if unknown. + + Returns where the reading starts in `textW`, and the reading. + + The reading is what the manuscript has between the letters aligned with + the letters of `a` just before and just after: these anchors and all + letters between them must be preserved. At the start or end of the verse + the manuscript's own first or last letter is the anchor; there an empty + reading is unknown, because the text may just be lost. + + `aligned` is `align_letters(textA, textW)`, if already made (it is the + same for every place of a verse). + """ + if not textW: + return None + if aligned is None: + aligned = align_letters(textA, textW) + edge = i1 == 0 or i2 == len(textA) + if i1 > 0: + left = aligned.get(i1 - 1) + if left is None or not known[left]: + return None + else: + left = -1 + if not known[0]: + return None + if i2 < len(textA): + right = aligned.get(i2) + if right is None or not known[right]: + return None + else: + right = len(textW) + if not known[-1]: + return None + if right <= left or not all(known[left + 1 : right]): + return None + reading = textW[left + 1 : right] + if edge and not reading: + return None + return (left + 1, reading) + + +def merge_units(units: list[Unit]) -> list[Unit]: + """Join differences in the same words into one place. + + MT מארת against SP מאורות is two differences (two ו), but one place: a + scroll reading מארות agrees with neither as a whole. + """ + merged: list[Unit] = [] + for unit in units: + last = merged[-1] if merged else None + if last and ( + set(unit.aWords) & set(last.aWords) or set(unit.bWords) & set(last.bWords) + ): + last.i2 = max(last.i2, unit.i2) + last.j2 = max(last.j2, unit.j2) + last.aWords = sorted(set(last.aWords) | set(unit.aWords)) + last.bWords = sorted(set(last.bWords) | set(unit.bWords)) + if unit.kind == VARIANT: + last.kind = VARIANT + else: + merged.append( + Unit( + unit.kind, + unit.i1, + unit.i2, + unit.j1, + unit.j2, + unit.aWords, + unit.bWords, + ) + ) + return merged + + +def compare_three( + service: CorpusService, + a: str, + aRef: str, + b: str, + bRef: str, + c: str, + cRef: str, + lang: str = "en", +) -> dict[str, Any]: + """Which scrolls of `c` side with `a` (MT) and which with `b` (SP). + + `bRef` is the passage of `b` that corresponds to `aRef`, `cRef` the one in + the book view of `c` (DSS). For every place where `a` and `b` differ in a + verse (see `diff_units`), each scroll that has the verse is read there + (see `witness_reading`): it agrees with `a`, with `b`, or with neither. + Scrolls whose text is lost or reconstructed there are left out. + + Raises + ------ + ValueError + If a reference is not a passage of its corpus. + """ + aApi, bApi, cApi = service.api(a), service.api(b), service.api(c) + aConfig, bConfig = service.config(a), service.config(b) + index = service.book_index(c) + aNode = node_from_section_ref( + aApi, aRef, aConfig.section_sep1, aConfig.section_sep2, lang + ) + bNode = node_from_section_ref( + bApi, bRef, bConfig.section_sep1, bConfig.section_sep2, lang + ) + + def incomparable(reason: str) -> dict[str, Any]: + return {"comparable": False, "reason": reason, "verses": []} + + parsed = parse_ref(cRef) + if index is None or parsed is None: + return incomparable(f"{c} cannot be read by biblical book.") + alignedB = service.align(a, aRef, b, lang) + alignedC = service.align(a, aRef, c, lang, view="books") + if alignedB is None or alignedB["node"] != bNode: + return incomparable("The panes show different passages.") + if alignedC is None or alignedC["ref"] != format_ref(*parsed): + return incomparable("The panes show different passages.") + bBook = bApi.T.sectionFromNode(bNode, lang="en")[0] + + def words_text(words: Words, nodes: list[int]) -> str: + chosen = set(nodes) + return " ".join(with_finals(c) for (w, c) in words if w in chosen) + + verses = [] + for aVerse in _verses(aApi, aNode): + section = aApi.T.sectionFromNode(aVerse, lang="en") + book = canonical_book(str(section[0])) + if book is None or len(section) < 3: + continue + (chapter, verse) = (int(section[1]), int(section[2])) + bVerse = bApi.T.nodeFromSection((bBook, *section[1:]), lang="en") + if bVerse is None or not index.has(book, chapter, verse): + continue + aWords = verse_words(aApi, aConfig, aVerse) + bWords = verse_words(bApi, bConfig, int(bVerse)) + units = merge_units(diff_units(aWords, bWords)) + if not units: + continue + textA = "".join(c for (_, c) in aWords) + textB = "".join(c for (_, c) in bWords) + readings: list[list[dict[str, Any]]] = [[] for _ in units] + for witness in index.books[book][chapter][verse]: + (wWords, known) = witness_letters(cApi, witness.words) + textW = "".join(c for (_, c) in wWords) + if not any(known): + continue + ownerW = [w for (w, c) in wWords for _ in c] + aligned = align_letters(textA, textW) + for k, unit in enumerate(units): + found = witness_reading(textA, textW, known, unit.i1, unit.i2, aligned) + if found is None: + continue + (start, reading) = found + if reading == textA[unit.i1 : unit.i2]: + agrees = "a" + elif reading == textB[unit.j1 : unit.j2]: + agrees = "b" + else: + agrees = "other" + inSpan = sorted({ownerW[j] for j in range(start, start + len(reading))}) + readings[k].append( + { + "scroll": witness.scroll, + "agrees": agrees, + # the letters in which it differs, and its words there + "reading": reading, + "words": inSpan, + "text": words_text(wWords, inSpan), + } + ) + places = [ + { + "kind": unit.kind, + "a": words_text(aWords, unit.aWords), + "b": words_text(bWords, unit.bWords), + "aWords": unit.aWords, + "bWords": unit.bWords, + "witnesses": rs, + } + for (unit, rs) in zip(units, readings) + if rs + ] + if places: + verses.append( + { + "verse": verse, + "ref": format_ref(book, chapter, verse), + "units": places, + } + ) + return {"comparable": True, "reason": None, "verses": verses} diff --git a/libs/explorer/cfabric_browser/concordance.py b/libs/explorer/cfabric_browser/concordance.py new file mode 100644 index 0000000..e724ff5 --- /dev/null +++ b/libs/explorer/cfabric_browser/concordance.py @@ -0,0 +1,322 @@ +"""The results of a search as a concordance: lines with the hit in context. + +As in Sketch Engine's KWIC view (Key Word In Context), each result is a line +with its *hit* (the words of the result's innermost nodes) in the middle and a +number of words to the left and right of it, within the same book. Lines can +be sorted by a feature of the hit or of a word at a position before or after +it (`L1` is the word just before the hit, `R2` the second word after it), +filtered by features of the hit's first word, reduced to one per verse, and +sampled. Facets count the values of a feature of the hit (or its book), and +collocations count the values of a feature of the words near the hit, with +their logDice score against the corpus. + +Positions and sorting follow the corpus' word order, so for Hebrew "left" +means "before" (on the right of the page). +""" + +from __future__ import annotations + +import math +import random +import re +import threading +from bisect import bisect_left +from collections import Counter +from typing import Any + +from cfabric_browser import formats +from cfabric_browser.compare import consonants +from cfabric_browser.words import BookSpans, orig_format, word_type, words_of + +MAX_SIDE = 15 +POSITION = re.compile(r"^(hit|[LR][1-9][0-9]?)$") +BOOK = "book" # the pseudo-feature for the book of the hit +TEXT = "text" # the pseudo-feature for the word's consonants +# node spans kept per concordance (first slot, last slot, number of slots) +SPAN_CACHE = 200_000 + +_frequencies: dict[tuple[int, str], Counter] = {} +_frequencies_lock = threading.Lock() + + +def parse_filters(value: str | None) -> list[tuple[str, str]]: + """`vs:qal,sp:verb` → [("vs", "qal"), ("sp", "verb")].""" + filters = [] + for part in (value or "").split(","): + feature, sep, wanted = part.partition(":") + if sep and feature.strip(): + filters.append((feature.strip(), wanted.strip())) + return filters + + +class Spans: + """Which nodes of a result lie within others, by their slots; a node's + first and last slot and number of slots are kept, as the nodes of results + repeat (their book, chapter, clause), and a book's slots are many.""" + + def __init__(self, api) -> None: + self.api = api + self._spans: dict[int, tuple[int, int, int]] = {} + + def span(self, node: int) -> tuple[int, int, int]: + """A node's first and last slot and its number of slots.""" + span = self._spans.get(node) + if span is None: + if node <= self.api.F.otype.maxSlot: + span = (node, node, 1) + else: + slots = self.api.E.oslots.s(node) + span = (int(slots[0]), int(slots[-1]), len(slots)) + if len(self._spans) < SPAN_CACHE: + self._spans[node] = span + return span + + def within(self, m: int, n: int) -> bool: + """Whether m's slots are a part of n's (and fewer).""" + (fm, lm, km) = self.span(m) + (fn, ln, kn) = self.span(n) + if km >= kn or fm < fn or lm > ln: + return False + if ln - fn + 1 == kn: + return True # n has no gaps + sn = self.api.E.oslots.s(n) + for s in self.api.E.oslots.s(m) if km > 1 else (fm,): + k = bisect_left(sn, s) + if k == len(sn) or sn[k] != s: + return False + return True + + def innermost(self, nodes: list[int]) -> list[int]: + """The nodes that contain no other node of the list.""" + return [ + n for n in nodes if not any(m != n and self.within(m, n) for m in nodes) + ] + + +class Concordance: + """The results of one search, by their hit words.""" + + def __init__(self, api, config, results: list[tuple[int, ...]]): + self.api = api + self.config = config + self.wType = word_type(api, config) + (self.first, self.last) = (int(n) for n in api.F.otype.sInterval(self.wType)) + types = list(api.T.sectionTypes or ()) + self.bookType = types[0] if types else None + self.verseType = types[-1] if types else None + self.fmt = orig_format(config) + # values that mean "no value" (NA, absent, ...) count as missing + self.empty = {str(v) for v in config.none_values if v is not None} + self.books = BookSpans(api, self.wType) + self._bookNames: dict[int, str | None] = {} + self._values: dict[tuple[int, str], str | None] = {} + self.spans = Spans(api) + self.hits: list[list[int]] = [] + self.extend(results) + + def extend(self, results: list[tuple[int, ...]]) -> None: + """Add results (of a search still running) after the ones there.""" + # made first, then added at once, for requests reading the hits meanwhile + new = [self._hit(tup) for tup in results] + self.hits.extend(new) + + # words + + def _hit(self, tup: tuple[int, ...]) -> list[int]: + """The words of the result's innermost nodes, in order.""" + inner = self.spans.innermost([int(n) for n in tup]) + words = sorted({w for n in inner for w in words_of(self.api, self.wType, n)}) + return words + + def book(self, word: int) -> int | None: + return self.books.book(word) + + def neighbour(self, hit: list[int], position: str) -> int | None: + """The word at a position (`L2`, `R1`) of a hit, if it is in the same book.""" + if not hit: + return None + k = int(position[1:]) + word = hit[0] - k if position[0] == "L" else hit[-1] + k + if not self.first <= word <= self.last: + return None + if self.book(word) != self.book(hit[0]): + return None + return word + + def value(self, word: int | None, feature: str) -> str | None: + """A feature of a word: also `text` (its consonants) and `book`.""" + if word is None: + return None + api = self.api + if feature == TEXT: + return consonants(api.T.text(word, fmt=self.fmt)) + if feature == BOOK: + book = self.book(word) + if book is None: + return None + if book not in self._bookNames: + section = api.T.sectionFromNode(book, lang="en") + self._bookNames[book] = str(section[0]) if section else None + return self._bookNames[book] + key = (word, feature) + if key not in self._values: + if feature not in api.Fall(): + return None + value = api.Fs(feature).v(word) + self._values[key] = ( + None if value is None or str(value) in self.empty else str(value) + ) + return self._values[key] + + def verse(self, word: int) -> int | None: + ups = self.api.L.u(word, otype=self.verseType) if self.verseType else () + return int(ups[0]) if ups else None + + # selection + + def select( + self, + filters: list[tuple[str, str]] = (), + per_verse: bool = False, + sample: int | None = None, + seed: int = 1, + ) -> list[int]: + """The indexes of the results kept, in result order.""" + keep = [] + seen: set[int | None] = set() + for i, hit in enumerate(self.hits): + if not hit: + continue + if any(self.value(hit[0], f) != v for (f, v) in filters): + continue + if per_verse: + verse = self.verse(hit[0]) + if verse in seen: + continue + seen.add(verse) + keep.append(i) + if sample is not None and sample < len(keep): + keep = sorted(random.Random(seed).sample(keep, sample)) + return keep + + def sort(self, indexes: list[int], key: str | None) -> list[int]: + """Sort by `.` (`L1.lex`, `hit.text`); ties and + missing values keep the result order.""" + if not key: + return indexes + (position, _, feature) = key.partition(".") + if not POSITION.match(position) or not feature: + raise ValueError(f"Cannot sort by {key!r}: use e.g. hit.lex or L1.text") + + def word(i: int) -> int | None: + hit = self.hits[i] + return hit[0] if position == "hit" else self.neighbour(hit, position) + + values = {i: self.value(word(i), feature) for i in indexes} + return sorted(indexes, key=lambda i: (values[i] is None, values[i] or "", i)) + + # output + + def line( + self, i: int, left: int, right: int, features: list[str], fmt: str | None + ) -> dict[str, Any]: + api = self.api + hit = self.hits[i] + fmt = formats.chosen(self.config, fmt) or formats.default(self.config) + + def word(w: int) -> dict[str, Any]: + values = {f: self.value(w, f) for f in features} + return { + "node": w, + "text": api.T.text(w, fmt=fmt).strip(), + "features": {f: v for f, v in values.items() if v is not None}, + } + + before = [self.neighbour(hit, f"L{k}") for k in range(left, 0, -1)] + after = [self.neighbour(hit, f"R{k}") for k in range(1, right + 1)] + verse = self.verse(hit[0]) if hit else None + return { + "index": i, + "verse": verse, + "left": [word(w) for w in before if w is not None], + "hit": [word(w) for w in hit], + "right": [word(w) for w in after if w is not None], + } + + def facet( + self, indexes: list[int], feature: str, top: int = 50 + ) -> list[dict[str, Any]]: + """The values of a feature of the hits' first words, most frequent first; + for `book`, in the order of the books.""" + counts: Counter = Counter() + order: dict[str, int] = {} + for i in indexes: + hit = self.hits[i] + value = self.value(hit[0], feature) + if value is None: + continue + counts[value] += 1 + order.setdefault(value, hit[0]) + if feature == BOOK: + items = sorted(counts.items(), key=lambda kv: order[kv[0]]) + else: + items = counts.most_common(top) + return [{"value": v, "count": c} for (v, c) in items] + + def frequencies(self, corpus_key: int, feature: str) -> Counter: + """How often each value of a feature occurs on the corpus' words (cached).""" + key = (corpus_key, feature) + with _frequencies_lock: + if key not in _frequencies: + counts: Counter = Counter() + api = self.api + if feature in api.Fall() and feature not in (TEXT, BOOK): + for value, n in api.Fs(feature).freqList(nodeTypes={self.wType}): + if str(value) not in self.empty: + counts[str(value)] += n + else: + for w in range(self.first, self.last + 1): + value = self.value(w, feature) + if value is not None: + counts[value] += 1 + _frequencies[key] = counts + return _frequencies[key] + + def collocations( + self, + indexes: list[int], + feature: str, + window: int, + corpus_key: int, + min_count: int = 2, + top: int = 40, + ) -> list[dict[str, Any]]: + """Values of a feature of the words within `window` words of the hits, + with their count there, their frequency in the corpus and their logDice + score (14 + log2(2·count / (hits + frequency)); higher is more typical).""" + counts: Counter = Counter() + for i in indexes: + hit = self.hits[i] + for k in range(1, window + 1): + for position in (f"L{k}", f"R{k}"): + value = self.value(self.neighbour(hit, position), feature) + if value is not None: + counts[value] += 1 + frequency = self.frequencies(corpus_key, feature) + hits = len(indexes) + rows = [] + for value, count in counts.items(): + if count < min_count: + continue + freq = frequency.get(value, count) + score = 14 + math.log2(2 * count / (hits + freq)) + rows.append( + { + "value": value, + "count": count, + "frequency": freq, + "logDice": round(score, 2), + } + ) + rows.sort(key=lambda r: (-r["logDice"], -r["count"])) + return rows[:top] diff --git a/libs/explorer/cfabric_browser/corpora.py b/libs/explorer/cfabric_browser/corpora.py new file mode 100644 index 0000000..2ad9bc7 --- /dev/null +++ b/libs/explorer/cfabric_browser/corpora.py @@ -0,0 +1,631 @@ +"""Finding corpora on disk and holding them while the browser runs. + +A corpus can be given as: + +* a data directory with `.tf` files and/or a `.cfm` cache; +* a repository directory in Text-Fabric layout (`/tf//`), + in which case the version named in the app config, or else the newest, + is used; +* a short name (`bhsa`) or `org/repo` (`ETCBC/bhsa`), looked up under + `~/text-fabric-data/github` and `~/github`, where Text-Fabric keeps + downloaded and cloned corpora; + +optionally prefixed with `name=` to choose the name shown in the browser. +""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from cfabric_browser import formats +from cfabric_browser.books import book_index, canonical_book +from cfabric_browser.display import ( + DisplayConfig, + find_app_config, + node_from_section_ref, + section_ref_for, +) +from cfabric_browser.display.config import read_app_config +from cfabric_browser.helpers import versionSort +from cfabric_browser.lexicon import Lexicon, has_lexicon +from cfabric_browser.registry import CorpusRegistry +from cfabric_browser.witnesses import BookIndex, format_ref, parse_ref + +logger = logging.getLogger(__name__) + +SEARCH_ROOTS = ("~/text-fabric-data/github", "~/github") +OVERRIDES = "~/.cfabric/display" +# display settings that come with the browser, for corpora whose app config +# lacks them; local ones (OVERRIDES) win +BUILT_IN = Path(__file__).parent / "display_settings" + + +@dataclass(frozen=True) +class CorpusSpec: + """A corpus to serve: its name in the browser and its data directory.""" + + name: str + path: Path + + def to_dict(self) -> dict[str, str]: + return {"name": self.name, "path": str(self.path)} + + +def corpus_id(path: Path) -> str | None: + """`org/repo` of a data directory in Text-Fabric layout (`//tf/`), + e.g. `ETCBC/bhsa`; None for a data directory elsewhere.""" + if path.parent.name != "tf": + return None + repo = path.parent.parent + return f"{repo.parent.name}/{repo.name}" + + +def _is_data_dir(path: Path) -> bool: + return (path / ".cfm").is_dir() or any(path.glob("otype.tf")) + + +LATEST = "latest" + + +def _pick_version(repo: Path, version: str | None = None) -> Path | None: + """The data directory of a repository in TF layout (`/tf/`). + + Without `version`: the version named in the app config, or else the newest. + With `version`: that version, or the newest for `latest`. + + Raises + ------ + FileNotFoundError + If the requested version is not on disk; the message lists those that are. + """ + tfDir = repo / "tf" + if not tfDir.is_dir(): + return None + versions = [d for d in tfDir.iterdir() if d.is_dir() and _is_data_dir(d)] + if not versions: + return None + + if version == LATEST: + return max(versions, key=lambda d: versionSort(d.name)) + if version is not None: + for d in versions: + if d.name == version: + return d + available = ", ".join(sorted((d.name for d in versions), key=versionSort)) + raise FileNotFoundError( + f"{repo.parent.name}/{repo.name} has no version {version}; " + f"on this computer: {available}" + ) + + appConfig = repo / "app" / "config.yaml" + if appConfig.is_file(): + try: + wanted = (read_app_config(appConfig).get("provenanceSpec") or {}).get( + "version" + ) + except Exception: # noqa: BLE001 - a broken config should not block startup + wanted = None + if wanted is not None: + for d in versions: + if d.name == str(wanted): + return d + + return max(versions, key=lambda d: versionSort(d.name)) + + +def _find_repo(name: str, roots: tuple[str, ...]) -> Path | None: + parts = name.strip("/").split("/") + for root in roots: + base = Path(root).expanduser() + if not base.is_dir(): + continue + if len(parts) == 2: + candidate = base / parts[0] / parts[1] + if candidate.is_dir(): + return candidate + # organisation names differ in case between sources (ETCBC / etcbc) + for org in base.iterdir(): + if org.name.lower() == parts[0].lower(): + for repo in org.iterdir(): + if repo.name.lower() == parts[1].lower(): + return repo + elif len(parts) == 1: + for org in sorted(base.iterdir()): + if not org.is_dir(): + continue + for repo in org.iterdir(): + if repo.name.lower() == parts[0].lower() and (repo / "tf").is_dir(): + return repo + return None + + +def resolve_corpus(spec: str, roots: tuple[str, ...] = SEARCH_ROOTS) -> CorpusSpec: + """Turn a command-line corpus specification into a name and a data directory. + + A version can be chosen with `@`: `DT-UCPH/sp@7.2.4`, or `sp@latest` for + the newest version on this computer. + + Raises + ------ + FileNotFoundError + If no corpus data can be found for the specification. + """ + name: str | None = None + target = spec + if "=" in spec: + (name, target) = spec.split("=", 1) + name = name.strip() or None + + version: str | None = None + path = Path(target).expanduser() + if not path.is_dir() and "@" in target: + (target, version) = target.rsplit("@", 1) + version = version.strip() or None + path = Path(target).expanduser() + + dataDir: Path | None = None + repo: Path | None = None + + if path.is_dir(): + if _is_data_dir(path): + dataDir = path + else: + dataDir = _pick_version(path, version) + repo = path + else: + repo = _find_repo(target, roots) + if repo is not None: + dataDir = _pick_version(repo, version) + + if dataDir is None: + raise FileNotFoundError(f"No corpus data found for '{spec}'") + + if name is None: + if repo is not None: + name = repo.name + elif dataDir.parent.name == "tf": + name = dataDir.parent.parent.name + else: + name = dataDir.name + return CorpusSpec(name=name.lower(), path=dataDir.resolve()) + + +def _app_title(repo: Path) -> str | None: + config = repo / "app" / "config.yaml" + if not config.is_file(): + return None + try: + provenance = read_app_config(config).get("provenanceSpec") or {} + except Exception: # noqa: BLE001 - a broken config only costs the title + return None + title = provenance.get("corpus") + return str(title) if title else None + + +def discover( + roots: tuple[str, ...] = SEARCH_ROOTS, + hide: tuple[str, ...] | list[str] = (), + collapse: bool = True, + only: tuple[str, ...] | list[str] = (), +) -> list[dict[str, Any]]: + """Corpora on this computer, in Text-Fabric layout (`//tf/`). + + Each entry has the version that opening it would load (`version`), besides + all versions on disk. A corpus found under several roots is listed once, + from the first root. + + Parameters + ---------- + hide: ids (`org/repo`) to leave out, e.g. corpora someone does not use. + only: if given, the ids (`org/repo`) to consider; all others are left out, + e.g. for a test with a few corpora. + collapse: list corpora with the same title (copies and forks of one + corpus, such as `DT-UCPH/sp` and `saulocantanhede/sp`) once, keeping the + one with the newest data; the others are named in `copies`. + """ + hidden = {h.lower() for h in hide} + allowed = {o.lower() for o in only} + found: dict[str, dict[str, Any]] = {} + for root in roots: + base = Path(root).expanduser() + if not base.is_dir(): + continue + for org in sorted(base.iterdir()): + if not org.is_dir() or org.name.startswith("."): + continue + for repo in sorted(org.iterdir()): + tfDir = repo / "tf" + # .jps1917.building: a translation being built (translations/) + if repo.name.startswith(".") or not tfDir.is_dir(): + continue + versions = sorted( + (d.name for d in tfDir.iterdir() if d.is_dir() and _is_data_dir(d)), + key=versionSort, + ) + if not versions: + continue + key = f"{org.name}/{repo.name}".lower() + if key in found or key in hidden or (allowed and key not in allowed): + continue + default = _pick_version(repo) + found[key] = { + "id": f"{org.name}/{repo.name}", + "name": repo.name.lower(), + "title": _app_title(repo), + "version": default.name if default else versions[-1], + "versions": versions, + } + entries = sorted(found.values(), key=lambda c: c["id"].lower()) + if not collapse: + return entries + + byTitle: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + if entry["title"]: + byTitle.setdefault(entry["title"].strip().lower(), []).append(entry) + dropped: set[str] = set() + for group in byTitle.values(): + if len(group) < 2: + continue + keep = max(group, key=lambda c: versionSort(c["versions"][-1])) + keep["copies"] = [c["id"] for c in group if c is not keep] + dropped.update(c["id"] for c in group if c is not keep) + return [e for e in entries if e["id"] not in dropped] + + +SETTINGS = "~/.cfabric/browser.yaml" + + +def load_settings(path: str | Path = SETTINGS) -> dict[str, Any]: + """Personal browser settings, e.g. corpora to leave out of the library:: + + library: + hide: + - CLARIAH/descartes-tf + """ + file = Path(path).expanduser() + if not file.is_file(): + return {} + try: + settings: dict[str, Any] = read_app_config(file) + return settings + except Exception: # noqa: BLE001 - a broken settings file must not stop the browser + logger.warning("Could not read browser settings from %s", file) + return {} + + +def _merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]: + """Deep merge of two config mappings; `extra` wins.""" + merged = dict(base) + for key, value in extra.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge(merged[key], value) + else: + merged[key] = value + return merged + + +def display_overrides( + path: Path, overrides: str | Path = OVERRIDES, built_in: Path = BUILT_IN +) -> tuple[dict[str, Any], list[Path]]: + """Display settings for a corpus, layered over its app config. + + For data in `//tf/`, these files are read, later ones + winning: `//.yaml`, `//@.yaml` + (those that come with the browser), then the same two in `` + (local ones). They use the format of a Text-Fabric app `config.yaml` + (`typeDisplay`, `dataDisplay`, ...) and need only contain what differs. + This is the place for settings that a corpus' own app config does not + (yet) have, without editing downloaded files. + """ + if path.parent.name != "tf": + return ({}, []) + repo = path.parent.parent + merged: dict[str, Any] = {} + used: list[Path] = [] + for root in (built_in, Path(overrides).expanduser()): + base = root / repo.parent.name + for candidate in ( + base / f"{repo.name}.yaml", + base / f"{repo.name}@{path.name}.yaml", + ): + if candidate.is_file(): + merged = _merge(merged, read_app_config(candidate)) + used.append(candidate) + return (merged, used) + + +class CorpusService: + """The corpora served by the browser, with their display configurations.""" + + def __init__( + self, + specs: list[CorpusSpec] | None = None, + roots: tuple[str, ...] = SEARCH_ROOTS, + overrides: str | Path = OVERRIDES, + hide: tuple[str, ...] | list[str] = (), + only: tuple[str, ...] | list[str] = (), + ) -> None: + self.registry = CorpusRegistry() + self.roots = roots # where the library looks for corpora + self.overrides = overrides # local display settings, see display_overrides + self.hide = tuple(hide) # library ids not to list + self.only = tuple(only) # if given, the only library ids to list and open + self._specs: dict[str, CorpusSpec] = {} + self._configs: dict[str, DisplayConfig] = {} + self._books: dict[str, dict[str, str]] = {} + self._bookIndexes: dict[str, BookIndex | None] = {} + self._lexicons: dict[str, Lexicon | None] = {} + self._translations: dict[str, dict[str, Any] | None] = {} + self._lock = threading.Lock() + # one build at a time per (cache, corpus): requests that come in together + # wait for it instead of building the same thing again + self._buildLocks: dict[tuple[int, str], threading.Lock] = {} + for spec in specs or (): + self.add(spec) + + def add(self, spec: CorpusSpec) -> None: + self.registry.register(spec.path, spec.name) + self._specs[spec.name] = spec + + def spec(self, name: str) -> CorpusSpec: + return self._specs[name] + + @property + def specs(self) -> list[CorpusSpec]: + return list(self._specs.values()) + + def names(self) -> list[str]: + return list(self._specs) + + def has(self, name: str) -> bool: + return name in self._specs + + def load_all(self) -> None: + for name in self._specs: + self.config(name) + + def api(self, name: str): + return self.registry.get_api(name) + + def _cached[T](self, cache: dict[str, T], name: str, build: Callable[[], T]) -> T: + """What `cache` holds for a corpus, built once by `build`.""" + with self._lock: + if name in cache: + return cache[name] + lock = self._buildLocks.setdefault((id(cache), name), threading.Lock()) + with lock: + with self._lock: + if name in cache: + return cache[name] + value = build() + with self._lock: + cache[name] = value + return value + + def config(self, name: str) -> DisplayConfig: + def build() -> DisplayConfig: + api = self.api(name) + path = self._specs[name].path + appConfig = find_app_config(path) + settings = read_app_config(appConfig) if appConfig else {} + (extra, used) = display_overrides(path, self.overrides) + for file in used: + logger.info("Display settings for %s from %s", name, file) + config = DisplayConfig.from_api(api, config=_merge(settings, extra)) + config.source = appConfig + return config + + return self._cached(self._configs, name, build) + + def title(self, name: str) -> str: + """The corpus' title, without loading it (that can take minutes the + first time).""" + with self._lock: + config = self._configs.get(name) + if config is not None: + provenance = config.provenance + else: + path = self._specs[name].path + appConfig = find_app_config(path) + try: + settings = read_app_config(appConfig) if appConfig else {} + except Exception: # noqa: BLE001 - a broken config only costs the title + settings = {} + (extra, _) = display_overrides(path, self.overrides) + provenance = _merge(settings, extra).get("provenanceSpec") or {} + return str(provenance.get("corpus") or name) + + def books(self, name: str) -> dict[str, str]: + """Canonical book ids of a corpus, mapped to its own book names.""" + + def build() -> dict[str, str]: + api = self.api(name) + T = api.T + names = ( + [ + str(T.sectionFromNode(int(n), lang="en")[0]) + for n in api.F.otype.s(T.sectionTypes[0]) + ] + if T.sectionTypes + else [] + ) + return book_index(names) + + return self._cached(self._books, name, build) + + def book_index(self, name: str) -> BookIndex | None: + """The biblical books in a corpus of manuscripts (DSS), if it has them. + + Only for corpora whose own sections are not books: see `witnesses.py`. + """ + return self._cached( + self._bookIndexes, + name, + lambda: None if self.books(name) else BookIndex.build(self.api(name)), + ) + + def lexicon(self, name: str) -> Lexicon | None: + """The lexicon of a corpus with lexeme nodes (BHSA), if it has one.""" + return self._cached(self._lexicons, name, lambda: Lexicon.build(self.api(name))) + + def glosses(self, name: str) -> tuple[Any, str] | None: + """For a corpus without glosses (the DSS): a lookup of glosses by ETCBC + lexeme in an open corpus with a lexicon (the BHSA), and its name.""" + if getattr(self.api(name).F, "gloss", None) is not None: + return None + for other in self.names(): + if other == name: + continue + lexicon = self.lexicon(other) + if lexicon is not None: + return (lexicon.gloss, other) + return None + + def align( + self, + source: str, + ref: str, + target: str, + lang: str = "en", + view: str = "sections", + ) -> dict[str, Any] | None: + """The passage in `target` that corresponds to `ref` in `source`. + + Books are matched through the book concordance (`1_Samuel` in one + corpus is `1Sam` in another); chapters and verses by number. When the + verse (or chapter) does not exist in the target, the nearest enclosing + section that does is returned, with `exact` false. + + Returns None when the target has no counterpart, e.g. a book it does + not contain, or a corpus with a different kind of sections. + + `ref` may also be a book view reference of `source` (`Genesis 1:1` in + the DSS); with `view="books"` the result is one in the book view of + `target` (its `node` is None there). + + Raises + ------ + ValueError + If `ref` is not a passage of `source`. + """ + sApi, tApi = self.api(source), self.api(target) + sConfig, tConfig = self.config(source), self.config(target) + section: tuple + sIndex = self.book_index(source) + parsed = parse_ref(ref) if sIndex else None + if parsed: + (book, chapter, verse) = parsed + section = (book, chapter) if verse is None else parsed + else: + node = node_from_section_ref( + sApi, ref, sConfig.section_sep1, sConfig.section_sep2, lang + ) + section = sApi.T.sectionFromNode(node, lang="en") + if not section: + return None + + book = str(section[0]) + canonical = canonical_book(book) + if view == "books": + return self._align_books(target, canonical, section) + if not tApi.T.sectionTypes: + return None + tBook = self.books(target).get(canonical) if canonical else None + if tBook is None: + # corpora with other kinds of sections: same name or nothing + tBook = book + parts = (tBook, *section[1 : len(tApi.T.sectionTypes)]) + for k in range(len(parts), 0, -1): + tNode = tApi.T.nodeFromSection(parts[:k], lang="en") + if tNode is not None: + return { + "node": int(tNode), + "ref": section_ref_for(tApi, tConfig, int(tNode), lang=lang), + "exact": k == len(section), + } + return None + + def _align_books( + self, target: str, canonical: str | None, section: tuple + ) -> dict[str, Any] | None: + index = self.book_index(target) + if index is None or canonical is None or canonical not in index.books: + return None + if len(section) < 2: + chapter = next(iter(index.books[canonical])) + return {"node": None, "ref": format_ref(canonical, chapter), "exact": False} + chapter = int(section[1]) + if not index.has(canonical, chapter): + return None + if len(section) > 2 and index.has(canonical, chapter, int(section[2])): + ref = format_ref(canonical, chapter, int(section[2])) + return {"node": None, "ref": ref, "exact": True} + return { + "node": None, + "ref": format_ref(canonical, chapter), + "exact": len(section) == 2, + } + + def translation(self, name: str) -> dict[str, Any] | None: + """What the app config says about a translation (`translation:`, written + by `cfabric_browser.translations`): licence, source, how to cite it. + None for a corpus that is not a translation.""" + with self._lock: + if name in self._translations: + return self._translations[name] + found = None + config = find_app_config(self._specs[name].path) + if config is not None: + try: + found = read_app_config(config).get("translation") or None + except Exception: # noqa: BLE001 - a broken config: not a translation + found = None + with self._lock: + self._translations[name] = found + return found + + def card(self, name: str) -> dict[str, Any]: + """Everything a client needs to know about a corpus to work with it.""" + api = self.api(name) + config = self.config(name) + F = api.F + T = api.T + types = [] + for nType in F.otype.all: + (first, last) = F.otype.sInterval(nType) + types.append( + { + "type": nType, + "count": int(last) - int(first) + 1, + "isSlot": nType == F.otype.slotType, + } + ) + provenance = config.provenance + return { + "name": name, + "id": corpus_id(self._specs[name].path), + "title": provenance.get("corpus") or name, + # the version actually loaded, which may differ from the app config's + "version": self._specs[name].path.name, + "doi": provenance.get("doi"), + "hasAppConfig": find_app_config(self._specs[name].path) is not None, + "slotType": F.otype.slotType, + "maxSlot": int(F.otype.maxSlot), + "maxNode": int(F.otype.maxNode), + "nodeTypes": types, + "sectionTypes": list(T.sectionTypes or ()), + # whether the corpus can also be read by biblical book (DSS) + "bookView": self.book_index(name) is not None, + # whether the corpus has a lexicon with glosses (BHSA) + "lexicon": has_lexicon(api), + # a translation (JPS 1917): licence, source, how to cite it + "translation": self.translation(name), + # only the text formats the browser offers (formats.py) + "display": formats.display(config), + } diff --git a/libs/explorer/cfabric_browser/dictionaries/__init__.py b/libs/explorer/cfabric_browser/dictionaries/__init__.py new file mode 100644 index 0000000..a4b5b11 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/__init__.py @@ -0,0 +1 @@ +"""Dictionaries for the lexicon of a corpus (first: BDB, for the BHSA).""" diff --git a/libs/explorer/cfabric_browser/dictionaries/bdb.py b/libs/explorer/cfabric_browser/dictionaries/bdb.py new file mode 100644 index 0000000..b223d89 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/bdb.py @@ -0,0 +1,216 @@ +"""BDB's entries: Open Scriptures' ids and short definitions joined with +Sefaria's text, and Sefaria's HTML cleaned. + +Open Scriptures' `BrownDriverBriggs.xml` gives every entry an id (`a.ac.aa`: +letter, root group, entry), its part of speech and short definitions, and the +links from the BHSA go to these ids. Its text is half skeletons, so the text +comes from Sefaria. Both follow BDB's order, so the two lists are joined by +order and by the consonants of their headwords (`join`). + +Sefaria's HTML is reduced to a few tags (`clean`): references to verses +become ``, references to other entries +``; any tag not in `TAGS` stops the build, so that nothing +slips in unseen. +""" + +from __future__ import annotations + +import html +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from html.parser import HTMLParser +from pathlib import Path + +from cfabric_browser.books import canonical_book +from cfabric_browser.dictionaries.link import OSHB, consonants +from cfabric_browser.dictionaries.sefaria import plain + +# tags kept, with the attributes kept +TAGS: dict[str, tuple[str, ...]] = { + "big": (), + "sub": (), + "sup": (), + "strong": (), + "em": (), + "br": (), + "span": ("dir",), + "a": (), # rewritten: data-osis or data-entry +} +VOID = {"br"} +# BDB's sign "<" (rather, better) is usually followed by a space, once by a +# name (" str: + return self.id.rsplit(".", 1)[0] + + +def read_entries(path: Path) -> list[Entry]: + """The entries of `BrownDriverBriggs.xml`, in its order.""" + root = ET.parse(path).getroot() + entries = [] + for part in root.iter(OSHB + "part"): + aramaic = part.get("{http://www.w3.org/XML/1998/namespace}lang") == "arc" + for e in part.iter(OSHB + "entry"): + w = e.find(OSHB + "w") + head = "".join(w.itertext()) if w is not None else "" + pos = e.find(OSHB + "pos") + # the definitions at the head of the entry, before its senses + defs = [ + "".join(d.itertext()).strip() + for d in e.findall(OSHB + "def") + if "".join(d.itertext()).strip() + ] + entries.append( + Entry( + id=e.get("id", ""), + headword=head, + cons=consonants(head), + pos=pos.text if pos is not None else None, + defs=defs, + root=e.get("type") == "root", + homonym=e.get("mod"), + aramaic=aramaic, + ) + ) + return entries + + +def headword_consonants(headword: str) -> str: + return consonants(plain(headword)) + + +def join(ours: list[str], theirs: list[str], reach: int = 3) -> dict[int, int]: + """Pairs of positions in two lists of headword consonants in the same + order: equal runs are joined, and so are the entries between two equal + runs when both have the same number of them (spellings that differ). + Then entries left over on both sides with the same consonants are joined + when they are at most `reach` places from where they would be expected + (two entries in the other order).""" + pairs: dict[int, int] = {} + matcher = SequenceMatcher(None, ours, theirs, autojunk=False) + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag == "equal" or (tag == "replace" and i2 - i1 == j2 - j1): + for k in range(i2 - i1): + pairs[i1 + k] = j1 + k + taken = set(pairs.values()) + left = [j for j in range(len(theirs)) if j not in taken] + offset = 0 + for i in range(len(ours)): + if i in pairs: + offset = pairs[i] - i + continue + for j in left: + if abs(j - (i + offset)) <= reach and theirs[j] == ours[i]: + pairs[i] = j + left.remove(j) + break + return pairs + + +def osis(ref: str) -> str | None: + """Sefaria's reference `II Samuel 12:4` as `2Sam.12.4` (also a chapter, + or a range, whose start is taken).""" + m = re.fullmatch(r"(.+?) (\d+)(?::(\d+))?(?:[-–].*)?", ref.strip()) + if not m: + return None + book = canonical_book(m.group(1)) + if book is None: + return None + return ".".join(x for x in (book, m.group(2), m.group(3)) if x) + + +@dataclass +class Cleaned: + html: str + refs: list[str] = field(default_factory=list) # OSIS references + entries: list[str] = field(default_factory=list) # `BDB, אבה` + unknown: list[str] = field(default_factory=list) # references not understood + + +class _Cleaner(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.out: list[str] = [] + self.open: list[str] = [] + self.result = Cleaned("") + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag not in TAGS: + raise CleanError(f"tag <{tag}>") + a = dict(attrs) + if tag == "a": + ref = a.get("data-ref") or "" + if ref.startswith(("BDB, ", "BDB Aramaic, ")): + # another entry, as `BDB, אבה`: resolved to its id in the build + self.result.entries.append(ref) + self.out.append(f'') + elif (o := osis(ref)) is not None: + self.result.refs.append(o) + self.out.append(f'') + else: + self.result.unknown.append(ref) + self.out.append("") + else: + kept = "".join( + f' {k}="{html.escape(v)}"' + for (k, v) in attrs + if k in TAGS[tag] and v is not None + ) + self.out.append(f"<{tag}{kept}>") + if tag not in VOID: + self.open.append(tag) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self.handle_starttag(tag, attrs) + if tag not in VOID and self.open and self.open[-1] == tag: + self.open.pop() + self.out.append(f"") + + def handle_endtag(self, tag: str) -> None: + if tag not in TAGS: + raise CleanError(f"tag ") + if tag in VOID: + return + if tag in self.open: + # close what is still open inside it + while self.open: + t = self.open.pop() + self.out.append(f"") + if t == tag: + break + + def handle_data(self, data: str) -> None: + self.out.append(html.escape(data, quote=False)) + + +def clean(raw: str) -> Cleaned: + """Sefaria's HTML of an entry with only the tags in `TAGS`.""" + cleaner = _Cleaner() + cleaner.feed(NOT_A_TAG.sub("<", raw)) + cleaner.close() + for t in reversed(cleaner.open): + cleaner.out.append(f"") + cleaner.result.html = "".join(cleaner.out).strip() + return cleaner.result diff --git a/libs/explorer/cfabric_browser/dictionaries/build.py b/libs/explorer/cfabric_browser/dictionaries/build.py new file mode 100644 index 0000000..33e2c24 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/build.py @@ -0,0 +1,672 @@ +"""Building the BDB dictionary file (`bdb.json.gz`), once, for publishing. + +The steps, each of which stops the build with a `BuildError` when a source is +not what is expected: + +1. `fetch_sources`: Open Scriptures' `HebrewLexicon` (BDB's ids and short + definitions, the lexical index, the augmented Strong's index) and the + OSHB's books, at pinned commits, the lexicon files by checksum. +2. Sefaria's two lexicons, walked entry by entry (`sefaria.walk`; from its + cache after the first time), and its list of abbreviations. +3. `join` Sefaria's entries to Open Scriptures' ids, then the joins of + `joins.yaml`; `clean` their HTML; references to other entries become + their ids. +4. Link the corpus' lexemes: align its words with the OSHB per verse, count + the BDB entries their lemmas lead to; where another entry cites the + lexeme's verses clearly better (`by_citations`), take that one; then + apply `corrections.yaml`. +5. `check`, and write the file with its metadata. +""" + +from __future__ import annotations + +import datetime +import gzip +import html +import json +import re +from collections import Counter +from importlib import resources +from pathlib import Path +from typing import Any + +import yaml + +from cfabric_browser import __version__ +from cfabric_browser.books import canonical_book +from cfabric_browser.dictionaries import bdb, link, sefaria +from cfabric_browser.jobs import Progress +from cfabric_browser.net import sha256_bytes + +CACHE = "~/.cfabric/dictionaries/cache" +BHSA = "~/text-fabric-data/github/ETCBC/bhsa/tf/2021" + +LEXICON_COMMIT = "21c9add13bc727d3a951361778e97e3ff7afd1ce" # 2019-09-02 +LEXICON_URL = ( + "https://raw.githubusercontent.com/openscriptures/HebrewLexicon/" + f"{LEXICON_COMMIT}/{{name}}" +) +LEXICON_FILES = { + "BrownDriverBriggs.xml": ( + "2b52658a4323d91674cda4090ab8b3ebddfff640f4f18143c28300e80b2c38f8" + ), + "LexicalIndex.xml": ( + "8f7a605c58899d2f44430149c143c00903976e1e91232476677972a69e5bc85f" + ), + "AugIndex.xml": "e7217ca8ff8ff3f21f9cf1bbe87411adf55f6aa88bcf5ed9ddc886cc6b160c5d", +} +OSHB_COMMIT = "3d15126fb1ef74867fc1434be1942e837932691f" # 2024-08-27, WLC 4.20 +OSHB_URL = ( + "https://raw.githubusercontent.com/openscriptures/morphhb/" + f"{OSHB_COMMIT}/wlc/{{name}}.xml" +) +# the books in OSIS ids, as the OSHB names its files +OSHB_BOOKS = ( + "Gen", + "Exod", + "Lev", + "Num", + "Deut", + "Josh", + "Judg", + "Ruth", + "1Sam", + "2Sam", + "1Kgs", + "2Kgs", + "1Chr", + "2Chr", + "Ezra", + "Neh", + "Esth", + "Job", + "Ps", + "Prov", + "Eccl", + "Song", + "Isa", + "Jer", + "Lam", + "Ezek", + "Dan", + "Hos", + "Joel", + "Amos", + "Obad", + "Jonah", + "Mic", + "Nah", + "Hab", + "Zeph", + "Hag", + "Zech", + "Mal", +) + +# a lexeme links to the entries that at least this share of its aligned +# occurrences lead to (the first one is the main entry) +SHARE = 0.1 +# lexemes this frequent must be linked to an entry with text +FREQUENT = 10 +# 22 of Sefaria's entries stop after the gender, as `[אֶ֫שֶׁר, or אָשָׁר] n.[m.` +CUT = re.compile(r"\[(?:m|f)\.\??(?: or f\.)?$") + + +def _cut(raw: str) -> bool: + """Whether Sefaria's text of an entry stops short.""" + return CUT.search(re.sub(r"<[^>]+>", "", raw).strip()) is not None + + +# another entry replaces the linked one when it cites at least this many of +# the lexeme's verses (all of them if it has fewer) and scores at least this +# many times as high +CITED = 2 +BETTER = 2.0 + + +class BuildError(Exception): + """A source is not what the builder expects; the message says where.""" + + +def fetch_sources( + cache: str | Path = CACHE, download: sefaria.Download = sefaria._download +) -> tuple[dict[str, Path], dict[str, str]]: + """The local paths of the Open Scriptures files, and the checksums of the + OSHB's books (for the metadata).""" + folder = Path(cache).expanduser() / "openscriptures" + paths: dict[str, Path] = {} + sums: dict[str, str] = {} + wanted = [ + (name, LEXICON_URL.format(name=name), LEXICON_COMMIT, sha) + for (name, sha) in LEXICON_FILES.items() + ] + [(f"{b}.xml", OSHB_URL.format(name=b), OSHB_COMMIT, None) for b in OSHB_BOOKS] + for name, url, commit, sha in wanted: + target = folder / commit[:7] / name + data = target.read_bytes() if target.is_file() else None + if data is None or (sha is not None and sha256_bytes(data) != sha): + try: + data = download(url) + except OSError as e: + raise BuildError(f"Could not download {url}: {e}") from e + if sha is not None and sha256_bytes(data) != sha: + raise BuildError(f"{url} has another checksum than expected") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + paths[name] = target + if sha is None: + sums[name] = sha256_bytes(data) + return paths, sums + + +def _package_yaml(name: str) -> dict[Any, Any]: + text = ( + resources.files("cfabric_browser.dictionaries") + .joinpath(name) + .read_text(encoding="utf-8") + ) + return yaml.safe_load(text) or {} + + +def corrections() -> dict[str, list[str]]: + """Links made by hand: ` ` → entry ids (the first is the + main entry), from `corrections.yaml`.""" + data = _package_yaml("corrections.yaml") + return {str(k): [str(i) for i in v["entries"]] for (k, v) in data.items()} + + +def joins() -> dict[str, str]: + """Joins made by hand: Sefaria's reference (`BDB, אֶלְקֹשִׁי`) → Open + Scriptures' id, from `joins.yaml`.""" + return {str(k): str(v) for (k, v) in _package_yaml("joins.yaml").items()} + + +def abbreviations(client: sefaria.Client) -> dict[str, str]: + """BDB's abbreviations (`AE`: `Aben Ezra`), from Sefaria.""" + text = client.get(sefaria.text_url("BDB", "Abbrevations")) + lines = text.get("text", []) if isinstance(text, dict) else [] + found = {} + for line in lines: + plainLine = re.sub(r"<[^>]+>", "", line).strip() + m = re.fullmatch(r"(\S.{0,30}?) = (.+)", plainLine) + if m: + found[m.group(1).strip()] = m.group(2).strip() + if len(found) < 400: + raise BuildError(f"Only {len(found)} abbreviations from Sefaria") + return found + + +def entries( + client: sefaria.Client, + ours: list[bdb.Entry], + index: dict[str, link.IndexEntry], + progress: Progress | None = None, + byHand: dict[str, str] | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """BDB's entries with Sefaria's text joined to Open Scriptures' ids, in + BDB's order (Hebrew, then Aramaic), and figures about the join. `byHand` + (Sefaria's reference → id) replaces what the join did with either side: + for entries it leaves out, and for neighbours with the same consonants + (a homonym, a root, a cross reference) that it takes for each other.""" + byHand = joins() if byHand is None else byHand + if len(set(byHand.values())) < len(byHand): + raise BuildError("joins.yaml: an id is joined twice") + numbers = _numbers(index) + + result: list[dict[str, Any]] = [] + figures: dict[str, Any] = {} + for lexicon, book in sefaria.LEXICONS.items(): + aramaic = book != "BDB" + + def report(n: int, headword: str, step: str = f"sefaria {book}") -> None: + if progress is not None: + progress(step, n, 0) + + theirs = sefaria.walk(client, lexicon, report) + mine = [e for e in ours if e.aramaic == aramaic] + pairs = bdb.join( + [e.cons for e in mine], + [bdb.headword_consonants(e["headword"]) for e in theirs], + ) + byTheirs = {j: mine[i] for (i, j) in pairs.items()} + joined = len(pairs) + refs = {f"{book}, {e['headword']}": j for (j, e) in enumerate(theirs)} + positions = {e.id: i for (i, e) in enumerate(mine)} + for ref, oid in byHand.items(): + if not ref.startswith(f"{book}, "): + continue + j, i = refs.get(ref), positions.get(oid) + if j is None or i is None: + raise BuildError(f"joins.yaml: no entry {ref if j is None else oid}") + if j in byTheirs: + del pairs[positions[byTheirs.pop(j).id]] + if i in pairs: + del byTheirs[pairs.pop(i)] + byTheirs[j] = mine[i] + pairs[i] = j + incomplete = [f"{book}, {e['headword']}" for e in theirs if _cut(e["html"])] + figures[book] = { + "sefaria": len(theirs), + "openscriptures": len(mine), + "joined": joined, + "joined by hand": sum(1 for r in byHand if r.startswith(f"{book}, ")), + "incomplete at Sefaria": incomplete, + } + for j, e in enumerate(theirs): + try: + cleaned = bdb.clean(e["html"]) + except bdb.CleanError as x: + raise BuildError(f"{book}, {e['headword']}: {x}") from x + os = byTheirs.get(j) + strong, twot = numbers.get(os.id, (set(), set())) if os else (set(), set()) + result.append( + { + "id": os.id if os else f"{'x' if aramaic else ''}s.{j}", + "lexicon": "arc" if aramaic else "heb", + "headword": e["headword"], + "ref": f"{book}, {e['headword']}", + "html": cleaned.html, + "incomplete": bool(_cut(e["html"])), + "refs": len(cleaned.refs), + "unknownRefs": cleaned.unknown, + "pos": os.pos if os else None, + "defs": os.defs if os else [], + "root": os.root if os else False, + "section": os.section if os else None, + "strong": sorted(strong, key=_number), + "twot": sorted(twot), + } + ) + # references to other entries: by Sefaria's reference to its id + ids = {e["ref"]: e["id"] for e in result} + unresolved = 0 + for entry in result: + + def resolve(m: re.Match[str]) -> str: + nonlocal unresolved + target = ids.get(html.unescape(m.group(1))) + if target is None: + unresolved += 1 + return "" + return f'' + + entry["html"] = re.sub(r'', resolve, entry["html"]) + figures["unresolved cross references"] = unresolved + return result, figures + + +def _numbers( + index: dict[str, link.IndexEntry], +) -> dict[str, tuple[set[str], set[str]]]: + """Per entry id its Strong's and TWOT numbers.""" + numbers: dict[str, tuple[set[str], set[str]]] = {} + for ix in index.values(): + if ix.bdb: + strongs, twots = numbers.setdefault(ix.bdb, (set(), set())) + if ix.strong: + strongs.add(ix.strong) + if ix.twot: + twots.add(ix.twot) + return numbers + + +def missing( + ours: list[bdb.Entry], + entries: list[dict[str, Any]], + linked: dict[str, dict[str, Any]], + index: dict[str, link.IndexEntry], +) -> list[dict[str, Any]]: + """Entries that lexemes link to and Sefaria lacks (its text has gaps): + Open Scriptures' headword, part of speech and short definitions, without + text.""" + have = {e["id"] for e in entries} + wanted = {i for v in linked.values() for i in v["entries"]} - have + numbers = _numbers(index) + result = [] + for os in ours: + if os.id in wanted: + strong, twot = numbers.get(os.id, (set(), set())) + result.append( + { + "id": os.id, + "lexicon": "arc" if os.aramaic else "heb", + "headword": os.headword, + "ref": None, + "html": "", + "incomplete": True, + "refs": 0, + "unknownRefs": [], + "pos": os.pos, + "defs": os.defs, + "root": os.root, + "section": os.section, + "strong": sorted(strong, key=_number), + "twot": sorted(twot), + } + ) + unknown = wanted - {e["id"] for e in result} + if unknown: + raise BuildError(f"links to entries that don't exist: {sorted(unknown)}") + return result + + +def resolved( + fixed: dict[str, list[str]], entries: list[dict[str, Any]] +) -> dict[str, list[str]]: + """Corrections with Sefaria's references (`BDB, כְּרוּב`, for entries + without an Open Scriptures id) replaced by the entries' ids.""" + ids = {e["ref"]: e["id"] for e in entries} + result = {} + for key, targets in fixed.items(): + result[key] = [] + for t in targets: + if ", " in t and t not in ids: + raise BuildError(f"corrections.yaml: no entry {t}") + result[key].append(ids.get(t, t)) + return result + + +def _number(strong: str) -> tuple[int, str]: + digits = re.match(r"\d+", strong) + return (int(digits.group()) if digits else 0, strong) + + +def is_cross_reference(html_: str) -> bool: + """Whether an entry only refers to another (`יסך v. סוּךְ`): short, with + `v.`, BDB's "see".""" + text = re.sub(r"<[^>]+>", "", html_) + return len(text) < 150 and re.search(r"\bv\.\s", text) is not None + + +def is_root(html_: str) -> bool: + """Whether an entry is a root that only introduces the words after it + (`קעקע (√ of foll.; …)`).""" + return "√ of foll" in re.sub(r"<[^>]+>", "", html_)[:80] + + +def is_verb(entry: dict[str, Any]) -> bool: + """Whether an entry is a verb: by Open Scriptures' part of speech, else + by `vb.` at the start of Sefaria's text.""" + if entry.get("pos"): + return str(entry["pos"]).startswith("vb") + return re.search(r"\bvb\.", re.sub(r"<[^>]+>", "", entry["html"])[:80]) is not None + + +def cited_verses(entries: list[dict[str, Any]]) -> dict[str, set[str]]: + """Per entry the verses its text cites (OSIS, as `Gen.3.6`).""" + return { + e["id"]: { + r + for r in re.findall(r'data-osis="([^"]+)"', e["html"]) + if r.count(".") == 2 + } + for e in entries + } + + +def citation_score(cited: set[str], verses: set[str]) -> tuple[float, int]: + """How well an entry's cited verses fit a lexeme's verses: the share of + the cited verses with the lexeme times the share of the lexeme's verses + cited (a root entry cites hundreds of verses, so it shouldn't win by + chance), and the number of verses both have.""" + hits = len(cited & verses) + if not hits: + return (0.0, 0) + return (hits / len(cited) * hits / len(verses), hits) + + +def by_citations( + current: list[str], + candidates: list[str], + verses: set[str], + cited: dict[str, set[str]], +) -> list[str] | None: + """The entries of a lexeme when a candidate other than the linked entry + cites its verses clearly better (at least `CITED` of them, or all if the + lexeme has fewer, and `BETTER` times the score), else None. The chosen entry comes first; of the others + only those that cite one of its verses stay.""" + main = current[0] + mine = citation_score(cited.get(main, set()), verses)[0] + best, bestScore = None, (0.0, 0) + for c in candidates: + if c == main: + continue + score = citation_score(cited.get(c, set()), verses) + if score > bestScore: + best, bestScore = c, score + enough = min(CITED, len(verses)) + if best is None or bestScore[1] < enough or bestScore[0] < BETTER * mine: + return None + rest = [ + c + for c in current + if c != best and citation_score(cited.get(c, set()), verses)[1] + ] + return [best, *rest] + + +def links( + api: Any, + verses: dict[tuple[str, int, int], list[link.Segment]], + aug: dict[str, str], + index: dict[str, link.IndexEntry], + fixed: dict[str, list[str]], + entries: list[dict[str, Any]] | None = None, +) -> dict[str, dict[str, Any]]: + """Per lexeme (` `) the BDB entries it links to, with how + the link was made: `oshb`, `cited` (another entry cites its verses + better, `by_citations`; among the entries its lemmas lead to and those + with its consonants, in its language, verbs for verbs and other words + for other words) or `manual`.""" + F, L, T = api.F, api.L, api.T + counts = link.lemma_counts(api, verses) + cited = cited_verses(entries or []) + language = {e["id"]: e["lexicon"] for e in entries or []} + verb = {e["id"]: is_verb(e) for e in entries or []} + # an entry that only refers to another or introduces a root may stay, + # but doesn't replace another + minor = { + e["id"] + for e in entries or [] + if is_cross_reference(e["html"]) or is_root(e["html"]) + } + byConsonants: dict[tuple[str, str], list[str]] = {} + for e in entries or []: + cons = bdb.headword_consonants(e["headword"]) + byConsonants.setdefault((e["lexicon"], cons), []).append(e["id"]) + result: dict[str, dict[str, Any]] = {} + for node in F.otype.s("lex"): + node = int(node) + key = f"{F.language.v(node)} {F.lex.v(node)}" + found = link.entry_counts(counts.get(node, Counter()), aug, index) + total = sum(n for (_, n) in found) + if key in fixed: + result[key] = {"entries": fixed[key], "how": "manual"} + continue + if not found: + continue + result[key] = { + "entries": [e for (e, n) in found if n >= SHARE * total], + "share": round(found[0][1] / total, 3), + "how": "oshb", + } + if not cited: + continue + lang = "arc" if F.language.v(node) == "Aramaic" else "heb" + cons = link.etcbc_consonants(F.lex.v(node) or "") + candidates = [ + c + for c in [e for (e, _) in found] + byConsonants.get((lang, cons), []) + if language.get(c) == lang + and verb[c] == (F.sp.v(node) == "verb") + and c not in minor + ] + lexVerses = set() + for w in L.d(node, "word"): + book, chapter, verse = T.sectionFromNode(w, lang="en") + lexVerses.add(f"{canonical_book(book)}.{chapter}.{verse}") + better = by_citations(result[key]["entries"], candidates, lexVerses, cited) + if better is not None: + result[key]["was"] = result[key]["entries"][0] + result[key]["entries"] = better + result[key]["how"] = "cited" + return result + + +def check( + api: Any, + entries: list[dict[str, Any]], + linked: dict[str, dict[str, Any]], +) -> dict[str, Any]: + """Stop the build if a frequent lexeme has no entry with text; figures + for the metadata.""" + F = api.F + withText = {e["id"] for e in entries if e["html"]} + missing = [] + unlinked = [] + for node in F.otype.s("lex"): + node = int(node) + key = f"{F.language.v(node)} {F.lex.v(node)}" + freq = int(F.freq_lex.v(node) or 0) + found = linked.get(key) + if found is None: + unlinked.append(key) + if freq >= FREQUENT: + missing.append(f"{key} (not linked, {freq}×)") + elif found["entries"][0] not in withText and freq >= FREQUENT: + missing.append(f"{key} ({found['entries'][0]} has no text, {freq}×)") + if missing: + raise BuildError( + f"{len(missing)} frequent lexemes without an entry: " + + ", ".join(missing[:20]) + ) + textless = sorted( + {i for v in linked.values() for i in v["entries"] if i not in withText} + ) + return { + "lexemes": len(F.otype.s("lex")), + "linked": len(linked), + "linked by citations": sum(1 for v in linked.values() if v["how"] == "cited"), + "unlinked": unlinked, + "entries without text that lexemes link to": textless, + "references": sum(e["refs"] for e in entries), + "references not understood": sum(len(e["unknownRefs"]) for e in entries), + } + + +def build( + out: str | Path, + corpus: str | Path = BHSA, + cache: str | Path = CACHE, + client: sefaria.Client | None = None, + progress: Progress | None = None, +) -> dict[str, Any]: + """Build the file; gives its metadata.""" + from cfabric import Fabric + + paths, sums = fetch_sources(cache) + if client is None: + client = sefaria.Client(Path(cache).expanduser() / "sefaria") + api = Fabric(locations=str(Path(corpus).expanduser()), silent="deep").load( + "g_cons lex language freq_lex sp book@en", silent="deep" + ) + if not api: + raise BuildError(f"Could not load the corpus in {corpus}") + index = link.read_lexical_index(paths["LexicalIndex.xml"]) + aug = link.read_aug_index(paths["AugIndex.xml"]) + ours = bdb.read_entries(paths["BrownDriverBriggs.xml"]) + verses: dict[tuple[str, int, int], list[link.Segment]] = {} + for b in OSHB_BOOKS: + verses.update(link.read_oshb(paths[f"{b}.xml"])) + + allEntries, figures = entries(client, ours, index, progress) + linked = links( + api, verses, aug, index, resolved(corrections(), allEntries), allEntries + ) + allEntries += missing(ours, allEntries, linked, index) + figures.update(check(api, allEntries, linked)) + for e in allEntries: + del e["unknownRefs"] + + meta = { + "title": "A Hebrew and English Lexicon of the Old Testament", + "short": "BDB", + "authors": "Francis Brown, S. R. Driver, Charles A. Briggs", + "edition": "Oxford, 1906", + "licence": "CC BY-NC 4.0", + "licenceNote": ( + "The file is licensed under CC BY-NC 4.0. The text of BDB (1906) is in " + "the public domain, and Open Scriptures' data in it (ids, short " + "definitions, Strong's and TWOT numbers) remains available under " + "CC BY 4.0. The links to the BHSA's lexemes derive from the BHSA " + "(CC BY-NC 4.0)." + ), + "sources": [ + { + "what": "the text of the entries", + "name": "Sefaria, BDB Dictionary and BDB Aramaic Dictionary", + "url": "https://www.sefaria.org/BDB", + "licence": "Public Domain (Sefaria's version metadata), " + "from a scan in the National Library of Israel", + }, + { + "what": "ids, parts of speech and short definitions of the " + "entries; Strong's and TWOT numbers", + "name": "Open Scriptures, HebrewLexicon", + "url": "https://github.com/openscriptures/HebrewLexicon", + "commit": LEXICON_COMMIT, + "licence": "CC BY 4.0, Open Scriptures Hebrew Bible Project", + "checksums": LEXICON_FILES, + }, + { + "what": "the lemmas that link the corpus' words to the entries", + "name": "Open Scriptures Hebrew Bible (morphhb)", + "url": "https://github.com/openscriptures/morphhb", + "commit": OSHB_COMMIT, + "licence": "CC BY 4.0", + "checksums": sums, + }, + { + "what": "the lexemes linked", + "name": "BHSA (ETCBC)", + "path": str(corpus), + "licence": "CC BY-NC 4.0", + }, + ], + "built": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), + "builder": f"cfabric-browser {__version__}", + "figures": figures, + } + data = { + "meta": meta, + "entries": allEntries, + "links": linked, + "abbreviations": abbreviations(client), + } + target = Path(out).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(data, ensure_ascii=False, separators=(",", ":")).encode() + target.write_bytes(gzip.compress(raw, mtime=0)) + return meta + + +def main() -> None: + """Build `bdb.json.gz` (run by hand, for publishing).""" + import argparse + import sys + + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument("out") + parser.add_argument("--corpus", default=BHSA) + parser.add_argument("--cache", default=CACHE) + args = parser.parse_args() + + def report(step: str, n: int, total: int) -> None: + if n % 500 == 0: + print(f"{step}: {n}", file=sys.stderr, flush=True) + + meta = build(args.out, args.corpus, args.cache, progress=report) + print(json.dumps(meta["figures"], ensure_ascii=False, indent=1)[:4000]) + + +if __name__ == "__main__": + main() diff --git a/libs/explorer/cfabric_browser/dictionaries/corrections.yaml b/libs/explorer/cfabric_browser/dictionaries/corrections.yaml new file mode 100644 index 0000000..1bbd371 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/corrections.yaml @@ -0,0 +1,43 @@ +# Links from lexemes to BDB entries made by hand, where the alignment with the +# OSHB gives the wrong entry. Key: language and lex as in the BHSA; entries: +# Open Scriptures ids of BDB entries, the main one first. + +# The OSHB's prefix lemmas (b, c, k, l, i) are Hebrew only, so in the +# Aramaic parts they led to the Hebrew entries. +Aramaic W: + entries: [xf.aa.aa] # וְ +Aramaic L: + entries: [xl.aa.aa] # ל +Aramaic B: + entries: [xb.aa.aa] # בְּ +Aramaic K: + entries: [xk.aa.aa] # כְּ +Aramaic H: + entries: [xe.aa.aa] # הֲ interrogative + +# Lexemes whose entry in Open Scriptures has no text in Sefaria, linked to the +# entry that has it. +Hebrew JLWN/: + entries: [j.bn.am] # יָלוֹן Jalon, 1 Ch 4:17: at the end of Sefaria's תּוֹלָד +Hebrew >TNN=/: + entries: [a.gp.af] # אֶתְנָן Ethnan, 1 Ch 4:7: BDB has it under א +Hebrew TLTJ>L/: + entries: [v.ag.ah] # שְׁאַלְתִּיאֵל Ezra 5:2: only in BDB's Hebrew part +Aramaic CLVN=/: + entries: [xv.as.ab, xv.as.ac] # שִׁלְטוֹן ruler, Dn 3:2-3: in BDB's addenda + # (p. 1127), which Sefaria lacks; its שָׁלְטָן (dominion) refers to it +Hebrew LZW: + entries: [e.bl.aa] # הַלֵּ֫זוּ: in Sefaria's entry הַלָּ֫זֶה, הַלֵּ֫זוּ + +# Entries without an Open Scriptures id, by Sefaria's reference. +Hebrew KRWB=/: + entries: ["BDB, כְּרוּב"] # I. Kerub, the place in Babylonia (Ezr 2:59) +Hebrew XSN[: + entries: ["BDB, חָסַן"] # be treasured up (Is 23:18) + +# Where BDB's citations mislead `by_citations`. +Hebrew >T=: + entries: [a.er.ad, a.er.ac] # אַתְּ (cites a few of its 57 verses, "+ 50 + # times"), not אתי, the ketiv's spelling, which cites all of its 7 diff --git a/libs/explorer/cfabric_browser/dictionaries/joins.yaml b/libs/explorer/cfabric_browser/dictionaries/joins.yaml new file mode 100644 index 0000000..7fd7739 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/joins.yaml @@ -0,0 +1,24 @@ +# Sefaria's entries joined by hand to Open Scriptures' ids. The join by order +# and headword leaves some out on both sides (other spellings, or a Sefaria +# entry that covers two headwords), and takes some neighbours with the same +# consonants for each other (a homonym, a root, a cross reference). Key: +# Sefaria's reference; value: the id. A join here replaces what the join did +# with either side. + +# left out +BDB, אֶלְקֹשִׁי: a.du.ac # Elkoshite; Open Scriptures spells אֶלְקוֹשִׁי +BDB, מַהֲתַלּוֹת: w.au.ab # deceptions; BDB prints it under מ and under תלל +BDB, סִרָה: o.au.af # Sefaria's entry also has יסור (Je 17:13 Kt) +BDB, קוֹנֵן²: s.bk.ai # chant a dirge; Open Scriptures spells קִינֵן +BDB Aramaic, גּוּחַ: xc.ad.ac # [גּוּחַ, גִּיחַ] burst forth +BDB Aramaic, שְׁאֵלָה: xv.aa.ab # affair; Open Scriptures spells שְׁאֵלָא + +# taken for a neighbour +BDB, כְּרוּב²: k.da.ac # II. cherub, not I. the place in Babylonia +BDB, תֹּ֫פֶת: w.ak.ab # I. spitting, not II. (v. תפת) +BDB, קִיץ: s.bl.aa # I. awake, not II. the root (summer) +BDB, פַּרְוָר: q.cv.ah # structure, not פָּרוּר pot (v. פור) +BDB, חָסֹן: h.ei.ac # adj. strong, not the verb חָסַן +BDB, ערב⁴: p.fg.aa # VI. the root (be black) +BDB, עֹרֵב: p.fg.ab # raven, not the root +BDB, עָצַם²: p.er.aa # shut the eyes (III. in Sefaria, II. in Open Scriptures) diff --git a/libs/explorer/cfabric_browser/dictionaries/link.py b/libs/explorer/cfabric_browser/dictionaries/link.py new file mode 100644 index 0000000..daae456 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/link.py @@ -0,0 +1,254 @@ +"""Linking the lexemes of the BHSA to entries of BDB. + +The BHSA's lexemes (`BR>[`, `DBR/`) carry no dictionary numbers. The Open +Scriptures Hebrew Bible (OSHB, `morphhb`, CC BY 4.0) gives every word a lemma: +the (augmented) Strong's numbers of its morphemes, as `b/7225` for בְּ/רֵאשִׁית. +Both are the Leningrad Codex, so the words of a verse are aligned by their +consonants, and each BHSA word gets the lemma of the OSHB morpheme it overlaps +most. A lexeme then gets the lemmas of its words, counted. Open Scriptures' +`AugIndex.xml` maps a lemma to an entry of its `LexicalIndex.xml`, which names +the BDB entry (`a.ab.ab`), the Strong's number and the root. + +Differences in segmentation: +- the BHSA writes prefixes (`B`, `W`, `H`, `L`) as words of their own, as the + OSHB splits them off with `/`; +- an article that is only a vowel (לַמֶּלֶךְ) is a word without consonants in + the BHSA and a lemma `d` without a part of the text in the OSHB; neither + takes part in the alignment; +- a pronominal suffix is part of the word in the BHSA and a part of its own + without a lemma in the OSHB: its letters count for no lemma. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections import Counter +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from pathlib import Path +from typing import Any + +from cfabric_browser.books import canonical_book + +OSIS = "{http://www.bibletechnologies.net/2003/OSIS/namespace}" +OSHB = "{http://openscriptures.github.com/morphhb/namespace}" + +# Hebrew letters as in the ETCBC's transliteration; shin and sin are one +# letter here (C), because unpointed text cannot tell them apart +HEBREW = dict( + zip( + "אבגדהוזחטיכךלמםנןסעפףצץקרשת", + ">BGDHWZXVJKKLMMNNSBGDHWZXVJKLMNS str: + """The consonants of Hebrew text, in the ETCBC's letters.""" + return "".join(HEBREW[c] for c in text if c in HEBREW) + + +def etcbc_consonants(cons: str) -> str: + """ETCBC consonants (`g_cons`) as `consonants` gives them: sin and shin + (and an unpointed shin) as C.""" + return "".join("C" if c in "FC#" else c for c in cons if c in ETCBC) + + +def lemma_id(part: str) -> str: + """An augmented Strong's number as `AugIndex.xml` writes it: `834 a` is + `834a`; `+` (part of a name of several words) is left out.""" + return part.replace(" ", "").replace("+", "") + + +@dataclass(frozen=True) +class Segment: + """A part of an OSHB word: its consonants, and the lemma it belongs to + (None for a suffix).""" + + cons: str + lemma: str | None + language: str # H or A, from the morphology + + +def segments(text: str, lemma: str, morph: str) -> list[Segment]: + """The parts of an OSHB word with their lemmas. + + The text and the morphology have the same parts (`בְּ/רֵאשִׁית`, `HR/Ncfsa`); + the lemma has the prefixes and the main word (`b/7225`), not the suffixes, + and may have a prefix that has no part in the text (the article in + לַמֶּלֶךְ: `l/d/4428` against `לַ/מֶּלֶךְ` and `HRd/Ncmsa`).""" + parts = text.split("/") + language = morph[:1] or "H" + kinds = morph[1:].split("/") + lemmas = lemma.split("/") + main = lemma_id(lemmas[-1]) + prefixes = lemmas[:-1] + if len(kinds) != len(parts): + return [Segment(consonants(text), main, language)] + suffixes = 0 + while suffixes < len(kinds) - 1 and kinds[len(kinds) - 1 - suffixes][:1] == "S": + suffixes += 1 + m = len(parts) - suffixes - 1 # the part of the main word + result = [] + for j in range(m): + cons = consonants(parts[j]) + own = None + # the next prefix written with this letter; prefixes without a letter + # of their own (an article in a vowel) are passed over + for k, p in enumerate(prefixes): + if PREFIXES.get(p) == cons: + own = p + prefixes = prefixes[k + 1 :] + break + if own is None and prefixes: + own = prefixes.pop(0) + result.append(Segment(cons, own, language)) + result.append(Segment(consonants(parts[m]), main, language)) + for j in range(m + 1, len(parts)): + result.append(Segment(consonants(parts[j]), None, language)) + return result + + +def _words(element: ET.Element) -> list[ET.Element]: + """The words of a verse in the text, not those in notes (the qere).""" + found = [] + for child in element: + if child.tag == OSIS + "note": + continue + if child.tag == OSIS + "w": + found.append(child) + else: + found.extend(_words(child)) + return found + + +def read_oshb(path: Path) -> dict[tuple[str, int, int], list[Segment]]: + """The segments per verse of an OSHB book file (`Gen.xml`), by + (OSIS book, chapter, verse).""" + verses: dict[tuple[str, int, int], list[Segment]] = {} + root = ET.parse(path).getroot() + for verse in root.iter(OSIS + "verse"): + book, chapter, number = verse.get("osisID", "").split(".") + found: list[Segment] = [] + for w in _words(verse): + found.extend(segments(w.text or "", w.get("lemma", ""), w.get("morph", ""))) + verses[(book, int(chapter), int(number))] = found + return verses + + +def align(words: list[tuple[int, str]], found: list[Segment]) -> dict[int, Segment]: + """The OSHB segment (with a lemma) each BHSA word of a verse overlaps + most, by consonants; `words` are (node, ETCBC consonants).""" + a: list[int] = [] + for i, (_, cons) in enumerate(words): + a.extend([i] * len(etcbc_consonants(cons))) + b: list[int] = [] + for j, s in enumerate(found): + b.extend([j] * len(s.cons)) + textA = "".join(etcbc_consonants(cons) for (_, cons) in words) + textB = "".join(s.cons for s in found) + overlap: dict[int, Counter[int]] = {} + matcher = SequenceMatcher(None, textA, textB, autojunk=False) + for block in matcher.get_matching_blocks(): + for k in range(block.size): + j = b[block.b + k] + if found[j].lemma is not None: + overlap.setdefault(a[block.a + k], Counter())[j] += 1 + return { + words[i][0]: found[counts.most_common(1)[0][0]] + for (i, counts) in overlap.items() + } + + +@dataclass +class IndexEntry: + """An entry of Open Scriptures' lexical index.""" + + id: str # aaf + word: str # אָבַד + language: str # heb, arc + pos: str | None = None + gloss: str | None = None + bdb: str | None = None # a.ac.aa + strong: str | None = None + twot: str | None = None + root: str | None = None # the entry of the root (for derived words) + derived: list[str] = field(default_factory=list) + + +def read_lexical_index(path: Path) -> dict[str, IndexEntry]: + entries = {} + root = ET.parse(path).getroot() + for part in root.iter(OSHB + "part"): + language = part.get("{http://www.w3.org/XML/1998/namespace}lang", "heb") + for e in part.iter(OSHB + "entry"): + w = e.find(OSHB + "w") + xref = e.find(OSHB + "xref") + pos = e.find(OSHB + "pos") + gloss = e.find(OSHB + "def") + etym = e.find(OSHB + "etym") + entry = IndexEntry( + id=e.get("id", ""), + word=(w.text or "") if w is not None else "", + language=language, + pos=pos.text if pos is not None else None, + gloss=gloss.text if gloss is not None else None, + ) + if xref is not None: + entry.bdb = xref.get("bdb") + entry.strong = xref.get("strong") + entry.twot = xref.get("twot") + if etym is not None and etym.text: + ids = [x.strip() for x in etym.text.split(",") if x.strip()] + if etym.get("type") == "sub": + entry.root = ids[0] if ids else None + else: + entry.derived = ids + entries[entry.id] = entry + return entries + + +def read_aug_index(path: Path) -> dict[str, str]: + """Augmented Strong's number (`834a`): lexical index id.""" + root = ET.parse(path).getroot() + return {w.get("aug", ""): (w.text or "").strip() for w in root.iter(OSHB + "w")} + + +def lemma_counts( + api: Any, verses: dict[tuple[str, int, int], list[Segment]] +) -> dict[int, Counter[str]]: + """Per lexeme node of a corpus with the BHSA's features (`g_cons`, lexeme + nodes `lex`), how often its words align with each OSHB lemma.""" + F, L, T = api.F, api.L, api.T + counts: dict[int, Counter[str]] = {} + for v in F.otype.s("verse"): + book, chapter, verse = T.sectionFromNode(v, lang="en") + found = verses.get((canonical_book(book) or "", int(chapter), int(verse))) + if found is None: + continue + words = [(int(w), F.g_cons.v(w) or "") for w in L.d(v, "word")] + aligned = align(words, found) + for w, _ in words: + s = aligned.get(w) + if s is not None and s.lemma is not None: + lexeme = int(L.u(w, "lex")[0]) + counts.setdefault(lexeme, Counter())[s.lemma] += 1 + return counts + + +def entry_counts( + lemmas: Counter[str], aug: dict[str, str], index: dict[str, IndexEntry] +) -> list[tuple[str, int]]: + """The BDB entries a lexeme's lemmas lead to, most frequent first (Strong's + may have two numbers for one entry, as for הלך).""" + counts: Counter[str] = Counter() + for lemma, n in lemmas.items(): + e = index.get(aug.get(lemma, "")) + if e is not None and e.bdb: + counts[e.bdb] += n + return counts.most_common() diff --git a/libs/explorer/cfabric_browser/dictionaries/manage.py b/libs/explorer/cfabric_browser/dictionaries/manage.py new file mode 100644 index 0000000..0fe5137 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/manage.py @@ -0,0 +1,134 @@ +"""Adding and removing dictionaries, from the library or the command line. + +A dictionary is one file that we build once (`build.py`) and publish; adding +it downloads that file, checked by its checksum, into +`~/.cfabric/dictionaries//`. It can also be added from a file on this +computer (`cfabric-browser add bdb FILE`), which is checked by its contents. +The browser reads the file into memory, so it can be removed while it runs. +""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from cfabric_browser.dictionaries.store import Dictionary, DictionaryError +from cfabric_browser.jobs import BackgroundJobs, Progress +from cfabric_browser.net import DownloadError, Fetch, download_file, fetch_to + +ROOT = "~/.cfabric/dictionaries" + + +@dataclass(frozen=True) +class Catalogued: + id: str + short: str # the name in its meta (`BDB`) + title: str + name: str + licence: str + licenceNote: str + sources: str + corpus: str # whose lexemes it links (`ETCBC/bhsa`) + url: str | None # None until it is published + sha256: str | None # of the published file + size: int + + +DICTIONARIES: dict[str, Catalogued] = { + "bdb": Catalogued( + id="bdb", + short="BDB", + title="Brown–Driver–Briggs", + name="BDB (Brown–Driver–Briggs, 1906)", + licence="CC BY-NC 4.0", + licenceNote=( + "A Hebrew and English Lexicon of the Old Testament (Oxford, 1906). " + "The file is CC BY-NC 4.0: BDB's text, from Sefaria, is public domain; " + "Open Scriptures' ids and short definitions remain CC BY 4.0; the " + "links derive from the BHSA (CC BY-NC 4.0)." + ), + sources="Sefaria; Open Scriptures HebrewLexicon and OSHB (CC BY 4.0)", + corpus="ETCBC/bhsa", + url="https://github.com/Context-Fabric/bdb/releases/download/v1.0/bdb.json.gz", + sha256="d320e12c35f08891bf31edb6ba53a2eb78f5a0b5c8f3f7472b2f504689a6b7e1", + size=4_554_209, + ), +} + + +def target(catalogued: Catalogued, root: str | Path = ROOT) -> Path: + return Path(root).expanduser() / catalogued.id / f"{catalogued.id}.json.gz" + + +def is_added(catalogued: Catalogued, root: str | Path = ROOT) -> bool: + return target(catalogued, root).is_file() + + +def check(catalogued: Catalogued, path: Path) -> None: + """Stop with a `DictionaryError` unless `path` is this dictionary.""" + found = Dictionary.load(path) + if found.meta.get("short") != catalogued.short: + raise DictionaryError(f"{path} is not {catalogued.name}") + + +def add( + catalogued: Catalogued, + root: str | Path = ROOT, + progress: Progress | None = None, + fetcher: Fetch = fetch_to, +) -> Path: + """Download the published file into its place.""" + if catalogued.url is None or catalogued.sha256 is None: + raise DownloadError(f"{catalogued.name} has not been published yet") + place = target(catalogued, root) + place.parent.mkdir(parents=True, exist_ok=True) + report = ( + (lambda done, total: progress("download", done, total or catalogued.size)) + if progress + else (lambda done, total: None) + ) + partial = place.with_name(place.name + ".new") + download_file(catalogued.url, partial, catalogued.sha256, fetcher, report) + check(catalogued, partial) + partial.replace(place) + return place + + +def add_file(catalogued: Catalogued, path: str | Path, root: str | Path = ROOT) -> Path: + """Put a file built here, or downloaded by hand, in its place.""" + source = Path(path).expanduser() + check(catalogued, source) + place = target(catalogued, root) + if source.resolve() == place.resolve(): + return place + place.parent.mkdir(parents=True, exist_ok=True) + partial = place.with_name(place.name + ".new") + shutil.copyfile(source, partial) + partial.replace(place) + return place + + +def remove(catalogued: Catalogued, root: str | Path = ROOT) -> bool: + """Delete the file; False if it wasn't there.""" + place = target(catalogued, root) + if not place.is_file(): + return False + place.unlink() + return True + + +class DictionaryJobs(BackgroundJobs): + """Downloads of dictionaries in the background, for the library.""" + + def __init__(self, root: str | Path = ROOT, fetcher: Fetch = fetch_to) -> None: + super().__init__(expected=(DownloadError, DictionaryError)) + self.root = root + self.fetcher = fetcher + + def start(self, catalogued: Catalogued) -> dict[str, Any]: + return self.run( + catalogued.id, + lambda progress: add(catalogued, self.root, progress, self.fetcher), + ) diff --git a/libs/explorer/cfabric_browser/dictionaries/sefaria.py b/libs/explorer/cfabric_browser/dictionaries/sefaria.py new file mode 100644 index 0000000..cdb0bd7 --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/sefaria.py @@ -0,0 +1,240 @@ +"""Getting BDB from Sefaria, entry by entry. + +Sefaria's *BDB Dictionary* and *BDB Aramaic Dictionary* (public domain, from a +scan in the National Library of Israel) are not in its export of texts; the +API gives one headword at a time. `walk` goes through a lexicon from its first +headword along `next_hw`, as BDB orders its entries. + +`/api/texts/BDB, ` gives the article as one piece of HTML, the +stems divided by `
`, and the next headword (`BDB, מֶ֫לֶךְ²`: homonyms are +numbered with superscripts). One request per entry, about 10,000 in all. +(`/api/words/` has the senses as a tree, with Strong's numbers, but +finds only about 60% of the headwords and none with a number, so it would +take more requests and still need this one.) + +Requests are made one at a time with a pause between them, and are retried +after a while when Sefaria is busy. Every answer is kept in a cache with its +checksum, so an interrupted walk goes on where it stopped and a second walk +makes no requests at all. +""" + +from __future__ import annotations + +import hashlib +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from cfabric_browser import __version__ +from cfabric_browser.net import read_url + +SITE = "https://www.sefaria.org" +CACHE = "~/.cfabric/dictionaries/cache/sefaria" +AGENT = ( + f"cfabric-browser/{__version__} " + "(BDB for its lexicon; https://github.com/Context-Fabric/context-fabric)" +) + +# lexicon: the title of its book (for references and the text API) +LEXICONS = {"BDB Dictionary": "BDB", "BDB Aramaic Dictionary": "BDB Aramaic"} + +# BDB's homonyms are numbered with superscripts in Sefaria's headwords +SUPERSCRIPTS = "¹²³⁴⁵⁶⁷⁸⁹⁰" + +Download = Callable[[str], bytes] +# called with the number of entries so far and the last headword +Progress = Callable[[int, str], None] + + +class SefariaError(Exception): + """Sefaria didn't give what was asked for; the message says what.""" + + +def plain(headword: str) -> str: + """A headword without its homonym number.""" + return headword.rstrip(SUPERSCRIPTS + " ") + + +def text_url(book: str, headword: str) -> str: + quoted = urllib.parse.quote(f"{book}, {headword}", safe="") + return f"{SITE}/api/texts/{quoted}?context=0&commentary=0&pad=0" + + +def index_url(book: str) -> str: + return f"{SITE}/api/v2/index/{urllib.parse.quote(book, safe='')}" + + +def _download(url: str) -> bytes: + return read_url(url, agent=AGENT) + + +class Client: + """Requests to Sefaria with a cache, a pause between requests, and + retries.""" + + def __init__( + self, + cache: str | Path = CACHE, + pause: float = 1.0, + download: Download = _download, + sleep: Callable[[float], None] = time.sleep, + retries: int = 6, + ) -> None: + self.folder = Path(cache).expanduser() + self.folder.mkdir(parents=True, exist_ok=True) + self.pause = pause + self.download = download + self.sleep = sleep + self.retries = retries + self.requests = 0 # made over the network + self._last = 0.0 + self._sums: dict[str, str] = {} + sums = self.folder / "checksums.tsv" + if sums.is_file(): + for line in sums.read_text(encoding="utf-8").splitlines(): + name, _, digest = line.partition("\t") + self._sums[name] = digest + + def _name(self, url: str) -> str: + return hashlib.sha1(url.encode("utf-8")).hexdigest() + ".json" + + def cached(self, url: str) -> bytes | None: + name = self._name(url) + path = self.folder / name + if name in self._sums and path.is_file(): + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() == self._sums[name]: + return data + return None + + def get(self, url: str) -> Any: + """The JSON at `url`, from the cache or from Sefaria.""" + data = self.cached(url) + if data is None: + data = self._fetch(url) + name = self._name(url) + (self.folder / name).write_bytes(data) + digest = hashlib.sha256(data).hexdigest() + self._sums[name] = digest + with open(self.folder / "checksums.tsv", "a", encoding="utf-8") as f: + f.write(f"{name}\t{digest}\n") + try: + return json.loads(data) + except ValueError as e: + raise SefariaError(f"Not JSON from {url}") from e + + def _fetch(self, url: str) -> bytes: + wait = 0.0 + for attempt in range(self.retries + 1): + gap = max(self.pause, wait) - (time.monotonic() - self._last) + if gap > 0: + self.sleep(gap) + self._last = time.monotonic() + try: + data = self.download(url) + self.requests += 1 + return data + except urllib.error.HTTPError as e: + if e.code not in (429, 500, 502, 503, 504): + raise SefariaError(f"{url}: HTTP {e.code}") from e + after = e.headers.get("Retry-After") if e.headers else None + wait = float(after) if after and after.isdigit() else 0.0 + problem: Exception = e + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + problem = e + # wait longer after each failure: 5 s, 10 s, 20 s ... (at most 5 min) + wait = max(wait, min(300.0, 5.0 * 2**attempt)) + raise SefariaError(f"{url}: {problem}") + + +def first_headword(client: Client, lexicon: str) -> str: + index = client.get(index_url(LEXICONS[lexicon])) + for node in index.get("schema", {}).get("nodes", []): + if node.get("lexiconName") == lexicon and node.get("firstWord"): + return str(node["firstWord"]) + raise SefariaError(f"No first word for {lexicon}") + + +def entry(client: Client, lexicon: str, headword: str) -> dict[str, Any]: + """One entry of a lexicon, by its headword as Sefaria writes it.""" + book = LEXICONS[lexicon] + text = client.get(text_url(book, headword)) + if not isinstance(text, dict) or "error" in text or "text" not in text: + raise SefariaError(f"{lexicon} has no entry {headword}: {text}") + lines = text["text"] + + def after(ref: str | None) -> str | None: + prefix = f"{book}, " + return ref[len(prefix) :] if ref and ref.startswith(prefix) else None + + return { + "lexicon": lexicon, + "headword": headword, + "html": "".join(lines) if isinstance(lines, list) else str(lines), + "prev_hw": after(text.get("prev")), + "next_hw": after(text.get("next")), + } + + +def walk( + client: Client, + lexicon: str, + progress: Progress | None = None, + limit: int | None = None, +) -> list[dict[str, Any]]: + """All entries of a lexicon, in BDB's order.""" + entries: list[dict[str, Any]] = [] + seen: set[str] = set() + headword: str | None = first_headword(client, lexicon) + while headword and headword not in seen: + if limit is not None and len(entries) >= limit: + break + seen.add(headword) + e = entry(client, lexicon, headword) + entries.append(e) + if progress is not None: + progress(len(entries), headword) + headword = e.get("next_hw") + return entries + + +def main() -> None: + """Walk both lexicons into the cache and write them to `bdb-sefaria.json` + in the cache's folder (run by hand once, for the build).""" + import argparse + import sys + + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument("--cache", default=CACHE) + parser.add_argument("--pause", type=float, default=1.0) + parser.add_argument("--limit", type=int, default=None) + args = parser.parse_args() + client = Client(args.cache, pause=args.pause) + start = time.monotonic() + + def report(n: int, headword: str) -> None: + if n % 100 == 0: + minutes = (time.monotonic() - start) / 60 + print( + f"{n} entries, {client.requests} requests, {minutes:.1f} min", + file=sys.stderr, + flush=True, + ) + + result = {} + for lexicon in LEXICONS: + print(f"{lexicon} ...", file=sys.stderr, flush=True) + result[lexicon] = walk(client, lexicon, report, args.limit) + print(f"{lexicon}: {len(result[lexicon])} entries", file=sys.stderr) + out = client.folder / "bdb-sefaria.json" + out.write_text(json.dumps(result, ensure_ascii=False), encoding="utf-8") + print(f"{out}, {client.requests} requests", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/libs/explorer/cfabric_browser/dictionaries/store.py b/libs/explorer/cfabric_browser/dictionaries/store.py new file mode 100644 index 0000000..8988e6e --- /dev/null +++ b/libs/explorer/cfabric_browser/dictionaries/store.py @@ -0,0 +1,192 @@ +"""A dictionary file as the browser reads it (`bdb.json.gz`, from `build.py`). + +The file has the entries in BDB's order (Hebrew, then Aramaic), the links +from the BHSA's lexemes (`"Hebrew BR>["`) to entry ids, the main entry +first, and BDB's abbreviations. An entry's `section` is Open Scriptures' +root group: the root and the words BDB derives from it, its root family. +""" + +from __future__ import annotations + +import gzip +import json +import re +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# the fields of an entry in a list (the lexeme page, the root family) +SUMMARY = ("id", "lexicon", "headword", "pos", "defs", "incomplete", "ref") + + +# the letter of each part of BDB, by the first letter of Open Scriptures' ids +# (`b.cw.aa` is in ב; the Aramaic part's ids have `x` before it) +PARTS = dict(zip("abcdefghijklmnopqrstuvw", "אבגדהוזחטיכלמנסעפצקרששת")) + + +class DictionaryError(Exception): + """The file is not a dictionary this browser can read.""" + + +# BDB's grammatical abbreviations, which start its bold words (`n.pr.m.`, +# `prep. in, with`) +GRAMMAR = re.compile( + r"^(?:(?:n|pr|m|f|pl|du|sg|adj|gent|loc|terr|div|mont|fl|vb|prep|adv|conj" + r"|interj|pron|subst|denom|coll|abstr|id|abs|cstr|intens)\.\s*)+" +) + + +def short(entry: dict[str, Any]) -> str: + """A short definition: Open Scriptures' definitions, else the first words + BDB sets in bold (its glosses) once its grammatical abbreviations are + left out; empty when only those are there (most names).""" + if entry.get("defs"): + return ", ".join(entry["defs"]) + for bold in re.findall(r"([^<]+)", entry.get("html") or "")[:3]: + words = GRAMMAR.sub("", bold.strip()).strip(" ,;:()") + if re.search(r"[a-z]{2}", words): + return words + return "" + + +@dataclass +class Dictionary: + meta: dict[str, Any] + entries: list[dict[str, Any]] + links: dict[str, dict[str, Any]] + abbreviations: dict[str, str] + byId: dict[str, int] = field(default_factory=dict) # id: position + bySection: dict[str, list[int]] = field(default_factory=dict) + lexemes: dict[str, list[str]] = field(default_factory=dict) # id: keys + + def __post_init__(self) -> None: + for i, e in enumerate(self.entries): + self.byId[e["id"]] = i + if e.get("section"): + self.bySection.setdefault(e["section"], []).append(i) + for key, link in self.links.items(): + for i in link["entries"]: + self.lexemes.setdefault(i, []).append(key) + + @classmethod + def load(cls, path: str | Path) -> Dictionary: + try: + data = json.loads(gzip.decompress(Path(path).read_bytes())) + except (OSError, ValueError) as e: + raise DictionaryError(f"{path} is not a dictionary file: {e}") from e + if not isinstance(data, dict) or not {"meta", "entries", "links"} <= set(data): + raise DictionaryError(f"{path} is not a dictionary file") + return cls( + data["meta"], + data["entries"], + data["links"], + data.get("abbreviations") or {}, + ) + + def summary(self, entry: dict[str, Any]) -> dict[str, Any]: + found = {k: entry.get(k) for k in SUMMARY} + found["short"] = short(entry) + return found + + def for_lexeme(self, language: str | None, lex: str | None) -> dict[str, Any]: + """The entries a lexeme links to, the main one first, and how the link + was made (`oshb`, `cited`, `manual`); no entries if it has no link.""" + link = self.links.get(f"{language} {lex}") + if link is None: + return {"entries": [], "how": None} + return { + "entries": [ + self.summary(self.entries[self.byId[i]]) + for i in link["entries"] + if i in self.byId + ], + "how": link.get("how"), + "share": link.get("share"), + } + + def letter(self, eid: str) -> str | None: + """The letter of the part of BDB an entry is in (its root's letter); + for an entry without an Open Scriptures id, that of the one before.""" + i = self.byId.get(eid) + while i is not None and i >= 0: + part = self.entries[i]["id"].split(".")[0].removeprefix("x") + if part in PARTS: + return PARTS[part] + i -= 1 + return None + + def for_list(self, language: str | None, lex: str | None) -> dict[str, Any]: + """What the lexicon's list shows of a lexeme's main entry: its short + definition, its place in BDB (for ordering by root), the letter of + its part, and whether it is a root; empty without a link.""" + link = self.links.get(f"{language} {lex}") + if not link or link["entries"][0] not in self.byId: + return {} + eid = link["entries"][0] + i = self.byId[eid] + return { + "bdbShort": short(self.entries[i]), + "bdbOrder": i, + "bdbLetter": self.letter(eid), + "bdbRoot": bool(self.entries[i].get("root")), + } + + def article(self, eid: str) -> dict[str, Any] | None: + """An entry with its text, its neighbours in BDB, its root family and + the lexemes linked to it (`"Hebrew BR>["`).""" + i = self.byId.get(eid) + if i is None: + return None + entry = self.entries[i] + family = [ + self.summary(self.entries[j]) + for j in self.bySection.get(entry.get("section") or "", []) + ] + + def neighbour(j: int) -> dict[str, Any] | None: + if 0 <= j < len(self.entries): + other = self.entries[j] + if other["lexicon"] == entry["lexicon"]: + return {"id": other["id"], "headword": other["headword"]} + return None + + return { + **self.summary(entry), + "html": entry.get("html") or "", + "root": entry.get("root"), + "strong": entry.get("strong") or [], + "twot": entry.get("twot") or [], + "previous": neighbour(i - 1), + "next": neighbour(i + 1), + "family": family if len(family) > 1 else [], + "lexemes": self.lexemes.get(eid, []), + } + + +class Dictionaries: + """The dictionaries in a folder (`//.json.gz`), read when + first asked for and again when the file changes (added, removed).""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root).expanduser() + self._lock = threading.Lock() + self._loaded: dict[str, tuple[float, Dictionary]] = {} + + def path(self, did: str) -> Path: + return self.root / did / f"{did}.json.gz" + + def get(self, did: str) -> Dictionary | None: + path = self.path(did) + try: + mtime = path.stat().st_mtime + except OSError: + with self._lock: + self._loaded.pop(did, None) + return None + with self._lock: + found = self._loaded.get(did) + if found is None or found[0] != mtime: + found = (mtime, Dictionary.load(path)) + self._loaded[did] = found + return found[1] diff --git a/libs/explorer/cfabric_browser/display/__init__.py b/libs/explorer/cfabric_browser/display/__init__.py new file mode 100644 index 0000000..7339537 --- /dev/null +++ b/libs/explorer/cfabric_browser/display/__init__.py @@ -0,0 +1,61 @@ +""" +# Display + +Structured, HTML-free displays of corpus material: a port of Text-Fabric's +`tf.advanced` without HTML. Moved here from the core (it was only on the +browser's branches), so that the browser works with any core; it uses only +the core's public API (`F`, `L`, `T`, `E`). + +* `display.config`: how a corpus wants to be displayed, read from + Text-Fabric app configurations (`config.yaml`) plus computed defaults. +* `display.sections`: section references such as `Genesis 1:1`. +* `display.tree`: render trees, the nested fragments of a node that + clients draw as plain or pretty displays. `unravel` must stay identical + to Text-Fabric's output. + +Example +------- + +>>> from cfabric_browser.display import DisplayConfig, RenderOptions, render_node +>>> config = DisplayConfig.from_api(api, corpus_path=path) +>>> tree = render_node(api, config, verse, mode="pretty") +>>> tree.to_dict() # JSON-ready +""" + +from cfabric_browser.display.config import DisplayConfig, TypeDisplay, find_app_config +from cfabric_browser.display.sections import ( + node_from_section_ref, + section_ref, + section_ref_for, + structure_ref, +) +from cfabric_browser.display.tree import ( + PLAIN, + PRETTY, + Fragment, + RenderOptions, + RenderTree, + render_node, + render_tuple, + tuple_containers, + unravel, +) + +__all__ = [ + "PLAIN", + "PRETTY", + "DisplayConfig", + "Fragment", + "RenderOptions", + "RenderTree", + "TypeDisplay", + "find_app_config", + "node_from_section_ref", + "render_node", + "render_tuple", + "section_ref", + "section_ref_for", + "structure_ref", + "tuple_containers", + "unravel", +] diff --git a/libs/explorer/cfabric_browser/display/config.py b/libs/explorer/cfabric_browser/display/config.py new file mode 100644 index 0000000..d786997 --- /dev/null +++ b/libs/explorer/cfabric_browser/display/config.py @@ -0,0 +1,500 @@ +"""Display configuration of a corpus. + +How a corpus is displayed (which features label which node types, how deep +displays unfold, writing direction, section separators, ...) is configured per +corpus. Text-Fabric keeps this in the `config.yaml` of the corpus' *app*. +We read those files as they are, so every existing TF corpus displays +correctly, and compute the same defaults as Text-Fabric for whatever the file +leaves open. + +The defaults follow `tf.advanced.settings` (Text-Fabric, MIT licensed, +by Dirk Roorda). +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import yaml + +if TYPE_CHECKING: + from cfabric.core.api import Api + +logger = logging.getLogger(__name__) + +APP_DIR = "app" +APP_CONFIG = "config.yaml" +APP_SEARCH_DEPTH = 4 + +VAR_PATTERN = re.compile(r"\{([^}]+)\}") + +WRITING = { + "akk": ("akkadian", "ltr"), + "hbo": ("hebrew", "rtl"), + "syc": ("syriac", "rtl"), + "uga": ("ugaritic", "ltr"), + "ara": ("arabic", "rtl"), + "grc": ("greek", "ltr"), + "cld": ("aramaic", "ltr"), + "eng": ("english", "ltr"), + "": ("", "ltr"), +} + +ORIG = "orig" +DEFAULT_FORMAT = "text-orig-full" + +LEVEL_FLOW = {4: "hor", 3: "hor", 2: "hor", 1: "hor", 0: "ver"} +FLOW_DEFAULTS = { + "ver": {"wrap": False, "stretch": False}, + "hor": {"wrap": True, "stretch": True}, +} + + +def parse_features(spec: Any) -> tuple[tuple[str, ...], dict[str, str]]: + """Parse a feature list like `"pdp vs lex:gloss"`. + + Returns the bare feature names and a mapping from feature name to the + node type it must be looked up on (for `type:feature` entries). + """ + if not spec: + return ((), {}) + items = spec.split() if isinstance(spec, str) else list(spec) + names: list[str] = [] + indirect: dict[str, str] = {} + for item in items: + if not item: + continue + parts = str(item).split(":", 1) + name = parts[-1] + names.append(name) + if len(parts) > 1: + indirect[name] = parts[0] + return (tuple(names), indirect) + + +@dataclass +class TypeDisplay: + """How nodes of one type are displayed.""" + + type: str + level: int + flow: str + wrap: bool + stretch: bool + template: str | bool + """Plain display: `True` means the node's text (or section heading), + a string is a template with `{feature}` placeholders, `""` means nothing.""" + label: str | bool + """Pretty display: same conventions as `template`.""" + features: tuple[str, ...] = () + features_indirect: dict[str, str] = field(default_factory=dict) + features_bare: tuple[str, ...] = () + features_bare_indirect: dict[str, str] = field(default_factory=dict) + style: str | None = None + hidden: bool = False + children: tuple[str, ...] = () + is_big: bool = False + is_lex: bool = False + line_number: str | None = None + graphics: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "type": self.type, + "level": self.level, + "flow": self.flow, + "wrap": self.wrap, + "stretch": self.stretch, + "template": self.template, + "label": self.label, + "features": list(self.features), + "featuresIndirect": dict(self.features_indirect), + "featuresBare": list(self.features_bare), + "featuresBareIndirect": dict(self.features_bare_indirect), + "style": self.style, + "hidden": self.hidden, + "children": list(self.children), + "isBig": self.is_big, + "isLex": self.is_lex, + "lineNumber": self.line_number, + "graphics": self.graphics, + } + + +@dataclass +class DisplayConfig: + """Everything needed to display nodes of a corpus.""" + + slot_type: str + node_types: tuple[str, ...] + types: dict[str, TypeDisplay] + section_types: tuple[str, ...] + structure_types: tuple[str, ...] + text_formats: tuple[str, ...] + default_format: str + format_styles: dict[str, str | None] + section_sep1: str = " " + section_sep2: str = ":" + none_values: frozenset[Any] = frozenset({None}) + excluded_features: frozenset[str] = frozenset() + language: str = "" + language_code: str = "" + direction: str = "ltr" + base_types: frozenset[str] = frozenset() + condense_type: str | None = None + verse_types: frozenset[str] = frozenset() + lex_types: frozenset[str] = frozenset() + lex_map: dict[str, str] = field(default_factory=dict) + exclusions: dict[str, dict[str, Any]] = field(default_factory=dict) + descendant_types: dict[str, frozenset[str]] = field(default_factory=dict) + show_verse_in_tuple: bool = False + browse_nav_level: int = 1 + example_section: str | None = None + provenance: dict[str, Any] = field(default_factory=dict) + source: Path | None = None + raw: dict[str, Any] = field(default_factory=dict, repr=False) + + @property + def hidden_types(self) -> frozenset[str]: + """Types that are hidden by default.""" + return frozenset(t for (t, d) in self.types.items() if d.hidden) + + @property + def big_types(self) -> frozenset[str]: + return frozenset(t for (t, d) in self.types.items() if d.is_big) + + def ltr(self, fmt: str | None) -> str: + """Writing direction for a text format: `rtl` or `ltr`. + + Only formats in the original script (`...-orig-...`) of a right-to-left + corpus are right-to-left; transliterations are left-to-right. + """ + fmt = fmt or self.default_format + if self.direction == "rtl" and (f"{ORIG}-" in fmt or f"-{ORIG}" in fmt): + return "rtl" + return "ltr" + + def to_dict(self) -> dict[str, Any]: + """JSON-ready representation, for clients that render displays.""" + return { + "slotType": self.slot_type, + "nodeTypes": list(self.node_types), + "types": {t: d.to_dict() for (t, d) in self.types.items()}, + "sectionTypes": list(self.section_types), + "structureTypes": list(self.structure_types), + "textFormats": list(self.text_formats), + "defaultFormat": self.default_format, + "formatStyles": dict(self.format_styles), + "sectionSep1": self.section_sep1, + "sectionSep2": self.section_sep2, + "noneValues": sorted(str(v) for v in self.none_values if v is not None), + "excludedFeatures": sorted(self.excluded_features), + "language": self.language, + "languageCode": self.language_code, + "direction": self.direction, + "baseTypes": sorted(self.base_types), + "condenseType": self.condense_type, + "verseTypes": sorted(self.verse_types), + "lexTypes": sorted(self.lex_types), + "hiddenTypes": sorted(self.hidden_types), + "bigTypes": sorted(self.big_types), + "browseNavLevel": self.browse_nav_level, + "exampleSection": self.example_section, + "provenance": _jsonable(self.provenance), + } + + @classmethod + def from_api( + cls, + api: Api, + config: str | Path | dict[str, Any] | None = None, + corpus_path: str | Path | None = None, + ) -> DisplayConfig: + """Build the display configuration of a loaded corpus. + + Parameters + ---------- + api: Api + The loaded corpus. + config: str | Path | dict | None + A TF app `config.yaml` (path) or its parsed contents. + If `None`, it is looked up near `corpus_path` with `find_app_config`. + corpus_path: str | Path | None + Directory of the corpus data, used to find the app config. + """ + source: Path | None = None + if config is None and corpus_path is not None: + source = find_app_config(corpus_path) + config = source + if isinstance(config, (str, Path)): + source = Path(config) + config = read_app_config(source) + return _build(api, config or {}, source) + + +def find_app_config(corpus_path: str | Path) -> Path | None: + """Find the TF app `config.yaml` that belongs to a corpus. + + Text-Fabric corpora keep their data in `/tf/` and their app + in `/app`. We look for `app/config.yaml` in the corpus directory and + up to a few levels above it. + """ + path = Path(corpus_path).expanduser().resolve() + for folder in [path, *path.parents][: APP_SEARCH_DEPTH + 1]: + candidate = folder / APP_DIR / APP_CONFIG + if candidate.is_file(): + return candidate + return None + + +def read_app_config(path: str | Path) -> dict[str, Any]: + """Read a TF app `config.yaml`.""" + with open(path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + if isinstance(data, dict): + return data + raise ValueError(f"App config {path} is not a mapping") + + +def _jsonable(value: Any) -> Any: + if isinstance(value, dict): + return {str(k): _jsonable(v) for (k, v) in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [_jsonable(v) for v in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def _transitive_closure( + relation: dict[str, set[str]], reflexive_exceptions: set[str] +) -> dict[str, frozenset[str]]: + descendants = {parent: set(children) for (parent, children) in relation.items()} + changed = True + while changed: + changed = False + for parent, children in relation.items(): + for child in children: + for grandChild in descendants.get(child, ()): + if grandChild not in descendants[parent]: + descendants[parent].add(grandChild) + changed = True + for parent in relation: + if parent not in reflexive_exceptions: + descendants[parent].add(parent) + return {k: frozenset(v) for (k, v) in descendants.items()} + + +def _build(api: Api, cfg: dict[str, Any], source: Path | None) -> DisplayConfig: + F = api.F + T = api.T + otypeRank = api.N.otypeRank + + slotType = F.otype.slotType + nTypes = tuple(F.otype.all) + sectionTypes = tuple(T.sectionTypes or ()) + structureTypes = tuple(getattr(T, "structureTypes", ()) or ()) + sectionalTypes = set(sectionTypes) | set(structureTypes) + + dataDisplay = cfg.get("dataDisplay", {}) or {} + typeInfo = cfg.get("typeDisplay", {}) or {} + + unknown = sorted(t for t in typeInfo if t not in nTypes) + if unknown: + logger.warning("App config mentions unknown node types: %s", ", ".join(unknown)) + typeInfo = {t: v for (t, v) in typeInfo.items() if t in nTypes} + + # writing system + + writing = cfg.get("writing", "") or "" + (language, direction) = WRITING.get(writing, ("", "ltr")) + + # text formats + + formats = tuple(sorted(T.formats)) if T.formats else () + givenFormats = dataDisplay.get("textFormats", {}) or {} + formatStyles: dict[str, str | None] = { + fmt: (givenFormats.get(fmt) or {}).get("style") for fmt in formats + } + defaultFormat = dataDisplay.get("textFormat") or ( + DEFAULT_FORMAT if DEFAULT_FORMAT in formats or not formats else formats[0] + ) + + # per type settings + + verseTypes = {sectionTypes[-1]} if sectionTypes else set() + verseRank = otypeRank[sectionTypes[-1]] if sectionTypes else None + lexMap: dict[str, str] = {} + baseTypes: set[str] = set() + condenseType: str | None = None + children: dict[str, set[str]] = {} + exclusions: dict[str, dict[str, Any]] = {} + givenLevels: dict[str, dict[str, Any]] = {} + + for nType, info in typeInfo.items(): + info = info or {} + if info.get("verselike", False): + verseTypes.add(nType) + lexOcc = info.get("lexOcc") + if lexOcc is not None: + lexMap[lexOcc] = nType + if "base" in info: + baseTypes.add(nType) + if info.get("condense", False): + condenseType = nType + if "children" in info: + kids = info["children"] or () + children[nType] = {kids} if isinstance(kids, str) else set(kids) + if any(k in info for k in ("level", "flow", "wrap", "stretch")): + givenLevels[nType] = { + k: info[k] for k in ("level", "flow", "wrap", "stretch") if k in info + } + if "exclude" in info: + exclusions[nType] = info["exclude"] or {} + + lexTypes = set(lexMap.values()) + nTypesNoLex = [t for t in nTypes if t not in lexTypes] + + # levels, as in TF: 4 = sections, 3 = verses, 0 = slots and lexemes, + # the remaining types are divided over 2 (bigger) and 1 (smaller) + + levelTypes: list[set[str]] = [set(), set(), set(), set(), set()] + levelTypes[4] = sectionalTypes - verseTypes + levelTypes[3] = set(verseTypes) + levelTypes[0] = {slotType} | lexTypes + remainingSet = set(nTypes) - levelTypes[4] - levelTypes[3] - levelTypes[0] + remaining = tuple(t for t in nTypes if t in remainingSet) + if len(remaining) == 1: + levelTypes[1] = {remaining[0]} + elif len(remaining) > 1: + mid = len(remaining) // 2 + levelTypes[2] = set(remaining[0:mid]) + levelTypes[1] = set(remaining[mid:]) + + # which types may occur directly inside which types + + childType: dict[str, set[str]] = { + t: {nTypesNoLex[i + 1]} + for (i, t) in enumerate(nTypesNoLex) + if i < len(nTypesNoLex) - 1 + } + for seq in (structureTypes, sectionTypes): + for i, t in enumerate(seq[:-1]): + childType.setdefault(t, set()).add(seq[i + 1]) + for t, kids in children.items(): + childType[t] = kids + + lowestSectional = set(verseTypes) + if sectionTypes: + lowestSectional.add(sectionTypes[-1]) + if structureTypes: + lowestSectional.add(structureTypes[-1]) + + biggestOther = slotType + for t in remaining: + if verseRank is None or otypeRank[t] < verseRank: + biggestOther = t + break + smallestOther = remaining[-1] if remaining else None + + for t in lexTypes: + childType.pop(t, None) + for t in lowestSectional: + childType.setdefault(t, set()).add(biggestOther) + if smallestOther is not None and smallestOther != slotType: + childType.setdefault(smallestOther, set()).add(slotType) + + if condenseType is None: + if sectionTypes: + condenseType = sectionTypes[-1] + elif remaining: + condenseType = ( + remaining[len(remaining) // 2] if len(remaining) > 1 else remaining[0] + ) + else: + condenseType = slotType + + descendantTypes = _transitive_closure(childType, {slotType}) + + types: dict[str, TypeDisplay] = {} + for i, levelSet in enumerate(levelTypes): + for nType in levelSet: + info = typeInfo.get(nType, {}) or {} + given = givenLevels.get(nType, {}) + level = given.get("level", i) + isVerse = nType in verseTypes + flow = given.get("flow", "hor" if isVerse else LEVEL_FLOW.get(level, "hor")) + flowDefaults = FLOW_DEFAULTS.get(flow, FLOW_DEFAULTS["hor"]) + defaultTpl: str | bool = ( + True + if nType == slotType or nType in sectionalTypes - verseTypes + else "" + ) + (features, featuresIndirect) = parse_features(info.get("features")) + (featuresBare, featuresBareIndirect) = parse_features( + info.get("featuresBare") + ) + style = info.get("style") + types[nType] = TypeDisplay( + type=nType, + level=level, + flow=flow, + wrap=given.get("wrap", flowDefaults["wrap"]), + stretch=given.get("stretch", flowDefaults["stretch"]), + template=info.get("template", defaultTpl), + label=info.get("label", defaultTpl), + features=features, + features_indirect=featuresIndirect, + features_bare=featuresBare, + features_bare_indirect=featuresBareIndirect, + style=style if style else None, + hidden=bool(info.get("hidden", False)), + children=tuple(t for t in nTypes if t in childType.get(nType, ())), + is_big=bool(info.get("isBig", False)), + is_lex=nType in lexTypes, + line_number=info.get("lineNumber"), + graphics=bool(info.get("graphics", False)), + ) + types = {t: types[t] for t in nTypes if t in types} + + noneValues = dataDisplay.get("noneValues") + noneSet = frozenset({None} | set(noneValues or ())) + + browseNavLevel = dataDisplay.get("browseNavLevel") + if browseNavLevel is None: + browseNavLevel = len(sectionTypes) - 1 if sectionTypes else 1 + + return DisplayConfig( + slot_type=slotType, + node_types=nTypes, + types=types, + section_types=sectionTypes, + structure_types=structureTypes, + text_formats=formats, + default_format=defaultFormat, + format_styles=formatStyles, + section_sep1=dataDisplay.get("sectionSep1", " "), + section_sep2=dataDisplay.get("sectionSep2", ":"), + none_values=noneSet, + excluded_features=frozenset(dataDisplay.get("excludedFeatures") or ()), + language=language, + language_code=writing, + direction=direction, + base_types=frozenset(baseTypes or {slotType}), + condense_type=condenseType, + verse_types=frozenset(verseTypes), + lex_types=frozenset(lexTypes), + lex_map=lexMap, + exclusions=exclusions, + descendant_types=descendantTypes, + show_verse_in_tuple=bool(dataDisplay.get("showVerseInTuple", False)), + browse_nav_level=browseNavLevel, + example_section=dataDisplay.get("exampleSection"), + provenance=dict(cfg.get("provenanceSpec", {}) or {}), + source=source, + raw=cfg, + ) diff --git a/libs/explorer/cfabric_browser/display/sections.py b/libs/explorer/cfabric_browser/display/sections.py new file mode 100644 index 0000000..abe2734 --- /dev/null +++ b/libs/explorer/cfabric_browser/display/sections.py @@ -0,0 +1,158 @@ +"""Human-readable section and structure references. + +Section references are built from the section levels of the corpus, joined by +the separators of its display configuration: `Genesis 1:1` for the BHSA, +`P005381 1:1` for a cuneiform tablet, `S1 1` for a corpus with two levels. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from cfabric.core.api import Api + + from cfabric_browser.display.config import DisplayConfig + + +def section_feature_types(api: Api) -> list[str]: + """The value type (`int` or `str`) of each section feature: the Python + core has them as `T.sectionFeatureTypes`, the Rust core per feature.""" + found = getattr(api.T, "sectionFeatureTypes", None) + if found: + return list(found) + return [ + "int" if getattr(api.Fs(name), "valueType", None) == "int" else "str" + for name in api.T.sectionFeatures + ] + + +def section_ref( + api: Api, + node: int, + sep1: str = " ", + sep2: str = ":", + lang: str = "en", + last_slot: bool = False, + fillup: bool = False, + level: int | None = None, +) -> str: + """The section reference of a node, e.g. `Genesis 1:1`. + + Parameters + ---------- + api: Api + The loaded corpus. + node: int + Any node; its first (or last) slot determines the section. + sep1, sep2: str + Separators between section levels 0 and 1, and 1 and 2. + lang: str + Language of section names that have translations (e.g. book names). + last_slot: bool + Take the section of the last slot instead of the first. + fillup: bool + See `cfabric.navigation.text.Text.sectionFromNode`. + level: int | None + Restrict to this section level. + + Returns + ------- + str + The reference, or the empty string if the corpus has no sections. + """ + T = api.T + if not T.sectionTypes: + return "" + section = T.sectionFromNode( + int(node), lang=lang, lastSlot=last_slot, fillup=fillup, level=level + ) + if not section: + return "" + seps = ("", sep1, sep2) + return "".join( + "" if part is None else f"{seps[i]}{part}" for (i, part) in enumerate(section) + ) + + +def section_ref_for( + api: Api, config: DisplayConfig, node: int, lang: str = "en", **kwargs +) -> str: + """`section_ref` with the separators of a display configuration.""" + return section_ref( + api, + node, + sep1=config.section_sep1, + sep2=config.section_sep2, + lang=lang, + **kwargs, + ) + + +def node_from_section_ref( + api: Api, + ref: str, + sep1: str = " ", + sep2: str = ":", + lang: str = "en", +) -> int: + """The section node for a reference like `Genesis 1:1`, `Genesis 1` or `Genesis`. + + Section names may contain `sep1` themselves (e.g. `Song of Songs 1:1`): + the reference is split at the last occurrence. + + Raises + ------ + ValueError + If the reference is malformed or does not exist in the corpus. + """ + T = api.T + sectionTypes = T.sectionTypes + if not sectionTypes: + raise ValueError("This corpus has no sections") + + ref = ref.strip() + if not ref: + raise ValueError("Empty section reference") + + # Candidate readings: split into levels at the last `sep1`, and, as a + # fallback, the whole reference as a level-0 name (which may contain `sep1`). + candidates: list[list[str]] = [] + if len(sectionTypes) > 1: + head, sep, tail = ref.rpartition(sep1) + if sep and head: + sub = tail.split(sep2) if len(sectionTypes) > 2 else [tail] + candidates.append([head, *sub]) + candidates.append([ref]) + + error = f'Not a valid passage: "{ref}"' + dataTypes = section_feature_types(api) + for parts in candidates: + if len(parts) > len(sectionTypes): + continue + typed: list[str | int] = [] + for i, part in enumerate(parts): + if dataTypes[i] == "int": + try: + typed.append(int(part)) + except ValueError: + error = f'{sectionTypes[i]} "{part}" is not a number' + break + else: + typed.append(part) + else: + node = T.nodeFromSection(tuple(typed), lang=lang) + if node is not None: + return int(node) + raise ValueError(error) + + +def structure_ref(api: Api, node: int) -> str: + """The structure heading of a node, e.g. `chapter 3, section 2`.""" + T = api.T + if not getattr(T, "structureTypes", None): + return "" + heading = T.headingFromNode(int(node)) + if not heading: + return "" + return ", ".join(f"{part[0]} {part[1]}" for part in heading if part is not None) diff --git a/libs/explorer/cfabric_browser/display/tree.py b/libs/explorer/cfabric_browser/display/tree.py new file mode 100644 index 0000000..b996296 --- /dev/null +++ b/libs/explorer/cfabric_browser/display/tree.py @@ -0,0 +1,704 @@ +"""Render trees: structured, HTML-free displays of nodes. + +A *render tree* shows a node with everything inside it, as nested fragments. +Clients (the Context-Fabric browser, notebooks, exporters) turn it into +visual displays; nothing here produces HTML. + +Nodes in a corpus do not nest neatly: a phrase may have gaps, a clause may +cross a verse boundary. *Unraveling* cuts nodes into fragments at the +boundaries of other nodes, so that the fragments do nest. Each fragment +records whether its edges are the real start/end of its node (`node`), +the edge of a gap in a discontinuous node (`gap`), or a cut made by +unraveling (`split`). + +The algorithm is a port of `tf.advanced.unravel` (Text-Fabric, MIT licensed, +by Dirk Roorda); see `tf.about.displaydesign` for its rationale. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field +from itertools import chain +from typing import TYPE_CHECKING, Any + +from cfabric_browser.display.sections import section_ref, structure_ref +from cfabric_browser.helpers import rangesFromList + +if TYPE_CHECKING: + from cfabric.core.api import Api + + from cfabric_browser.display.config import DisplayConfig + +PRETTY = "pretty" +PLAIN = "plain" + +NODE = "node" +GAP = "gap" +SPLIT = "split" + + +@dataclass +class RenderOptions: + """Options that change the shape or content of a render tree. + + Options that only change how a tree *looks* (show node numbers, show + standard features, ...) are left to the client. + """ + + fmt: str | None = None + """Text format; defaults to the corpus' default format.""" + full: bool = False + """Unfold big nodes (books, chapters) instead of showing only their heading.""" + hide_types: bool = True + """Leave out nodes of `hidden_types`.""" + hidden_types: Iterable[str] | None = None + """Types to hide; defaults to the types marked hidden in the configuration.""" + base_types: Iterable[str] | None = None + """Pretty displays do not unfold nodes of these types; defaults to the + configured base types (normally the slot type).""" + condense_type: str | None = None + """Container type for tuples (e.g. `verse`); defaults to the configured one.""" + extra_features: Iterable[str] = () + """Features to show besides the configured ones, e.g. those of a query. + Entries may be `type:feature` to take the value from an enclosing node.""" + highlights: dict[int, str] | None = None + """Nodes to highlight, mapped to a color (`""` for the default color).""" + lang: str = "en" + """Language for section names.""" + + +@dataclass +class Fragment: + """A piece of a node that nests properly inside its parent fragment.""" + + node: int + type: str + begin: int + end: int + start: str + """`node`, `gap` or `split`: what the begin of this fragment is.""" + stop: str + """`node`, `gap` or `split`: what the end of this fragment is.""" + children: list[Fragment] = field(default_factory=list) + label: str | None = None + heading: str | None = None + text: str | None = None + features: list[dict[str, Any]] | None = None + highlight: str | None = None + base: bool = False + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "node": self.node, + "type": self.type, + "range": [self.begin, self.end], + "start": self.start, + "end": self.stop, + } + for key in ("label", "heading", "text", "features", "highlight"): + value = getattr(self, key) + if value is not None: + result[key] = value + if self.base: + result["base"] = True + if self.children: + result["children"] = [c.to_dict() for c in self.children] + return result + + +@dataclass +class RenderTree: + """The display of one node: its fragments plus what is needed to draw them.""" + + node: int + type: str + mode: str + fmt: str + direction: str + heading: str + fragments: list[Fragment] + + def to_dict(self) -> dict[str, Any]: + return { + "node": self.node, + "type": self.type, + "mode": self.mode, + "fmt": self.fmt, + "direction": self.direction, + "heading": self.heading, + "fragments": [f.to_dict() for f in self.fragments], + } + + def walk(self) -> Iterable[tuple[int, Fragment]]: + """All fragments, depth first, with their depth.""" + + def visit(frags: list[Fragment], depth: int): + for f in frags: + yield (depth, f) + yield from visit(f.children, depth + 1) + + yield from visit(self.fragments, 0) + + def outline(self) -> str: + """A compact text view of the tree, for debugging and tests.""" + lines = [] + for depth, f in self.walk(): + rng = f"{f.begin}" if f.begin == f.end else f"{f.begin}-{f.end}" + base = "*" if f.base else "" + edges = "" if f.start == NODE and f.stop == NODE else f" {f.start}/{f.stop}" + lines.append(f"{' ' * depth}{f.type}{base} {f.node} {{{rng}}}{edges}") + return "\n".join(lines) + + +# unraveling + + +def _is_big( + api: Api, + config: DisplayConfig, + pretty: bool, + full: bool, + condenseType: str | None, + nType: str, +) -> bool: + if full: + return False + otypeRank = api.N.otypeRank + if not pretty and nType in config.big_types: + return True + sectional = set(config.section_types) | set(config.structure_types) + if sectional and nType in sectional: + return condenseType is None or otypeRank[nType] > otypeRank[condenseType] + return condenseType is not None and otypeRank[nType] > otypeRank[condenseType] + + +def _split_points(p: tuple[int, int], q: tuple[int, int]) -> set[int]: + """Where the boundaries of range `p` cut through range `q`. + + A split point is the slot where the second part starts. + """ + (b1, e1) = p + (b2, e2) = q + if b2 == e2 or (b1 <= b2 and e1 >= e2): + return set() + points = set() + if b2 < b1 <= e2: + points.add(b1) + if b2 <= e1 < e2: + points.add(e1 + 1) + return points + + +def _apply_splits( + chunks: set[tuple[int, tuple[int, int]]], + splits: dict[tuple[int, tuple[int, int]], set[int]], +) -> None: + for target, points in splits.items(): + if not points: + continue + chunks.remove(target) + (m, (b, e)) = target + prev = b + for sp in sorted(points): + chunks.add((m, (prev, sp - 1))) + prev = sp + chunks.add((m, (prev, e))) + + +def unravel( + api: Api, + config: DisplayConfig, + n: int, + pretty: bool = True, + options: RenderOptions | None = None, + in_tuple: bool = False, +) -> list[Fragment]: + """Cut a node and the nodes inside it into properly nested fragments. + + Returns the top-level fragments: those of `n` itself (more than one if + `n` has gaps). Fragments are bare: no labels, texts or features yet; + see `render_node` for dressed-up trees. + """ + options = options or RenderOptions() + F = api.F + L = api.L + N = api.N + Fs = api.Fs + eOslots = api.E.oslots.s + fOtypeV = F.otype.v + nTypes = F.otype.all + + n = int(n) + nType = fOtypeV(n) + lexTypes = config.lex_types + descendantTypes = config.descendant_types + + condenseType = options.condense_type or config.condense_type + + if not pretty and nType in config.verse_types and not config.show_verse_in_tuple: + isBig = in_tuple + else: + isBig = _is_big(api, config, pretty, options.full, condenseType, nType) + + # the nodes that take part + + if isBig and not options.full: + iNodes: set[int] = set() + elif nType in descendantTypes: + iNodes = {int(m) for m in L.i(n, otype=descendantTypes[nType])} + elif nType in lexTypes: + iNodes = {n} + else: + iNodes = {int(m) for m in L.i(n)} + + if options.hide_types: + hidden = ( + config.hidden_types + if options.hidden_types is None + else frozenset(options.hidden_types) + ) + if hidden: + iNodes = {m for m in iNodes if fOtypeV(m) not in hidden} + + iNodes.add(n) + + # chunks: maximal contiguous slot ranges of each node, and their true boundaries + + nSlots = tuple(int(s) for s in eOslots(n)) + if nType in lexTypes: + nSlots = nSlots[:1] + nSlotSet = set(nSlots) + + chunks: dict[str, set[tuple[int, tuple[int, int]]]] = {} + boundaries: dict[int, dict[int, tuple[bool | None, bool | None]]] = {} + exclusions = config.exclusions + + for m in iNodes: + mType = fOtypeV(m) + if mType in exclusions and any( + Fs(feat).v(m) == value for (feat, value) in exclusions[mType].items() + ): + continue + + slots = [int(s) for s in eOslots(m)] + if nType in lexTypes: + slots = slots[:1] + if m != n and mType == nType and nSlotSet <= set(slots): + continue + minSlot = slots[0] + maxSlot = slots[-1] + bounds: dict[int, tuple[bool | None, bool | None]] = {} + for b, e in rangesFromList(slots): + chunks.setdefault(mType, set()).add((m, (b, e))) + bounds[b] = (b == minSlot, None if b != e else e == maxSlot) + bounds[e] = (b == minSlot if b == e else None, e == maxSlot) + boundaries[m] = bounds + + # fragmentize: bigger types cut smaller types, and same-type nodes cut each other + + sortKeyChunk = N.sortKeyChunk + sortKeyChunkLength = N.sortKeyChunkLength + typeLen = len(nTypes) - 1 + + for p, pType in enumerate(nTypes): + pChunks = chunks.get(pType) + if not pChunks: + continue + splits: dict[tuple[int, tuple[int, int]], set[int]] = {} + pSorted = sorted(pChunks, key=sortKeyChunkLength) + for i, pChunk in enumerate(pSorted): + for p2Chunk in pSorted[i + 1 :]: + splits.setdefault(p2Chunk, set()).update( + _split_points(pChunk[1], p2Chunk[1]) + ) + _apply_splits(pChunks, splits) + + for q in range(p + 1, typeLen): + qChunks = chunks.get(nTypes[q]) + if not qChunks: + continue + splits = {} + for qChunk in qChunks: + for pChunk in pChunks: + splits.setdefault(qChunk, set()).update( + _split_points(pChunk[1], qChunk[1]) + ) + _apply_splits(qChunks, splits) + + # each fragment now lies either inside the top node or outside it; + # testing its first slot suffices + + ordered = sorted( + (c for c in chain.from_iterable(chunks.values()) if c[1][0] in nSlotSet), + key=sortKeyChunk, + ) + + # stack the fragments + + def edge(m: int, slot: int, side: int) -> str: + bounds = boundaries[m].get(slot) + code = None if bounds is None else bounds[side] + return SPLIT if code is None else NODE if code else GAP + + top: list[Fragment] = [] + stack: list[Fragment] = [] + for m, (b, e) in ordered: + frag = Fragment( + node=m, + type=fOtypeV(m), + begin=b, + end=e, + start=edge(m, b, 0), + stop=edge(m, e, 1), + ) + while stack and e > stack[-1].end: + stack.pop() + (stack[-1].children if stack else top).append(frag) + stack.append(frag) + + return top + + +# dressing + + +class _Dresser: + """Adds labels, texts, headings, features and highlights to fragments.""" + + def __init__( + self, + api: Api, + config: DisplayConfig, + pretty: bool, + options: RenderOptions, + ) -> None: + self.api = api + self.config = config + self.pretty = pretty + self.options = options + T = api.T + self.fmt = options.fmt or config.default_format + self.descendType = T.formats.get(self.fmt, config.slot_type) or config.slot_type + + baseTypes = ( + frozenset(options.base_types) + if options.base_types is not None + else config.base_types + ) + subBaseTypes: set[str] = set() + if baseTypes and baseTypes != {config.slot_type}: + for bt in baseTypes: + subBaseTypes |= config.descendant_types.get(bt, frozenset()) + self.baseTypes = baseTypes + self.subBaseTypes = subBaseTypes + + (extra, extraIndirect) = _parse_extra(options.extra_features) + self.extra = extra + self.extraIndirect = extraIndirect + self.edgeFeatures = {e for e in api.Eall() if e != "oslots"} + self.highlights = options.highlights or {} + self._cache: dict[int, dict[str, Any]] = {} + + def is_base(self, nType: str) -> bool: + return nType != self.config.slot_type and ( + nType in self.baseTypes or nType in self.subBaseTypes + ) + + def dress(self, fragments: list[Fragment]) -> None: + for frag in fragments: + self._dress(frag) + + def _dress(self, frag: Fragment) -> None: + props = self._node_props(frag.node, frag.type) + frag.label = props["label"] + frag.heading = props["heading"] + frag.features = props["features"] + frag.highlight = self.highlights.get(frag.node) + + config = self.config + nType = frag.type + isSlot = nType == config.slot_type + isDescend = nType == self.descendType + isBase = self.pretty and self.is_base(nType) + + if isSlot or isDescend or nType in config.lex_types: + frag.text = props["text"] + frag.children = [] + elif isBase: + frag.base = True + frag.text = self._fragment_text(frag) + frag.children = [] + for child in frag.children: + self._dress(child) + + def _fragment_text(self, frag: Fragment) -> str: + eOslots = self.api.E.oslots.s + slots = [int(s) for s in eOslots(frag.node) if frag.begin <= s <= frag.end] + return str(self.api.T.text(slots, fmt=self.fmt)) + + def _node_props(self, n: int, nType: str) -> dict[str, Any]: + cached = self._cache.get(n) + if cached is not None: + return cached + + api = self.api + config = self.config + T = api.T + typeDisplay = config.types.get(nType) + tpl: str | bool = "" + if typeDisplay is not None: + tpl = typeDisplay.label if self.pretty else typeDisplay.template + + heading = None + label = None + text = None + if nType in config.section_types: + heading = section_ref( + api, + n, + sep1=config.section_sep1, + sep2=config.section_sep2, + lang=self.options.lang, + ) + elif nType in config.structure_types: + heading = structure_ref(api, n) + + if ( + nType == config.slot_type + or nType == self.descendType + or nType in config.lex_types + ): + descend = ( + False + if nType == self.descendType or nType in config.lex_types + else None + ) + text = T.text(n, fmt=self.fmt, descend=descend) + + if isinstance(tpl, str) and tpl: + label = self._fill(n, nType, tpl) + + features = self._features(n, nType) if self.pretty else None + + props = {"label": label, "heading": heading, "text": text, "features": features} + self._cache[n] = props + return props + + def _value(self, n: int, feat: str, keep_none_values: bool = False) -> Any: + api = self.api + if feat in self.edgeFeatures: + return None + fObj = api.Fs(feat) if hasattr(api.F, feat) else None + if fObj is None: + return None + value = fObj.v(n) + if not keep_none_values and value in self.config.none_values: + return None + return value + + def _fill(self, n: int, nType: str, tpl: str) -> str: + from cfabric_browser.display.config import VAR_PATTERN + + # As in TF, templates show values like `NA` that feature lists suppress. + def replace(match): + value = self._value(n, match.group(1), keep_none_values=True) + return "" if value is None else str(value) + + return VAR_PATTERN.sub(replace, tpl) + + def _features(self, n: int, nType: str) -> list[dict[str, Any]] | None: + typeDisplay = self.config.types.get(nType) + entries: list[tuple[str, str, str | None]] = [] + if typeDisplay is not None: + entries += [ + (f, "bare", typeDisplay.features_bare_indirect.get(f)) + for f in typeDisplay.features_bare + ] + entries += [ + (f, "standard", typeDisplay.features_indirect.get(f)) + for f in typeDisplay.features + ] + configured = {f for (f, _, _) in entries} + entries += [ + (f, "extra", self.extraIndirect.get(f)) + for f in self.extra + if f not in configured + ] + + L = self.api.L + result = [] + seen = set() + for name, kind, refType in entries: + if name in seen: + continue + seen.add(name) + refNode: int | None = n + if refType is not None: + ups = L.u(n, otype=refType) + refNode = int(ups[0]) if ups else None + if refNode is None: + continue + value = self._value(refNode, name) + if value is None: + continue + item: dict[str, Any] = { + "name": name, + "value": value if isinstance(value, (int, str)) else str(value), + "kind": kind, + } + if refType is not None: + item["on"] = refType + if name in self.extra: + item["extra"] = True + result.append(item) + return result or None + + +def _parse_extra(features: Iterable[str]) -> tuple[list[str], dict[str, str]]: + names: list[str] = [] + indirect: dict[str, str] = {} + for spec in features or (): + if not spec: + continue + parts = str(spec).split(":", 1) + name = parts[-1] + if name not in names: + names.append(name) + if len(parts) > 1: + indirect[name] = parts[0] + return (names, indirect) + + +# public API + + +def render_node( + api: Api, + config: DisplayConfig, + n: int, + mode: str = PRETTY, + options: RenderOptions | None = None, + in_tuple: bool = False, +) -> RenderTree: + """Build the render tree of a node. + + Parameters + ---------- + api: Api + The loaded corpus. + config: DisplayConfig + Its display configuration. + n: int + The node to display. + mode: str + `pretty` (nested boxes with labels and features) + or `plain` (running text with nested highlights). + options: RenderOptions | None + See `RenderOptions`. + """ + if mode not in (PRETTY, PLAIN): + raise ValueError(f"mode must be '{PRETTY}' or '{PLAIN}', not '{mode}'") + options = options or RenderOptions() + pretty = mode == PRETTY + n = int(n) + fragments = unravel( + api, config, n, pretty=pretty, options=options, in_tuple=in_tuple + ) + dresser = _Dresser(api, config, pretty, options) + dresser.dress(fragments) + nType = api.F.otype.v(n) + return RenderTree( + node=n, + type=nType, + mode=mode, + fmt=dresser.fmt, + direction=config.ltr(dresser.fmt), + heading=section_ref( + api, + n, + sep1=config.section_sep1, + sep2=config.section_sep2, + lang=options.lang, + ), + fragments=fragments, + ) + + +def tuple_containers( + api: Api, + config: DisplayConfig, + tup: Iterable[int], + condense_type: str | None = None, +) -> list[int]: + """The nodes to display a result tuple in, e.g. the verses that contain it. + + Nodes of the condense type are their own container; smaller nodes are shown + in the container(s) of their first and last slot; bigger nodes are shown + as themselves. + """ + F = api.F + L = api.L + N = api.N + eOslots = api.E.oslots.s + fOtypeV = F.otype.v + slotType = F.otype.slotType + otypeRank = N.otypeRank + condenseType = condense_type or config.condense_type or slotType + condenseRank = otypeRank[condenseType] + + containers: set[int] = set() + for n in tup: + n = int(n) + nType = fOtypeV(n) + if nType == condenseType: + containers.add(n) + elif nType == slotType: + up = L.u(n, otype=condenseType) + if up: + containers.add(int(up[0])) + elif otypeRank[nType] < condenseRank: + slots = eOslots(n) + for s in (slots[0], slots[-1]): + up = L.u(int(s), otype=condenseType) + if up: + containers.add(int(up[0])) + else: + containers.add(n) + return [int(c) for c in N.sortNodes(containers)] + + +def render_tuple( + api: Api, + config: DisplayConfig, + tup: Iterable[int], + mode: str = PRETTY, + options: RenderOptions | None = None, + colors: dict[int, str] | None = None, +) -> list[RenderTree]: + """Render a result tuple. + + In pretty mode, the tuple is shown inside its containers (see + `tuple_containers`), with the tuple's nodes highlighted. In plain mode, + each node of the tuple is rendered on its own. + + Parameters + ---------- + colors: dict[int, str] | None + Highlight color per position in the tuple (1-based, as in query + templates). Positions without a color get the default highlight. + """ + options = options or RenderOptions() + tup = tuple(int(n) for n in tup) + colors = colors or {} + highlights = dict(options.highlights or {}) + for i, n in enumerate(tup): + highlights.setdefault(n, colors.get(i + 1, "")) + opts = RenderOptions(**{**options.__dict__, "highlights": highlights}) + + if mode == PLAIN: + return [render_node(api, config, n, PLAIN, opts, in_tuple=True) for n in tup] + return [ + render_node(api, config, c, PRETTY, opts) + for c in tuple_containers(api, config, tup, opts.condense_type) + ] diff --git a/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp.yaml b/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp.yaml new file mode 100644 index 0000000..1d855a8 --- /dev/null +++ b/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp.yaml @@ -0,0 +1,7 @@ +# Display settings for the Samaritan Pentateuch (DT-UCPH/sp), all versions. +# Its slots are signs, and its app config does not say that words are the +# units to show, hover over and click; without this, those are single letters. +# These lines can move into the corpus' app config; then this file can go. +typeDisplay: + word: + base: true diff --git a/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp@7.2.4.yaml b/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp@7.2.4.yaml new file mode 100644 index 0000000..d356364 --- /dev/null +++ b/libs/explorer/cfabric_browser/display_settings/DT-UCPH/sp@7.2.4.yaml @@ -0,0 +1,16 @@ +# Display settings for the Samaritan Pentateuch 7.2.4 (DT-UCPH/sp), on top of sp.yaml. +# The corpus' app config (DT-UCPH/sp app/config.yaml) does not describe the +# phrase and clause layers: 7.2.4 is the first version with clauses and with the +# phrase features typ and function (slots are signs since the earliest versions). +# These lines can move into that app config; then this file can be deleted. +# Clauses have no features in any version so far, so they get no label. +typeDisplay: + word: + features: lex + featuresBare: gloss + phrase: + label: '{typ} {function}' + clause_atom: + hidden: true + phrase_atom: + hidden: true diff --git a/libs/explorer/cfabric_browser/downloads.py b/libs/explorer/cfabric_browser/downloads.py new file mode 100644 index 0000000..0132ab9 --- /dev/null +++ b/libs/explorer/cfabric_browser/downloads.py @@ -0,0 +1,347 @@ +"""Downloading corpora that are not on this computer yet. + +The corpora come from their own releases on GitHub, the same files that +Text-Fabric downloads (`tf.advanced.repo`): a zip attached to a release, either +`complete.zip` (every data folder of the release, from `/` down) or a zip +of one data directory with only its feature files (`flat`). Each download is +pinned: its release, the zip's checksum, and its size. + +They are unpacked into the place where Text-Fabric keeps its downloads +(`~/text-fabric-data/github`), in Text-Fabric's layout and with its +`__checkout__.txt` files (release and commit), so that Text-Fabric takes them +for its own downloads and the library finds them like any other corpus. + +Unpacking is careful: + +* a zip is checked against its checksum before anything is unpacked; +* names that would leave the destination (`..`, absolute paths) or belong to + another organisation are refused; +* everything is unpacked into a folder next to the destination first, and + each data folder (a folder with a `__checkout__.txt`) is moved into place + only when all are unpacked; +* a data folder that is already there is left as it is: nothing on this + computer is overwritten. +""" + +from __future__ import annotations + +import shutil +import uuid +import zipfile +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any + +from cfabric_browser.jobs import BackgroundJobs +from cfabric_browser.net import ( + CHUNK, + CORPUS_ROOT, + DownloadError, + Fetch, + download_file, + fetch_to, +) + +ROOT = CORPUS_ROOT +CACHE = "~/.cfabric/downloads" +CHECKOUT = "__checkout__.txt" + +# called with a step ("download", "unpack"), the amount done and the total +Progress = Callable[[str, int, int], None] + + +@dataclass(frozen=True) +class Extra: + """A file that the release zip lacks, from the repository at the release, + e.g. the DSS's `app/config.yaml` (display settings).""" + + path: str # relative to the repository folder, e.g. app/config.yaml + url: str + sha256: str + + +@dataclass(frozen=True) +class Download: + """A corpus that can be downloaded.""" + + id: str # org/repo, as in the library + name: str # the short name, as on the command line + title: str + version: str # the data version, the folder under tf/ + release: str # the release tag on GitHub + commit: str # the release's commit, for __checkout__.txt + asset: str # the zip attached to the release + size: int # bytes to download + unpacked: int # bytes once unpacked + sha256: str + layout: str # "complete" (folders from / down) or "flat" (tf/ only) + terms: str # where its terms of use are + extras: tuple[Extra, ...] = field(default_factory=tuple) + # the licence of the data, where it is known (confirmed by the user) + licence: str | None = None + + @property + def org(self) -> str: + return self.id.split("/")[0] + + @property + def repo(self) -> str: + return self.id.split("/")[1] + + @property + def url(self) -> str: + return f"https://github.com/{self.id}/releases/download/{self.release}/{self.asset}" + + def data_dir(self, root: str | Path = ROOT) -> Path: + return Path(root).expanduser() / self.org / self.repo / "tf" / self.version + + +DOWNLOADS: dict[str, Download] = { + d.id: d + for d in ( + Download( + id="ETCBC/bhsa", + name="bhsa", + title="BHSA = Biblia Hebraica Stuttgartensia Amstelodamensis", + version="2021", + release="v1.8.1", + commit="b112c161cfd21eae403d51a2733740d8743460e7", + asset="complete.zip", + size=33_963_367, + unpacked=178_427_610, + sha256="8104fae1151c926cfcfd01f7e8a30a09af8c607546f14482990833b624b73168", + layout="complete", + terms="https://github.com/ETCBC/bhsa", + licence="CC BY-NC 4.0", + ), + Download( + id="DT-UCPH/sp", + name="sp", + title="The Samaritan Pentateuch", + version="7.2.4", + release="v7.2.4", + commit="6da31aa1e02478fb5eab77c6e51259aaa0446c8f", + asset="complete.zip", + size=3_860_569, + unpacked=21_520_139, + sha256="8d3f2be7ca035d87c1d22a5164a63b43c114b0da437b1f30a99183e9cbf299b8", + layout="complete", + terms="https://github.com/DT-UCPH/sp", + licence="CC BY-NC 4.0", + ), + Download( + id="ETCBC/dss", + name="dss", + title="Dead Sea Scrolls", + version="2.0.1", + release="v2.0.1", + commit="47ecc1738fad6e67df7d40a3e84d73e4a125e265", + asset="tf-2.0.1.zip", + size=29_981_816, + unpacked=166_080_661, + sha256="26a390ed4f089f4a683c0df475a32cdd6932a83ab383d9379ff6799752f90cba", + layout="flat", + terms="https://github.com/ETCBC/dss", + licence="CC BY-NC 4.0", + extras=( + Extra( + "app/config.yaml", + "https://raw.githubusercontent.com/ETCBC/dss/v2.0.1/app/config.yaml", + "360bb1e732a2ae5689749d5ba432045a535760004d892030fba9535c1ddfe746", + ), + ), + ), + ) +} + + +def find(name: str) -> Download | None: + """A download by its id (`ETCBC/bhsa`) or short name (`bhsa`), any case.""" + wanted = name.lower() + for d in DOWNLOADS.values(): + if wanted in (d.id.lower(), d.name): + return d + return None + + +def is_present(download: Download, root: str | Path = ROOT) -> bool: + """Whether the corpus' data directory is on this computer.""" + data = download.data_dir(root) + return (data / "otype.tf").is_file() or (data / ".cfm").is_dir() + + +def fetch( + download: Download, + cache: str | Path = CACHE, + progress: Progress | None = None, + fetcher: Fetch = fetch_to, +) -> tuple[Path, dict[str, Path]]: + """The release zip and the extra files, downloaded into `cache` (or taken + from there when their checksums match).""" + folder = Path(cache).expanduser() + folder.mkdir(parents=True, exist_ok=True) + stem = f"{download.org}-{download.repo}-{download.release}" + report = ( + (lambda done, total: progress("download", done, total or download.size)) + if progress + else (lambda done, total: None) + ) + archive = folder / f"{stem}-{download.asset}" + download_file(download.url, archive, download.sha256, fetcher, report) + extras = {} + for extra in download.extras: + target = folder / f"{stem}-{extra.path.replace('/', '-')}" + download_file( + extra.url, target, extra.sha256, fetcher, lambda done, total: None + ) + extras[extra.path] = target + return (archive, extras) + + +def _checkout(download: Download) -> str: + return f"{download.release}\ng{download.commit}\n" + + +def _members(download: Download, archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]: + """The files of the zip, after checking their names and total size.""" + files = [i for i in archive.infolist() if not i.is_dir()] + for info in files: + name = info.filename + parts = PurePosixPath(name).parts + if name.startswith(("/", "\\")) or "\\" in name or ":" in name or ".." in parts: + raise DownloadError( + f"{download.id}: the zip has a file outside its folder: {name}" + ) + if download.layout == "complete" and parts[0] != download.org: + raise DownloadError( + f"{download.id}: the zip has a file of another organisation: {name}" + ) + if download.layout == "flat" and len(parts) != 1: + raise DownloadError( + f"{download.id}: the zip has a folder, not only files: {name}" + ) + total = sum(i.file_size for i in files) + if total > 2 * download.unpacked: + raise DownloadError( + f"{download.id}: the zip unpacks to {total} bytes, far more than expected" + ) + return files + + +def install( + download: Download, + archive: Path, + extras: dict[str, Path] | None = None, + root: str | Path = ROOT, + progress: Progress | None = None, +) -> list[Path]: + """Unpack a downloaded corpus into `root`; returns the data folders put in + place (folders already there are left alone and not listed).""" + base = Path(root).expanduser() + base.mkdir(parents=True, exist_ok=True) + staging = base / f".download-{uuid.uuid4().hex[:8]}" + staging.mkdir() + try: + with zipfile.ZipFile(archive) as z: + files = _members(download, z) + prefix = ( + PurePosixPath() + if download.layout == "complete" + else PurePosixPath(download.org, download.repo, "tf", download.version) + ) + for i, info in enumerate(files): + target = staging.joinpath(*(prefix / info.filename).parts) + target.parent.mkdir(parents=True, exist_ok=True) + with z.open(info) as source, open(target, "wb") as out: + shutil.copyfileobj(source, out, CHUNK) + if progress is not None: + progress("unpack", i + 1, len(files)) + if download.layout == "flat": + data = staging / download.org / download.repo / "tf" / download.version + (data / CHECKOUT).write_text(_checkout(download), encoding="utf-8") + for path, source in (extras or {}).items(): + target = staging / download.org / download.repo / PurePosixPath(path) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + # the folder of an extra file is a data folder of its own (app/) + (target.parent / CHECKOUT).write_text(_checkout(download), encoding="utf-8") + + units = sorted(p.parent for p in staging.rglob(CHECKOUT)) + for unit in units: + if any(other != unit and other in unit.parents for other in units): + raise DownloadError( + f"{download.id}: a data folder inside another: {unit.relative_to(staging)}" + ) + if not units: + raise DownloadError(f"{download.id}: the zip has no data folders") + placed = [] + for unit in units: + destination = base / unit.relative_to(staging) + if destination.exists(): + continue + destination.parent.mkdir(parents=True, exist_ok=True) + unit.rename(destination) + placed.append(destination) + finally: + shutil.rmtree(staging, ignore_errors=True) + if not is_present(download, root): + raise DownloadError( + f"{download.id}: no data in tf/{download.version} after unpacking" + ) + return placed + + +def get( + download: Download, + root: str | Path = ROOT, + cache: str | Path = CACHE, + progress: Progress | None = None, + fetcher: Fetch = fetch_to, + keep: bool = False, +) -> Path: + """Download and unpack a corpus, unless it is there already; returns its + data directory. The downloaded files are removed afterwards, unless `keep`.""" + if not is_present(download, root): + (archive, extras) = fetch( + download, cache=cache, progress=progress, fetcher=fetcher + ) + install(download, archive, extras, root=root, progress=progress) + if not keep: + for path in (archive, *extras.values()): + path.unlink(missing_ok=True) + return download.data_dir(root) + + +class DownloadJobs(BackgroundJobs): + """Downloads running in the background, one at a time per corpus, with their + progress for the library to show.""" + + def __init__( + self, root: str | Path = ROOT, cache: str | Path = CACHE, **options: Any + ) -> None: + super().__init__(expected=(DownloadError,)) + self.root = root + self.cache = cache + self.options = options # e.g. fetcher=, for tests + + def start( + self, + download: Download, + then: Callable[[Progress], Any] | None = None, + ) -> dict[str, Any]: + """Download and unpack a corpus; then `then(progress)`, e.g. open and + prepare it (the first loading compiles it, which takes minutes).""" + + def work(progress: Progress) -> None: + get( + download, + root=self.root, + cache=self.cache, + progress=progress, + **self.options, + ) + if then is not None: + then(progress) + + return self.run(download.id, work) diff --git a/libs/explorer/cfabric_browser/formats.py b/libs/explorer/cfabric_browser/formats.py new file mode 100644 index 0000000..a4f6af2 --- /dev/null +++ b/libs/explorer/cfabric_browser/formats.py @@ -0,0 +1,53 @@ +"""The text formats the browser offers: the BHSA's vocalized and unvocalized +text, and only the full text of the other corpora. + +A corpus defines more formats (transliterations, lexemes, the ketiv, the +DSS' word-level and source formats); the browser leaves them out of its +display options, and a request for one of them (an old link, a saved +workspace) gets the corpus' default. The full list stays in the corpus' +`DisplayConfig`, because the comparison and the Concordance choose their +consonants from it. +""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +from cfabric_browser.display import DisplayConfig, RenderOptions + +# vocalized (vowels, accents, the qere) and unvocalized (consonants, the ketiv) +FORMATS = ("text-orig-full", "text-orig-plain") + + +def shown(config: DisplayConfig) -> list[str]: + """The formats of a corpus the browser offers, in the order of `FORMATS`.""" + return [f for f in FORMATS if f in config.text_formats] + + +def default(config: DisplayConfig) -> str: + """The format a corpus is shown in when none is chosen.""" + offered = shown(config) + if config.default_format in offered or not offered: + return str(config.default_format) + return offered[0] + + +def chosen(config: DisplayConfig, fmt: str | None) -> str | None: + """The format to use for a request: `fmt` if the browser offers it for + this corpus, else None (the default).""" + return fmt if fmt in shown(config) else None + + +def options(config: DisplayConfig, opts: RenderOptions) -> RenderOptions: + """Display options with a format the browser offers for this corpus.""" + fmt = chosen(config, opts.fmt) + return opts if fmt == opts.fmt else replace(opts, fmt=fmt) + + +def display(config: DisplayConfig) -> dict[str, Any]: + """`config.to_dict()` with only the formats offered.""" + d: dict[str, Any] = config.to_dict() + d["textFormats"] = shown(config) + d["defaultFormat"] = default(config) + return d diff --git a/libs/explorer/cfabric_browser/guard.py b/libs/explorer/cfabric_browser/guard.py new file mode 100644 index 0000000..531e3e6 --- /dev/null +++ b/libs/explorer/cfabric_browser/guard.py @@ -0,0 +1,86 @@ +"""Keeping other web sites out of the browser's server. + +The server listens on this computer, but any web page open in the user's web +browser can send it requests. Two checks stop that: + +* **Origin:** a request a web page sends to another site carries its own + origin; requests from the browser's own page carry the server's (or none, + for plain page loads). A foreign origin is refused, so another site can't + start a download or a build, or read the answers. +* **Host:** a site can make its own name point at 127.0.0.1 (DNS rebinding), + after which its requests look like the browser's own. When the server + listens only on this computer, it answers only to the names of this + computer (`127.0.0.1`, `localhost`, `[::1]`, on any port). +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from typing import Any +from urllib.parse import urlsplit + +LOCAL_HOSTS = ("127.0.0.1", "localhost", "[::1]") + +Scope = dict[str, Any] + + +def _host_name(host: str) -> str: + """`localhost:8107` -> `localhost`; `[::1]:8107` -> `[::1]`.""" + if host.startswith("["): + return host.split("]", 1)[0] + "]" + return host.rsplit(":", 1)[0] if ":" in host else host + + +class Guard: + """ASGI middleware that refuses requests from other sites. + + Parameters + ---------- + app: + The application to protect. + hosts: Iterable[str] | None + The host names (without port) the server answers to; `None` for any + (the server is reachable from other machines under names we can't + know). + """ + + def __init__(self, app: Any, hosts: Iterable[str] | None = LOCAL_HOSTS) -> None: + self.app = app + self.hosts = None if hosts is None else {h.lower() for h in hosts} + + async def __call__(self, scope: Scope, receive: Any, send: Any) -> None: + if scope["type"] == "http": + problem = self.problem(scope) + if problem is not None: + await _refuse(send, problem) + return + await self.app(scope, receive, send) + + def problem(self, scope: Scope) -> str | None: + headers = { + k.decode("latin-1").lower(): v.decode("latin-1") + for (k, v) in scope.get("headers", []) + } + host = headers.get("host", "") + if self.hosts is not None and _host_name(host).lower() not in self.hosts: + return "Unknown host" + origin = headers.get("origin") + if origin is not None and urlsplit(origin).netloc.lower() != host.lower(): + return "Requests from other sites are not allowed" + return None + + +async def _refuse(send: Any, message: str) -> None: + body = json.dumps({"detail": message}).encode() + await send( + { + "type": "http.response.start", + "status": 403, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode()), + ], + } + ) + await send({"type": "http.response.body", "body": body}) diff --git a/libs/explorer/cfabric_browser/helpers.py b/libs/explorer/cfabric_browser/helpers.py new file mode 100644 index 0000000..3ed5363 --- /dev/null +++ b/libs/explorer/cfabric_browser/helpers.py @@ -0,0 +1,51 @@ +"""Small helpers the browser needs, copied from the Python core +(`cfabric.utils.helpers`), so that the browser works with any core: the +Python one and the Rust one (0.6), which has no `cfabric.utils`.""" + +from __future__ import annotations + +import re +from collections.abc import Generator +from typing import Any + +NUM_ALFA_RE = re.compile(r"^([0-9]*)([^0-9]*)(.*)$") + + +def rangesFromList(nodeList: list[int]) -> Generator[tuple[int, int]]: + """Consecutive runs in a sorted list of nodes, as (first, last) pairs.""" + curstart: int = -1 + curend: int = -1 + started = False + for n in nodeList: + if not started: + curstart = n + curend = n + started = True + elif n == curend + 1: + curend = n + else: + yield (curstart, curend) + curstart = n + curend = n + if started: + yield (curstart, curend) + + +def tsvEsc(x: Any) -> str: + """A value for TSV data: a `"` or `'` at its start gets a backslash before + it (only there), as Text-Fabric does.""" + s = str(x) + return s if s == "" else f"\\{s}" if s[0] in {"'", '"'} else s + + +def versionSort(x: str) -> tuple[tuple[int, str, str], ...]: + """A sort key for version numbers such as `2021`, `c` or `1.10.2`.""" + parts: list[tuple[int, str, str]] = [] + for p in x.split("."): + match = NUM_ALFA_RE.match(p) + if match: + (num, alfa, rest) = match.group(1, 2, 3) + parts.append((int(num) if num else 0, alfa, rest)) + else: + parts.append((0, p, "")) + return tuple(parts) diff --git a/libs/explorer/cfabric_browser/interfaces.py b/libs/explorer/cfabric_browser/interfaces.py new file mode 100644 index 0000000..4b8a128 --- /dev/null +++ b/libs/explorer/cfabric_browser/interfaces.py @@ -0,0 +1,49 @@ +"""The browser's interfaces, and the settings that choose among them. + +The interface itself is chosen in the browser (and remembered there); the +server only says which one to start in and which ones to leave out of the menu: +`--ui NAME`, or in `~/.cfabric/browser.yaml` either `ui: parallel` or + + ui: + default: parallel + hide: [panes, concordance] +""" + +from __future__ import annotations + +from typing import Any + +# the ids the interface uses; Panes is "classic" there +UIS = ("classic", "parallel", "reader", "interlinear", "concordance") +ALIASES = {"panes": "classic"} + + +def ui_id(name: str) -> str: + """The id of an interface named in the settings (`panes` is `classic`).""" + key = str(name).strip().lower() + key = ALIASES.get(key, key) + if key not in UIS: + names = ", ".join(["panes", *UIS[1:]]) + raise ValueError(f"unknown interface {name!r}; choose from {names}") + return key + + +def read_ui_settings(value: Any) -> tuple[str | None, list[str]]: + """The default interface and the hidden ones from the `ui:` setting. + + Raises + ------ + ValueError + For an unknown interface or a setting of another shape. + """ + if value is None: + return (None, []) + if isinstance(value, str): + return (ui_id(value), []) + if isinstance(value, dict): + default = value.get("default") + hide = value.get("hide") or [] + if isinstance(hide, str): + hide = [hide] + return (ui_id(default) if default else None, [ui_id(h) for h in hide]) + raise ValueError("ui: should be an interface name, or default: and hide:") diff --git a/libs/explorer/cfabric_browser/interlinear.py b/libs/explorer/cfabric_browser/interlinear.py new file mode 100644 index 0000000..b81224a --- /dev/null +++ b/libs/explorer/cfabric_browser/interlinear.py @@ -0,0 +1,185 @@ +"""The interlinear: a passage word by word, with lines under each word. + +Per word: its text, and the values of the lines, read by role as on the card +shown when hovering over a word (`cfabric_browser.summary`): transliteration +(of the lexeme), lexeme, gloss, part of speech and morphology in words; and +the values of any other features asked for. Per verse also the phrases and +clauses its words are in, with their phrase function and clause type, for +showing the structure. Optionally a second corpus under the first, word by +word: the words that correspond, found as in the comparison +(`cfabric_browser.compare.align_words`), with their differences. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from cfabric_browser import formats +from cfabric_browser.compare import align_words, diff_words, verse_words +from cfabric_browser.display import RenderOptions, section_ref_for +from cfabric_browser.summary import summary +from cfabric_browser.words import word_type, words_of + +# the lines built from the summary, by its labels +MORPHOLOGY = ("stem", "tense", "person", "gender", "number", "state") +PHRASE_LABEL = "function" +CLAUSE_LABEL = "typ" + + +def _groups( + api, words: list[int], nType: str, feature: str +) -> tuple[list[int], list[dict[str, Any]]]: + """Per word the index of its group (the `nType` node it lies in, or -1), and + the groups: node and label (the value of `feature`).""" + if nType not in api.F.otype.all: + return ([-1] * len(words), []) + fLabel = api.Fs(feature) if feature in api.Fall() else None + index: dict[int, int] = {} + groups: list[dict[str, Any]] = [] + owners: list[int] = [] + for w in words: + ups = api.L.u(w, otype=nType) + if not ups: + owners.append(-1) + continue + node = int(ups[0]) + if node not in index: + index[node] = len(groups) + label = fLabel.v(node) if fLabel else None + groups.append( + {"node": node, "label": None if label is None else str(label)} + ) + owners.append(index[node]) + return (owners, groups) + + +def word_lines(items: list[dict[str, str]]) -> dict[str, str]: + """The lines of a word from its summary items.""" + by = {i["label"]: i["value"] for i in items} + lines: dict[str, str] = {} + for line, label in ( + ("translit", "transliteration"), + ("lexeme", "lexeme"), + ("gloss", "gloss"), + ("pos", "part of speech"), + ): + if label in by: + lines[line] = by[label] + morph = " ".join(by[m] for m in MORPHOLOGY if m in by) + if "suffix" in by: + morph = f"{morph} + {by['suffix']}" if morph else f"+ {by['suffix']}" + if morph: + lines["morph"] = morph + if "language" in by: + lines["language"] = by["language"] + return lines + + +def interlinear_verse( + service, + corpus: str, + verse: int, + opts: RenderOptions, + features: list[str], + other: str | None, + bdb_for: Callable[[int], str | None] | None = None, +) -> dict[str, Any]: + """A verse word by word; see the module. `bdb_for(word)` gives BDB's + short definition of a word, for the line `bdb`.""" + api = service.api(corpus) + config = service.config(corpus) + fmt = formats.chosen(config, opts.fmt) or formats.default(config) + empty = {str(v) for v in config.none_values if v is not None} + glosses = service.glosses(corpus) + wType = word_type(api, config) + words = words_of(api, wType, verse) + extra = [(f, api.Fs(f)) for f in features if f in api.Fall()] + + rows = [] + for w in words: + if glosses is None: + items = summary(api, w, empty) + else: + (gloss_for, source) = glosses + items = summary(api, w, empty, gloss_for, f"gloss in {source}") + values = {} + for name, feature in extra: + value = feature.v(w) + if value is not None and str(value) not in empty: + values[name] = str(value) + lines = word_lines(items) + definition = bdb_for(w) if bdb_for else None + if definition: + lines["bdb"] = definition + rows.append( + { + "node": w, + "text": api.T.text(w, fmt=fmt).strip(), + "lines": lines, + "features": values, + } + ) + + (phraseOf, phrases) = _groups(api, words, "phrase", PHRASE_LABEL) + (clauseOf, clauses) = _groups(api, words, "clause", CLAUSE_LABEL) + for row, p, c in zip(rows, phraseOf, clauseOf, strict=True): + row["phrase"] = p + row["clause"] = c + + ref = section_ref_for(api, config, verse, lang=opts.lang) + number = api.T.sectionFromNode(verse, lang="en")[-1] + result: dict[str, Any] = { + "verse": int(number) if str(number).isdigit() else str(number), + "ref": ref, + "words": rows, + "phrases": phrases, + "clauses": clauses, + } + if other: + result["under"] = under_verse(service, corpus, ref, verse, other, opts) + return result + + +def under_verse( + service, corpus: str, ref: str, verse: int, other: str, opts: RenderOptions +) -> dict[str, Any] | None: + """The verse in `other`, word by word under the words of `corpus`, with the + words that differ (`matres` or `variant`, as in the comparison). + + A translation is not paired word by word: it comes as the verse's text + (`translation`, `text`, and a `note` printed after it, if any).""" + aligned = service.align(corpus, ref, other, opts.lang) + if aligned is None or not aligned["exact"]: + return None + oApi = service.api(other) + oConfig = service.config(other) + if service.translation(other) is not None: + node = aligned["node"] + noteFeature = getattr(oApi.F, "note", None) + return { + "corpus": other, + "ref": aligned["ref"], + "translation": True, + "text": oApi.T.text(node, fmt=formats.default(oConfig)).strip(), + "note": noteFeature.v(node) if noteFeature else None, + "words": [], + "under": [], + "marks": {}, + "marksHere": {}, + } + a = verse_words(service.api(corpus), service.config(corpus), verse) + b = verse_words(oApi, oConfig, aligned["node"]) + fmt = formats.chosen(oConfig, opts.fmt) or formats.default(oConfig) + (marksA, marksB) = diff_words(a, b) + return { + "corpus": other, + "ref": aligned["ref"], + "words": [{"node": n, "text": oApi.T.text(n, fmt=fmt).strip()} for (n, _) in b], + # per word of the corpus read, the nodes of the words under it + "under": align_words(a, b), + # the words that differ, under and in the corpus read (node ids of two + # corpora may coincide, so they are kept apart) + "marks": {str(n): k for (n, k) in marksB.items()}, + "marksHere": {str(n): k for (n, k) in marksA.items()}, + } diff --git a/libs/explorer/cfabric_browser/jobs.py b/libs/explorer/cfabric_browser/jobs.py new file mode 100644 index 0000000..26eae24 --- /dev/null +++ b/libs/explorer/cfabric_browser/jobs.py @@ -0,0 +1,105 @@ +"""Work that runs in the background with progress for the interface to show: +building a translation, downloading a corpus. + +A job has a key (a translation id, a corpus id); one job per key runs at a +time. Its status is a small dict that the API returns as it is:: + + {"state": "running" | "done" | "failed" | "cancelled", "step": ..., "done": n, "total": m, "error": None} + +A job is cancelled at its next report of progress (`cancel`). +""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Callable +from typing import Any + +logger = logging.getLogger(__name__) + +# called by the work with a step, the amount done and the total +Progress = Callable[[str, int, int], None] + + +class JobCancelled(Exception): + """Raised inside a job when it has been cancelled.""" + + +class BackgroundJobs: + """Jobs in threads, with their progress and how they ended. + + `expected` are the exceptions whose message is meant for the user as it is + (a source that isn't what was expected); an `OSError` is reported as a + problem with writing or the network, and anything else as a failure. + """ + + def __init__(self, expected: tuple[type[Exception], ...] = ()) -> None: + self.expected = expected + self._lock = threading.Lock() + self._status: dict[str, dict[str, Any]] = {} + self._cancelled: set[str] = set() + + def status(self, key: str) -> dict[str, Any] | None: + with self._lock: + status = self._status.get(key) + return dict(status) if status else None + + def run( + self, key: str, work: Callable[[Progress], Any], first_step: str = "download" + ) -> dict[str, Any]: + """Start `work(progress)` in a thread, unless a job for `key` runs already.""" + with self._lock: + current = self._status.get(key) + if current and current["state"] == "running": + return dict(current) + status = { + "state": "running", + "step": first_step, + "done": 0, + "total": 0, + "error": None, + } + self._status[key] = status + self._cancelled.discard(key) + thread = threading.Thread(target=self._run, args=(key, work), daemon=True) + thread.start() + return dict(status) + + def cancel(self, key: str) -> bool: + """Stop a running job at its next report of progress; False if none runs.""" + with self._lock: + current = self._status.get(key) + if not current or current["state"] != "running": + return False + self._cancelled.add(key) + return True + + def _progress(self, key: str, step: str, done: int, total: int) -> None: + with self._lock: + # a step that has just finished (perhaps the last, after the work + # was put in place) is not stopped; the next one is + if key in self._cancelled and not (total > 0 and done >= total): + raise JobCancelled(key) + self._status[key].update(step=step, done=done, total=total) + + def _run(self, key: str, work: Callable[[Progress], Any]) -> None: + try: + work(lambda step, done, total: self._progress(key, step, done, total)) + except JobCancelled: + with self._lock: + self._status[key].update(state="cancelled", error=None) + self._cancelled.discard(key) + except self.expected as e: + self._finish(key, str(e)) + except OSError as e: + self._finish(key, f"Could not write it: {e}") + except Exception as e: + logger.exception("The job %s failed", key) + self._finish(key, f"It failed: {e}") + else: + self._finish(key, None) + + def _finish(self, key: str, error: str | None) -> None: + with self._lock: + self._status[key].update(state="failed" if error else "done", error=error) diff --git a/libs/explorer/cfabric_browser/lexicon.py b/libs/explorer/cfabric_browser/lexicon.py new file mode 100644 index 0000000..27bd215 --- /dev/null +++ b/libs/explorer/cfabric_browser/lexicon.py @@ -0,0 +1,225 @@ +"""The lexicon of a corpus with lexeme nodes (BHSA): its lexemes, and where they occur. + +A lexeme is a node of type `lex` whose words are the slots it contains. Its +features are read by role, from the first feature of that role the corpus has +(see `ROLES`): the BHSA has all of them. A corpus has a lexicon when it has +lexeme nodes with glosses; the DSS has lexeme nodes, but no glosses. +""" + +from __future__ import annotations + +import threading +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from cfabric_browser.display import ( + DisplayConfig, + RenderOptions, + render_node, + section_ref_for, +) +from cfabric_browser.words import BookSpans + +LEXEME_TYPE = "lex" +VERSE_CACHE = 32 # lexemes whose verses are kept + +# role: candidate features of a lexeme node, the first one present is used +ROLES: dict[str, tuple[str, ...]] = { + "lex": ("lex",), # transliteration, unique within a language: BR>[ + "form": ("voc_lex_utf8", "lex_utf8"), # shown: בָּרָא + "cons": ("lex_utf8",), # consonants, which give the first letter + "gloss": ("gloss",), + "sp": ("sp",), + "language": ("language",), + "freq": ("freq_lex",), + "rank": ("rank_lex",), + "nametype": ("nametype",), +} + +# the ETCBC transliteration in the order of the Hebrew alphabet; F (sin) and C +# (shin) are both ש, so they sort together +ALPHABET = ">BGDHWZXVJKLMNS bool: + """Whether a corpus has lexeme nodes with glosses.""" + F = api.F + if LEXEME_TYPE not in F.otype.all: + return False + gloss = getattr(F, "gloss", None) + if gloss is None: + return False + first = F.otype.s(LEXEME_TYPE) + return len(first) > 0 and gloss.v(int(first[0])) is not None + + +def sort_key(lex: str) -> tuple[int, ...]: + """Alphabetical order of a transliterated lexeme, in Hebrew order.""" + return tuple(ORDER[c] for c in lex if c in ORDER) + + +def first_letter(cons: str | None) -> str | None: + """The Hebrew letter a lexeme starts with (vowels and marks left out).""" + for c in cons or "": + if c in LETTERS: + return c + return None + + +@dataclass +class Lexicon: + api: Any + features: dict[str, str] # role: feature of the corpus + entries: list[dict[str, Any]] + byNode: dict[int, dict[str, Any]] + _verses: dict[tuple[int, str | None, str], list[tuple[int, list[int]]]] = field( + default_factory=dict + ) + _byLex: dict[tuple[Any, Any], dict[str, Any]] = field(default_factory=dict) + # the books as runs of slots, and their names per (book, language) + _spans: BookSpans | None = None + _bookNames: dict[tuple[int, str], str] = field(default_factory=dict) + # requests come in on several threads + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + @classmethod + def build(cls, api: Any) -> Lexicon | None: + if not has_lexicon(api): + return None + F = api.F + features = {} + for role, candidates in ROLES.items(): + for name in candidates: + if getattr(F, name, None) is not None: + features[role] = name + break + values = {role: getattr(F, name).v for role, name in features.items()} + entries = [] + for n in F.otype.s(LEXEME_TYPE): + n = int(n) + entry: dict[str, Any] = {"node": n} + for role, value in values.items(): + v = value(n) + entry[role] = int(v) if role in ("freq", "rank") and v else v + if "freq" not in features: + entry["freq"] = len(api.L.d(n, F.otype.slotType)) + entry["letter"] = first_letter(entry.get("cons") or entry.get("form")) + entries.append(entry) + entries.sort( + key=lambda e: (e.get("language") or "", sort_key(e.get("lex") or "")) + ) + return cls(api, features, entries, {e["node"]: e for e in entries}) + + def entry(self, node: int) -> dict[str, Any] | None: + return self.byNode.get(node) + + def gloss(self, lex: str, language: str | None = None) -> str | None: + """The gloss of a lexeme by its transliteration (`BR>[`) and language + (without one, Hebrew); if the lexeme is not in that language, as in + another one (the DSS marks some words Aramaic whose lexeme the BHSA only + has in Hebrew).""" + entry = self.lookup(lex, language) + return entry.get("gloss") if entry else None + + def lookup(self, lex: str, language: str | None = None) -> dict[str, Any] | None: + """The lexeme by its transliteration and language (without one, + Hebrew), else in another language, as `gloss` does.""" + return self.find(lex, language or "Hebrew") or self.find(lex, None) + + def find(self, lex: str, language: str | None) -> dict[str, Any] | None: + """The lexeme with this transliteration in this language (None: any).""" + with self._lock: + if not self._byLex: + for e in self.entries: + self._byLex.setdefault((e.get("lex"), e.get("language")), e) + self._byLex.setdefault((e.get("lex"), None), e) + return self._byLex.get((lex, language)) + + def words(self, node: int) -> list[int]: + """The words of a lexeme, in canonical order.""" + return [int(w) for w in self.api.L.d(node, self.api.F.otype.slotType)] + + def verses( + self, node: int, book: str | None = None, lang: str = "en" + ) -> list[tuple[int, list[int]]]: + """The verses a lexeme occurs in, each with its words there; with + `book`, only those in the book of that name (in language `lang`).""" + key = (node, book, lang) + with self._lock: + cached = self._verses.get(key) + if cached is not None: + return cached + api = self.api + verseType = api.T.sectionTypes[-1] + verses: dict[int, list[int]] = {} + for w in self.words(node): + if book is not None and self.book_of(w, lang) != book: + continue + up = api.L.u(w, verseType) + if up: + verses.setdefault(int(up[0]), []).append(w) + result = list(verses.items()) + # paging through a frequent lexeme asks for the same list again and again + with self._lock: + if len(self._verses) >= VERSE_CACHE: + self._verses.pop(next(iter(self._verses))) + self._verses[key] = result + return result + + def book_of(self, word: int, lang: str = "en") -> str: + """The name of the book a word is in; by its slot among the books' + slot ranges (asking `L.u` for each word is slow for frequent + lexemes).""" + api = self.api + slotType = api.F.otype.slotType + if self._spans is None: + self._spans = BookSpans(api, slotType) + book = self._spans.book(word) if api.F.otype.v(word) == slotType else None + if book is None: + return book_name(api, word, api.T.sectionTypes[0], lang) + name = self._bookNames.get((book, lang)) + if name is None: + name = str(api.T.sectionFromNode(book, lang=lang)[0]) + self._bookNames[(book, lang)] = name + return name + + def distribution(self, node: int, lang: str = "en") -> list[dict[str, Any]]: + """How often a lexeme occurs per book, in the order of the books.""" + counts: Counter[str] = Counter() + order: list[str] = [] + for w in self.words(node): + name = self.book_of(w, lang) + if name not in counts: + order.append(name) + counts[name] += 1 + return [{"book": name, "count": counts[name]} for name in order] + + +def book_name(api: Any, node: int, bookType: str, lang: str = "en") -> str: + up = api.L.u(node, bookType) + return str(api.T.sectionFromNode(up[0] if up else node, lang=lang)[0]) + + +def render_verse( + api: Any, + config: DisplayConfig, + verse: int, + words: list[int], + fmt: str | None = None, + lang: str = "en", +) -> dict[str, Any]: + """A verse as running text, with the words of a lexeme marked.""" + options = RenderOptions( + fmt=fmt, full=True, lang=lang, highlights={w: "" for w in words} + ) + return { + "node": verse, + "ref": section_ref_for(api, config, verse, lang=lang), + "words": words, + "tree": render_node(api, config, verse, "plain", options).to_dict(), + } diff --git a/libs/explorer/cfabric_browser/net.py b/libs/explorer/cfabric_browser/net.py new file mode 100644 index 0000000..f91efb4 --- /dev/null +++ b/libs/explorer/cfabric_browser/net.py @@ -0,0 +1,104 @@ +"""Downloading: certificates, requests under the browser's name, checksums. + +Shared by the corpus downloads (`downloads.py`), the translations +(`translations/`) and the dictionaries (`dictionaries/`). +""" + +from __future__ import annotations + +import hashlib +import http.client +import ssl +import urllib.request +from collections.abc import Callable +from pathlib import Path + +from cfabric_browser import __version__ + +# where Text-Fabric keeps its downloads, and the library looks first +CORPUS_ROOT = "~/text-fabric-data/github" +AGENT = f"cfabric-browser/{__version__}" +CHUNK = 1 << 16 + +# reports (bytes done, bytes in all; 0 when unknown) +Report = Callable[[int, int], None] +# fetches a URL into a file, reporting as it goes +Fetch = Callable[[str, Path, Report], None] + + +class DownloadError(Exception): + """A download that failed or is not what was expected; the message says why.""" + + +def ssl_context() -> ssl.SSLContext: + """The system's certificates, or certifi's where it is installed: the + standalone program on macOS has none of its own.""" + try: + import certifi + except ImportError: + return ssl.create_default_context() + return ssl.create_default_context(cafile=certifi.where()) + + +def read_url(url: str, agent: str = AGENT) -> bytes: + """The contents of a URL.""" + request = urllib.request.Request(url, headers={"User-Agent": agent}) + with urllib.request.urlopen(request, timeout=60, context=ssl_context()) as r: + data: bytes = r.read() + return data + + +def fetch_to(url: str, target: Path, report: Report) -> None: + """Download a URL into a file, in chunks.""" + request = urllib.request.Request(url, headers={"User-Agent": AGENT}) + with urllib.request.urlopen(request, timeout=60, context=ssl_context()) as response: + total = int(response.headers.get("Content-Length") or 0) + done = 0 + with open(target, "wb") as out: + while chunk := response.read(CHUNK): + out.write(chunk) + done += len(chunk) + report(done, total) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + digest.update(chunk) + return digest.hexdigest() + + +def download_file( + url: str, target: Path, sha256: str, fetch: Fetch, report: Report +) -> None: + """Download into `target`, unless a file with the right checksum is there; + tried twice, then the error is passed on. Nothing is left of a download + that fails, is cancelled or has another checksum.""" + if target.is_file() and sha256_file(target) == sha256: + return + partial = target.with_name(target.name + ".part") + try: + for attempt in (1, 2): + try: + fetch(url, partial, report) + break + # IncompleteRead (a connection cut off) is no OSError + except (OSError, http.client.HTTPException) as e: + if attempt == 2: + raise DownloadError(f"Could not download {url}: {e}") from e + except BaseException: + partial.unlink(missing_ok=True) + raise + found = sha256_file(partial) + if found != sha256: + partial.unlink(missing_ok=True) + raise DownloadError( + f"The download of {url} is not the expected file " + f"(checksum {found[:12]}…, expected {sha256[:12]}…)" + ) + partial.replace(target) diff --git a/libs/explorer/cfabric_browser/queries.py b/libs/explorer/cfabric_browser/queries.py new file mode 100644 index 0000000..b2568ac --- /dev/null +++ b/libs/explorer/cfabric_browser/queries.py @@ -0,0 +1,377 @@ +"""Checking search templates and reporting problems as structured data. + +The browser and (later) language models both need errors they can act on: +which line, what is wrong, and which feature or type is involved. The search +engine reports problems as `(line, message)` pairs with 0-based lines; here +they become dictionaries with 1-based lines. + +The browser works with either core. With the Python one, a template is +checked by running only the engine's cheap phases (parsing and preparing, +not the search); the Rust one (0.6) checks it with `S.study`, which is fast +there. The features per atom of a template (for exports and statistics) come +from parsing the template here (`template_atoms`), the same for both. +""" + +from __future__ import annotations + +import logging +import re +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +try: # the Python core's engine + from cfabric.search.searchexe import SearchExe +except ImportError: # the Rust core (0.6) has none + SearchExe = None + +LANGUAGES = ("cf-template",) +"""Query languages the browser accepts. A natural-language mode will later be +translated into `cf-template` before searching.""" + +MISSING_FEATURE = re.compile(r'Missing feature "([^"]+)" in line\(s\) ([\d, ]+)') +UNKNOWN_TYPE = re.compile(r'Unknown object type: "([^"]+)"') +TEMPLATE_ECHO = re.compile(r"^\s*\d+ ") +QUANTIFIER_ERROR = re.compile(r"^line\s+(\d+): Error under (/\w+/):") +LINE_MESSAGE = re.compile(r"^line \d+:") + + +def _structure(kind: str, line: int | None, message: str) -> list[dict[str, Any]]: + """One engine message as one or more error records.""" + m = MISSING_FEATURE.search(message) + if m: + lines = [int(x) for x in re.findall(r"\d+", m.group(2))] + return [ + { + "kind": kind, + "line": ln + 1, + "message": f'Unknown feature "{m.group(1)}"', + "feature": m.group(1), + } + for ln in lines + ] + record: dict[str, Any] = { + "kind": kind, + "line": None if line is None else int(line) + 1, + "message": message, + } + m = UNKNOWN_TYPE.search(message) + if m: + record["nodeType"] = m.group(1) + return [record] + + +def check_template( + api, template: str, sets: dict[str, set[int]] | None = None +) -> list[dict[str, Any]]: + """Check a template without searching; return its problems. + + Only the cheap phases run: parsing, checking types, features and names, + and checking that the template is connected. Problems inside quantifiers + surface only when searching; the search worker reports them in the same + form (see `problems_of`). + + Each problem is a dict with `kind` (`syntax` or `semantics`), `line` + (1-based, or `None` for general remarks), `message`, and where it applies + `feature` or `nodeType`. An empty list means the template is fine. + """ + if not template.strip(): + return [{"kind": "syntax", "line": None, "message": "Empty search template"}] + if SearchExe is None: + return study(api, template, sets) + + exe = SearchExe( + api, + template, + outerTemplate=template, + quKind=None, + offset=0, + sets=sets, + shallow=False, + silent="deep", + setInfo={}, + ) + with capture_engine_messages() as messages: + exe._parse() + exe._prepare() + return problems_of(exe, messages) + + +def template_constraints(api, template: str) -> list[list[tuple[str, bool]]]: + """The node features mentioned per atom of a template, in result-tuple order. + + Each feature comes with whether the template fixes it to a single value + (`sp=verb`) rather than leaving it open (`lex*`, `sp=verb|subs`, + `gloss~a`, `sp#verb`). Atoms inside quantifiers are not part of results + and are left out; so are edge features, and features the corpus doesn't + have. + """ + nodeFeatures = set(api.Fall()) + return [ + [ + (name, fixed) + for (name, fixed) in atom + if name in nodeFeatures and name != "otype" + ] + for atom in template_atoms(template) + ] + + +def template_features(api, template: str) -> list[list[str]]: + """The node features mentioned per atom of a template, in result-tuple order. + + `clause typ*` mentions `typ`, `word sp=verb gloss~a` mentions `sp` and + `gloss`. As in Text-Fabric, mentioning a feature with `*` (any value) is the + way to get it into exports without restricting the search. + """ + return [ + [name for (name, _) in atom] for atom in template_constraints(api, template) + ] + + +def problems_of( + exe: SearchExe, messages: list[str] | None = None +) -> list[dict[str, Any]]: + """The problems recorded on a search execution, as structured records. + + Some problems (such as a template that falls apart into unconnected parts) + are only logged by the engine, not recorded; pass the messages captured + with `capture_engine_messages` to include them. + """ + if exe.good: + return [] + problems: list[dict[str, Any]] = [] + for kind, attr in (("syntax", "badSyntax"), ("semantics", "badSemantics")): + for line, message in getattr(exe, attr, None) or (): + problems.extend(_structure(kind, line, message)) + if not problems: + for message in messages or (): + if TEMPLATE_ECHO.match(message) or LINE_MESSAGE.match(message): + continue + problems.append({"kind": "semantics", "line": None, "message": message}) + if not problems: + problems.append( + {"kind": "semantics", "line": None, "message": "Invalid search template"} + ) + return problems + + +def quantifier_problems(messages: list[str]) -> list[dict[str, Any]]: + """Problems inside quantifiers, found in the engine's captured messages. + + The engine (like Text-Fabric) reports an error inside a quantifier such as + `/without/` only in its log, and then carries on as if the quantifier did + not exclude anything. That gives wrong results without warning, so the + browser treats these messages as errors. Lines refer to the quantifier's + line in the full template; the engine's own line numbers inside the + quantifier refer to a sub-template and are dropped. + """ + problems: list[dict[str, Any]] = [] + context: tuple[int, str] | None = None + for message in messages: + m = QUANTIFIER_ERROR.match(message) + if m: + context = (int(m.group(1)) + 1, m.group(2)) + continue + if context is None or TEMPLATE_ECHO.match(message): + continue + (line, kind) = context + for problem in _structure("semantics", None, message): + problem["line"] = line + problem["message"] = f"{problem['message']} (inside {kind})" + problem["quantifier"] = kind + problems.append(problem) + return problems + + +class _ThreadCapture(logging.Handler): + def __init__(self, thread: int) -> None: + super().__init__(level=logging.ERROR) + self.thread = thread + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + if record.thread == self.thread: + for line in record.getMessage().splitlines(): + if line.strip(): + self.messages.append(line.rstrip()) + + +@contextmanager +def capture_engine_messages() -> Iterator[list[str]]: + """Collect the engine's error messages of this thread instead of printing them. + + Other threads keep logging as usual. + """ + thread = threading.get_ident() + cfLogger = logging.getLogger("cfabric") + capture = _ThreadCapture(thread) + silence = logging.Filter() + silence.filter = lambda record: record.thread != thread # type: ignore[method-assign] + handlers = list(cfLogger.handlers) + for handler in handlers: + handler.addFilter(silence) + cfLogger.addHandler(capture) + try: + yield capture.messages + finally: + cfLogger.removeHandler(capture) + for handler in handlers: + handler.removeFilter(silence) + + +# parsing templates, only as far as the features of each atom + +QUANTIFIER = re.compile(r"^/(without|where|have|with|or|-)/$") +OPENING = {"without", "where", "with"} +ATOM_HEAD = re.compile(r"^(?:[A-Za-z_]\w*:)?[A-Za-z_][\w.]*$") +FEATURE = re.compile(r"^([A-Za-z_][\w@-]*)(\*|[=#~<>].*)?$") + + +def _tokens(line: str) -> list[str]: + r"""The words of a line; `\ ` is a space inside a value.""" + return [t.replace("\0", " ") for t in line.replace("\\ ", "\0").split()] + + +def template_atoms(template: str) -> list[list[tuple[str, bool]]]: + """Per atom of a template outside quantifiers, in order, the features it + mentions with whether it fixes them to one value. + + Comments (`%`), quantifier blocks (`/without/` … `/-/`), relation lines + between named atoms (`p1 < p2`) and feature comparisons (`.f=g.`) are + left out; an atom may start with a relation (`<: word`) and a name + (`w1:word`). A feature written without a value (`gloss`) or with `*` + is open, as are `#`, `~`, `<` and `>`; `=` with one value (no `|`) is + fixed. + """ + names: set[str] = set() + depth = 0 + atoms: list[list[tuple[str, bool]]] = [] + for raw in template.splitlines(): + line = raw.strip() + if not line or line.startswith("%"): + continue + q = QUANTIFIER.match(line) + if q: + if q.group(1) in OPENING: + depth += 1 + elif q.group(1) == "-": + depth = max(0, depth - 1) + continue + tokens = _tokens(line) + if len(tokens) == 3 and tokens[0] in names and tokens[2] in names: + continue # a relation between named atoms + if tokens and not ATOM_HEAD.match(tokens[0]): + tokens = tokens[1:] # a relation before the atom + if not tokens or not ATOM_HEAD.match(tokens[0]): + continue + if ":" in tokens[0]: + names.add(tokens[0].split(":", 1)[0]) + if depth: + continue + features: list[tuple[str, bool]] = [] + seen: set[str] = set() + for token in tokens[1:]: + if token.startswith("."): + continue # comparing features of two atoms + m = FEATURE.match(token) + if not m or m.group(1) in seen: + continue + seen.add(m.group(1)) + spec = m.group(2) or "" + features.append((m.group(1), spec.startswith("=") and "|" not in spec)) + atoms.append(features) + return atoms + + +_studyLocks: dict[int, threading.Lock] = {} +_studyGuard = threading.Lock() + + +def study( + api, template: str, sets: dict[str, set[int]] | None = None +) -> list[dict[str, Any]]: + """Check a template with the core's own `S.study` (the Rust core): its + problems, as `check_template` gives them. One study at a time per corpus, + since `S` keeps the template it studied.""" + with _studyGuard: + lock = _studyLocks.setdefault(id(api), threading.Lock()) + with lock, capture_engine_messages() as messages: + S = api.S + try: + S.study(template, sets=sets, silent="deep") + except ValueError as e: # the Rust core's SearchTemplateError is one + return _from_error(e) + exe = S.exe + if exe is None or exe.good: + return [] + # problems inside a quantifier come as "Error under /without/:" + # followed by the problems themselves, as the Python engine logs them + semantics = list(exe.badSemantics) + if any(str(m).startswith("Error under /") for (_, m) in semantics): + lines = [ + f"line {line}: {m}" if str(m).startswith("Error under /") else str(m) + for (line, m) in semantics + ] + nested = quantifier_problems(lines) + if nested: + return nested + problems = problems_of(exe, messages) + generic = [ + {"kind": "semantics", "line": None, "message": "Invalid search template"} + ] + if problems == generic and not exe.badSyntax and not semantics: + return unconnected(template) or problems + return problems + + +def unconnected(template: str) -> list[dict[str, Any]]: + """For an engine that rejects a template without saying why: whether its + objects at the top level are related. Several of them without a relation + between them can't be searched (Text-Fabric calls that not connected).""" + names: set[str] = set() + tops = 0 + relations = 0 + depth = 0 + for raw in template.splitlines(): + line = raw.strip() + if not line or line.startswith("%"): + continue + q = QUANTIFIER.match(line) + if q: + depth += 1 if q.group(1) in OPENING else -1 if q.group(1) == "-" else 0 + continue + tokens = _tokens(line) + if len(tokens) == 3 and tokens[0] in names and tokens[2] in names: + relations += 1 + continue + if tokens and ":" in tokens[0] and ATOM_HEAD.match(tokens[0]): + names.add(tokens[0].split(":", 1)[0]) + if not depth and raw[: len(raw) - len(raw.lstrip())] == "": + tops += 1 + if tops > 1 and relations == 0: + return [ + { + "kind": "semantics", + "line": None, + "message": "The template is not connected: its objects at the " + "top level have no relation between them; put them inside one " + "object, or relate them by name (p1 < p2)", + } + ] + return [] + + +def _from_error(error: Exception) -> list[dict[str, Any]]: + """An engine's exception as problems: one per line of its message.""" + problems: list[dict[str, Any]] = [] + for text in str(error).splitlines() or [repr(error)]: + m = re.match(r"^\s*line\s+(\d+):\s*(.*)$", text) + if m: + problems.extend(_structure("syntax", int(m.group(1)) - 1, m.group(2))) + elif text.strip(): + problems.extend(_structure("syntax", None, text.strip())) + return problems or [ + {"kind": "syntax", "line": None, "message": "Invalid search template"} + ] diff --git a/libs/explorer/cfabric_browser/registry.py b/libs/explorer/cfabric_browser/registry.py new file mode 100644 index 0000000..e485552 --- /dev/null +++ b/libs/explorer/cfabric_browser/registry.py @@ -0,0 +1,183 @@ +"""Thread-safe registry of named corpora. + +A `CorpusRegistry` holds several corpora at once, each under a short name +(e.g. `bhsa`, `dss`), for the browser's server and its search workers. Moved +here from the core (it was only on the browser's branches), so that the +browser works with any core: the Python one and the Rust one (0.6). + +Corpora are registered with `register()` and loaded on first use, at most +once per corpus, also when several threads ask for the same corpus at the same +time. `get`, `get_api`, `list_corpora` and `current` are also what the tools of +`cfabric-mcp` ask of a registry (`cfabric_mcp.tools.using`). +""" + +from __future__ import annotations + +import logging +import threading +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from cfabric import Fabric + +if TYPE_CHECKING: + from cfabric.core.api import Api + +logger = logging.getLogger(__name__) + + +@dataclass +class CorpusEntry: + """A registered corpus and, once loaded, its Fabric and Api.""" + + name: str + path: Path + features: str | None = None + fabric: Fabric | None = None + api: Api | None = None + load_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + @property + def loaded(self) -> bool: + return self.api is not None + + +class CorpusRegistry: + """Holds named corpora and loads each at most once. + + Examples + -------- + >>> registry = CorpusRegistry() + >>> registry.register("~/text-fabric-data/github/ETCBC/bhsa/tf/2021", "bhsa") + >>> api = registry.get_api("bhsa") # loads on first use + """ + + def __init__(self) -> None: + self._lock = threading.RLock() + self._entries: dict[str, CorpusEntry] = {} + self._current: str | None = None + + # registration and loading + + def register( + self, + path: str | Path, + name: str | None = None, + features: str | list[str] | None = None, + ) -> str: + """Register a corpus without loading it. + + Parameters + ---------- + path: str | Path + Directory with the corpus data (`.tf` files and/or a `.cfm` cache). + name: str | None + Name for the corpus. Defaults to the directory name. + features: str | list[str] | None + Features to load. Defaults to all features. + + Returns + ------- + str + The name under which the corpus is registered. + + Raises + ------ + FileNotFoundError + If the path does not exist. + ValueError + If the name is already registered for a different path. + """ + pathObj = Path(path).expanduser().resolve() + if not pathObj.exists(): + logger.error("Corpus path not found: %s", path) + raise FileNotFoundError(f"Corpus path not found: {path}") + + name = name or pathObj.name + if isinstance(features, list): + features = " ".join(features) + + with self._lock: + existing = self._entries.get(name) + if existing is not None: + if existing.path != pathObj: + raise ValueError( + f"Corpus name '{name}' is already used for {existing.path}" + ) + return name + self._entries[name] = CorpusEntry(name, pathObj, features or None) + if self._current is None: + self._current = name + logger.info("Registered corpus '%s' at %s", name, pathObj) + return name + + def _entry(self, name: str | None) -> CorpusEntry: + with self._lock: + name = name or self._current + if not name: + logger.error("Attempted to access corpus but none loaded") + raise RuntimeError("No corpus loaded") + entry = self._entries.get(name) + if entry is None: + logger.error("Corpus not found: %s", name) + raise KeyError(f"Corpus not found: {name}") + return entry + + def _ensure_loaded(self, name: str) -> CorpusEntry: + entry = self._entry(name) + if entry.loaded: + return entry + with entry.load_lock: + if entry.loaded: + return entry + logger.info("Loading corpus '%s' from %s", name, entry.path) + fabric = Fabric(locations=str(entry.path), silent="deep") + if entry.features: + api = fabric.load(entry.features, silent="deep") + else: + api = fabric.loadAll(silent="deep") + # the Python core gives False for a corpus it could not load + if api is not None and api is not False: + entry.fabric = fabric + entry.api = api + return entry + logger.error("Failed to load corpus from %s", entry.path) + raise RuntimeError(f"Failed to load corpus from {entry.path}") + + # access + + def get(self, name: str | None = None) -> tuple[Fabric, Api]: + """The Fabric and Api of a corpus, loading it if needed. + + Parameters + ---------- + name: str | None + Corpus name. Defaults to the current corpus. + + Raises + ------ + RuntimeError + If no name is given and there is no current corpus. + KeyError + If the corpus is not registered. + """ + entry = self._ensure_loaded(self._entry(name).name) + assert entry.fabric is not None and entry.api is not None + return (entry.fabric, entry.api) + + def get_api(self, name: str | None = None) -> Api: + """The Api of a corpus, loading it if needed.""" + return self.get(name)[1] + + # bookkeeping + + def list_corpora(self) -> list[str]: + """Names of the loaded corpora.""" + with self._lock: + return [n for (n, e) in self._entries.items() if e.loaded] + + @property + def current(self) -> str | None: + """The name of the current (default) corpus.""" + return self._current diff --git a/libs/explorer/cfabric_browser/routes/__init__.py b/libs/explorer/cfabric_browser/routes/__init__.py new file mode 100644 index 0000000..e90ae73 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/__init__.py @@ -0,0 +1,59 @@ +"""The browser's JSON API, all routes under `/api`. + +Endpoints mirror the MCP tools of `cfabric-mcp` where they overlap (corpus +description, features, passages, search with paging), so that people in the +browser and language models through MCP work with the same operations. + +All corpus-specific routes are under `/api/corpora/{corpus}`; searches have +their own ids under `/api/searches/{id}`. Nothing in the API exposes paths on +the server's file system. + +One module per area; each builds its routes on the shared `Context`. +""" + +from __future__ import annotations + +from fastapi import APIRouter + +from cfabric_browser.assistant.tools import Tools +from cfabric_browser.corpora import CorpusService +from cfabric_browser.dictionaries.manage import DictionaryJobs +from cfabric_browser.dictionaries.store import Dictionaries +from cfabric_browser.downloads import DownloadJobs +from cfabric_browser.routes import ( + lexicon, + parallel, + proposals, + reading, + search, +) +from cfabric_browser.routes import library as library_routes +from cfabric_browser.routes.common import Context +from cfabric_browser.translations.manage import Jobs +from cfabric_browser.workers import SearchPool + + +def build_router( + service: CorpusService, + pool: SearchPool, + library: bool = True, + translations: Jobs | None = None, + downloads: DownloadJobs | None = None, + dictionaries: DictionaryJobs | None = None, + assistant: Tools | None = None, +) -> APIRouter: + lookups = dictionaries if dictionaries is not None else DictionaryJobs() + ctx = Context( + service=service, + pool=pool, + library=library, + tools=assistant if assistant is not None else Tools(service, pool), + lookups=lookups, + store=Dictionaries(lookups.root), + jobs=translations if translations is not None else Jobs(service.roots[0]), + fetches=downloads if downloads is not None else DownloadJobs(service.roots[0]), + ) + router = APIRouter(prefix="/api") + for area in (library_routes, reading, parallel, search, proposals, lexicon): + router.include_router(area.build(ctx)) + return router diff --git a/libs/explorer/cfabric_browser/routes/common.py b/libs/explorer/cfabric_browser/routes/common.py new file mode 100644 index 0000000..40612a9 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/common.py @@ -0,0 +1,234 @@ +"""What the routes of all areas share: request models, display options, +the shared state (`Context`) and helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Literal + +from fastapi import HTTPException, Query +from fastapi.responses import Response +from pydantic import BaseModel, Field + +from cfabric_browser.assistant.tools import Tools +from cfabric_browser.corpora import CorpusService +from cfabric_browser.dictionaries.manage import DictionaryJobs +from cfabric_browser.dictionaries.store import Dictionaries +from cfabric_browser.display import ( + RenderOptions, + render_node, + section_ref_for, +) +from cfabric_browser.downloads import DownloadJobs +from cfabric_browser.queries import LANGUAGES +from cfabric_browser.translations.manage import Jobs +from cfabric_browser.witnesses import dress_signs, place +from cfabric_browser.workers import SearchJob, SearchPool + +MAX_PAGE = 500 +MAX_DOWN = 200 +MAX_PASSAGE = 200 +STATS_SAMPLE = 300 + + +class OpenRequest(BaseModel): + id: str = Field( + description="A corpus id from the library, e.g. ETCBC/dss, optionally with" + " a version: DT-UCPH/sp@7.2.4 or DT-UCPH/sp@latest" + ) + name: str | None = Field(default=None, description="Name to show it under") + + +class ProposalRequest(BaseModel): + """A search proposed by a language model.""" + + corpus: str + template: str + request: str = Field(description="The user's request in their own words") + explanation: str = "" + model: str | None = None + + +class ProposalState(BaseModel): + state: Literal["new", "run", "dismissed"] + + +class SearchRequest(BaseModel): + template: str + language: str = Field(default="cf-template", description=f"One of {LANGUAGES}") + limit: int | None = Field(default=None, ge=1) + timeout: float | None = Field(default=None, gt=0) + + +def split_items(value: str | None) -> list[str] | None: + if value is None: + return None + return [x for x in value.replace(",", " ").split() if x] + + +def render_options( + fmt: str | None = None, + full: bool = False, + hide_types: bool = True, + hidden_types: str | None = Query(None, description="Comma separated types"), + base_types: str | None = Query(None, description="Comma separated types"), + condense_type: str | None = None, + extra_features: str | None = Query( + None, description="Comma separated features, `type:feature` allowed" + ), + lang: str = "en", +) -> RenderOptions: + """Display options as query parameters; see `cfabric_browser.display.RenderOptions`.""" + return RenderOptions( + fmt=fmt or None, + full=full, + hide_types=hide_types, + hidden_types=split_items(hidden_types), + base_types=split_items(base_types), + condense_type=condense_type or None, + extra_features=split_items(extra_features) or (), + lang=lang, + ) + + +@dataclass +class Context: + """What the routes of all areas share.""" + + service: CorpusService + pool: SearchPool + # whether people may open other corpora found on this computer + library: bool + # searching in plain language: the model's tools, and its proposals + tools: Tools + # dictionaries (BDB): downloaded into ~/.cfabric/dictionaries, read when + # first needed + lookups: DictionaryJobs + store: Dictionaries + # translations are built, and corpora downloaded, into the first place + # the library looks + jobs: Jobs + fetches: DownloadJobs + + def corpus_api(self, corpus: str): + if not self.service.has(corpus): + raise HTTPException(404, f"Unknown corpus: {corpus}") + return self.service.api(corpus) + + @staticmethod + def check_node(api, node: int) -> int: + if not 1 <= node <= int(api.F.otype.maxNode): + raise HTTPException(404, f"No node {node}") + return node + + def search_job(self, search_id: str) -> SearchJob: + job = self.pool.get(search_id) + if job is None: + raise HTTPException(404, f"Unknown or expired search: {search_id}") + return job + + def allowed(self, corpusId: str) -> bool: + """Whether a corpus may be offered (`only` limits the library).""" + only = {o.lower() for o in self.service.only} + return not only or corpusId.lower() in only + + +def witness_row(api, config, index, w, mode: str, opts: RenderOptions) -> dict: + """The text of one verse in one scroll, for the book view.""" + # a witness links to its fragment in the scroll view, for context + types = list(api.T.sectionTypes or []) + placeType = types[1] if len(types) > 1 else (types[0] if types else None) + fragments: list[dict[str, Any]] = [] + tree: dict[str, Any] = {"fmt": None, "direction": config.direction} + for word in w.words: + tree = render_node(api, config, word, mode, opts).to_dict() + fragments.extend(tree["fragments"]) + dress_signs(api, fragments) + where = api.L.u(w.words[0], otype=placeType)[0] if placeType else None + return { + "scroll": w.scroll, + "place": place(api, w.words), + "ref": section_ref_for(api, config, int(where), lang=opts.lang) + if where + else None, + "words": w.words, + "alsoNonBiblical": w.also_non_biblical, + "tree": { + "node": w.words[0], + "type": index.word_type, + "mode": mode, + "fmt": tree["fmt"], + "direction": tree["direction"], + "heading": f"{w.scroll} {place(api, w.words)}", + "fragments": fragments, + }, + } + + +def tsv_response(content: str, filename: str, excel: bool) -> Response: + """Tab-separated values as a download, as UTF-8 or as UTF-16 for Excel.""" + return Response( + content.encode("utf-16") if excel else content.encode("utf-8"), + media_type="text/tab-separated-values; charset=" + + ("utf-16" if excel else "utf-8"), + headers={"Content-Disposition": f'attachment; filename="{filename}.tsv"'}, + ) + + +def section_item(api, config, node: int, lang: str) -> dict[str, Any]: + """A section node as shown in navigation: its label and full reference.""" + T = api.T + section = T.sectionFromNode(node, lang=lang) + nType = api.F.otype.v(node) + item = { + "node": node, + "type": nType, + "label": str(section[-1]) if section else str(node), + "ref": section_ref_for(api, config, node, lang=lang), + } + # the feature that names it, with its own value, for search templates: the + # BHSA's book feature says Jesaia where the label (book@en) says Isaiah + types = list(T.sectionTypes or ()) + feats = list(T.sectionFeats or ()) + if nType in types and types.index(nType) < len(feats): + feature = feats[types.index(nType)] + value = api.Fs(feature).v(node) if api.Fs(feature) is not None else None + if value is not None: + item["feature"] = feature + item["value"] = str(value) + return item + + +def section_path(api, config, node: int, lang: str) -> list[dict[str, Any]]: + """The sections a section node lies in, from the top down, ending with itself.""" + F, L, T = api.F, api.L, api.T + nType = F.otype.v(node) + path = [] + for sType in T.sectionTypes: + if sType == nType: + path.append(section_item(api, config, node, lang)) + break + ups = L.u(node, otype=sType) + if ups: + path.append(section_item(api, config, int(ups[0]), lang)) + return path + + +def section_neighbours(api, config, node: int, lang: str) -> dict[str, Any]: + """The previous and next section of the same level, across parents.""" + F = api.F + siblings = F.otype.s(F.otype.v(node)) + (first, last) = (int(siblings[0]), int(siblings[-1])) + # section nodes of one type are numbered consecutively in text order + prev = node - 1 if node > first else None + nxt = node + 1 if node < last else None + return { + "prev": None if prev is None else section_item(api, config, prev, lang), + "next": None if nxt is None else section_item(api, config, nxt, lang), + } + + +def edge_item(x: Any) -> dict[str, Any]: + if isinstance(x, tuple): + return {"node": int(x[0]), "value": x[1]} + return {"node": int(x)} diff --git a/libs/explorer/cfabric_browser/routes/lexicon.py b/libs/explorer/cfabric_browser/routes/lexicon.py new file mode 100644 index 0000000..3e8bd88 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/lexicon.py @@ -0,0 +1,146 @@ +"""The lexicon of corpora with lexeme nodes (BHSA), with BDB.""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import Response + +from cfabric_browser import formats +from cfabric_browser.dictionaries.manage import DICTIONARIES +from cfabric_browser.helpers import tsvEsc +from cfabric_browser.lexicon import Lexicon, render_verse +from cfabric_browser.routes.common import ( + MAX_PAGE, + Context, + tsv_response, +) + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + service = ctx.service + store = ctx.store + corpus_api = ctx.corpus_api + allowed = ctx.allowed + + def corpus_lexicon(corpus: str) -> Lexicon: + corpus_api(corpus) + lexicon = service.lexicon(corpus) + if lexicon is None: + raise HTTPException(404, f"{corpus} has no lexicon") + return lexicon + + def lexeme(lexicon: Lexicon, node: int) -> dict[str, Any]: + entry = lexicon.entry(node) + if entry is None: + raise HTTPException(404, f"No lexeme {node}") + return entry + + @router.get("/corpora/{corpus}/lexicon") + def lexicon_entries(corpus: str) -> dict[str, Any]: + """All lexemes, in alphabetical order per language. + + Each has its node, `lex` (transliteration), `form` (vocalised), + `cons` (consonants), `letter` (the first one), `gloss`, `sp` (part of + speech), `language`, `freq`, `rank` and `nametype`, as far as the + corpus has them; `features` names the corpus feature behind each. + With BDB, a linked lexeme also has `bdbShort` (its main entry's short + definition), `bdbOrder` (that entry's place in BDB), `bdbLetter` (the + letter of its part, its root's) and `bdbRoot` (whether it is a root). + """ + lexicon = corpus_lexicon(corpus) + bdb = store.get("bdb") if allowed(DICTIONARIES["bdb"].corpus) else None + if bdb is None: + return {"features": lexicon.features, "entries": lexicon.entries} + return { + "features": lexicon.features, + "entries": [ + {**e, **bdb.for_list(e.get("language"), e.get("lex"))} + for e in lexicon.entries + ], + "bdb": True, + } + + @router.get("/corpora/{corpus}/lexicon/{node}") + def lexicon_entry(corpus: str, node: int, lang: str = "en") -> dict[str, Any]: + """One lexeme, how often it occurs per book, and in how many verses; + `bdb`: its entries in BDB when that has been added, else None.""" + lexicon = corpus_lexicon(corpus) + entry = lexeme(lexicon, node) + bdb = store.get("bdb") if allowed(DICTIONARIES["bdb"].corpus) else None + return { + **entry, + "verses": len(lexicon.verses(node)), + "books": lexicon.distribution(node, lang=lang), + "bdb": bdb.for_lexeme(entry.get("language"), entry.get("lex")) + if bdb + else None, + } + + @router.get("/corpora/{corpus}/lexicon/{node}/occurrences") + def lexicon_occurrences( + corpus: str, + node: int, + offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=MAX_PAGE), + book: str | None = None, + fmt: str | None = None, + lang: str = "en", + ) -> dict[str, Any]: + """A page of the verses a lexeme occurs in, as running text with its + words marked; with `book`, only the verses in that book.""" + lexicon = corpus_lexicon(corpus) + lexeme(lexicon, node) + api = corpus_api(corpus) + config = service.config(corpus) + fmt = formats.chosen(config, fmt) + verses = lexicon.verses(node, book=book, lang=lang) + return { + "total": len(verses), + "offset": offset, + "rows": [ + render_verse(api, config, verse, words, fmt=fmt or None, lang=lang) + for (verse, words) in verses[offset : offset + limit] + ], + } + + @router.get("/corpora/{corpus}/lexicon/{node}/occurrences.tsv") + def lexicon_tsv( + corpus: str, + node: int, + book: str | None = None, + fmt: str | None = None, + lang: str = "en", + excel: bool = Query( + False, + description="UTF-16 with byte-order mark, which Excel opens correctly", + ), + ) -> Response: + """Every occurrence of a lexeme as tab-separated values: its section, + the word (node and text) and the text of its verse.""" + lexicon = corpus_lexicon(corpus) + entry = lexeme(lexicon, node) + api = corpus_api(corpus) + fmt = formats.chosen(service.config(corpus), fmt) + T = api.T + sectionTypes = list(T.sectionTypes or ()) + header = [*(t.upper() for t in sectionTypes), "WORD", "TEXT", "VERSE_TEXT"] + lines = ["\t".join(header)] + for verse, words in lexicon.verses(node, book=book, lang=lang): + section = T.sectionFromNode(verse, lang=lang) + verseText = tsvEsc(T.text(verse, fmt=fmt or None)) + for w in words: + cells = [tsvEsc("" if s is None else s) for s in section] + cells += [str(w), tsvEsc(T.text(w, fmt=fmt or None)), verseText] + lines.append("\t".join(cells)) + content = "\n".join(lines) + "\n" + name = "".join(c if c.isalnum() else "_" for c in entry.get("lex") or "") + suffix = "-excel" if excel else "" + filename = f"{corpus}-{name.strip('_') or node}{suffix}" + return tsv_response(content, filename, excel) + + return router + + return router diff --git a/libs/explorer/cfabric_browser/routes/library.py b/libs/explorer/cfabric_browser/routes/library.py new file mode 100644 index 0000000..083d010 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/library.py @@ -0,0 +1,367 @@ +"""The library: corpora on this computer to open, translations to build, +dictionaries and corpora to download.""" + +from __future__ import annotations + +import re +import threading +from typing import Any + +from fastapi import APIRouter, HTTPException + +from cfabric_browser.corpora import discover, resolve_corpus +from cfabric_browser.dictionaries.manage import ( + DICTIONARIES, + Catalogued, + is_added, +) +from cfabric_browser.dictionaries.manage import remove as remove_dictionary +from cfabric_browser.downloads import DOWNLOADS, Download, is_present +from cfabric_browser.routes.common import ( + Context, + OpenRequest, +) +from cfabric_browser.translations import TRANSLATIONS, Translation +from cfabric_browser.translations.manage import ( + cancel_removal, + is_built, + is_pending, + remove, + repo_of, + schedule_removal, +) + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + service = ctx.service + pool = ctx.pool + library = ctx.library + lookups = ctx.lookups + store = ctx.store + jobs = ctx.jobs + fetches = ctx.fetches + allowed = ctx.allowed + + @router.get("/library") + def corpus_library() -> dict[str, Any]: + if not library: + return {"enabled": False, "corpora": []} + # a corpus counts as open when its data directory is being served + opened = {spec.path for spec in service.specs} + corpora = [] + for entry in discover(service.roots, hide=service.hide, only=service.only): + try: + path = resolve_corpus(entry["id"], service.roots).path + except FileNotFoundError: + continue + corpora.append({**entry, "open": path in opened}) + return {"enabled": True, "corpora": corpora} + + def open_from_library(requestedId: str, name: str | None = None) -> dict[str, Any]: + """Open a corpus of the library by its id (`ETCBC/dss`, optionally with + `@version`); `opened` is False when it was open already.""" + (requested, _, version) = requestedId.partition("@") + # any corpus on disk can be opened by its id, also copies and hidden ones, + # but not one outside a library limited with `only` + ids = { + entry["id"].lower(): entry["id"] + for entry in discover(service.roots, collapse=False, only=service.only) + } + corpusId = ids.get(requested.lower()) + if corpusId is None: + raise HTTPException(404, f"Not in the library: {requestedId}") + target = f"{corpusId}@{version}" if version else corpusId + try: + spec = resolve_corpus(f"{name}={target}" if name else target, service.roots) + except FileNotFoundError as e: + raise HTTPException(404, str(e)) from None + if service.has(spec.name): + if service.spec(spec.name).path != spec.path: + raise HTTPException( + 409, f"Another corpus is already open as {spec.name!r}" + ) + return {"name": spec.name, "opened": False} + service.add(spec) + pool.add_corpus(spec.name, str(spec.path)) + return {"name": spec.name, "opened": True} + + @router.post("/corpora", status_code=201) + def open_corpus(request: OpenRequest) -> dict[str, Any]: + """Open a corpus from the library. Only library ids are accepted, no paths.""" + if not library: + raise HTTPException(403, "Opening corpora is switched off on this server") + result = open_from_library(request.id, request.name) + if result["opened"]: + # start loading now, so the corpus is ready by the time it is shown + threading.Thread( + target=service.config, args=(result["name"],), daemon=True + ).start() + return result + + # translations: built from their source into a corpus, then in the library + + def translation_of(tid: str) -> Translation: + translation = TRANSLATIONS.get(tid) + if translation is None or not offered(translation): + raise HTTPException(404, f"Unknown translation: {tid}") + return translation + + def offered(translation: Translation) -> bool: + return allowed(translation.corpus_id) + + def translation_entry(translation: Translation) -> dict[str, Any]: + added = is_built(translation, jobs.root) + repo = repo_of(translation, jobs.root).resolve() + return { + "id": translation.id, + "title": translation.title, + "name": translation.name, + "language": translation.language, + "corpus": translation.corpus_id, + "licence": translation.licence, + "licenceNote": translation.licence_note, + "source": translation.source, + "added": added, + "open": any(spec.path.is_relative_to(repo) for spec in service.specs), + # removed at the next start, because it is open + "removing": added and is_pending(translation, jobs.pending), + "job": jobs.status(translation.id), + } + + @router.get("/translations") + def list_translations() -> dict[str, Any]: + if not library: + return {"enabled": False, "translations": []} + return { + "enabled": True, + "translations": [ + translation_entry(t) for t in TRANSLATIONS.values() if offered(t) + ], + } + + @router.get("/translations/{tid}") + def describe_translation(tid: str) -> dict[str, Any]: + return translation_entry(translation_of(tid)) + + @router.post("/translations/{tid}", status_code=202) + def add_translation(tid: str) -> dict[str, Any]: + """Download and build a translation; `job` in the answer, and in + `GET /translations/{tid}`, shows the progress.""" + if not library: + raise HTTPException(403, "Adding texts is switched off on this server") + translation = translation_of(tid) + if is_built(translation, jobs.root): + # still there: adding it again only cancels a removal + cancel_removal(translation, jobs.pending) + else: + jobs.start(translation) + return translation_entry(translation) + + @router.delete("/translations/{tid}") + def remove_translation(tid: str) -> dict[str, Any]: + if not library: + raise HTTPException(403, "Removing texts is switched off on this server") + translation = translation_of(tid) + if not is_built(translation, jobs.root): + raise HTTPException(404, f"{translation.title} has not been added") + if translation_entry(translation)["open"]: + schedule_removal(translation, jobs.pending) + else: + try: + remove(translation, jobs.root) + except OSError as e: + raise HTTPException(409, f"Could not remove it: {e}") from None + return translation_entry(translation) + + # dictionaries: one file each, built once and published, from the library + + def dictionary_of(did: str) -> Catalogued: + catalogued = DICTIONARIES.get(did) + if catalogued is None or not allowed(catalogued.corpus): + raise HTTPException(404, f"Unknown dictionary: {did}") + return catalogued + + def dictionary_entry(catalogued: Catalogued) -> dict[str, Any]: + found = store.get(catalogued.id) + return { + "id": catalogued.id, + "title": catalogued.title, + "name": catalogued.name, + "licence": catalogued.licence, + "licenceNote": catalogued.licenceNote, + "sources": catalogued.sources, + "corpus": catalogued.corpus, + "size": catalogued.size, + # can be downloaded (not before it is published) + "available": catalogued.url is not None, + "added": found is not None, + "built": found.meta.get("built") if found else None, + "job": lookups.status(catalogued.id), + } + + @router.get("/dictionaries") + def list_dictionaries() -> dict[str, Any]: + return { + "enabled": library, + "dictionaries": [ + dictionary_entry(d) for d in DICTIONARIES.values() if allowed(d.corpus) + ], + } + + @router.get("/dictionaries/{did}") + def describe_dictionary(did: str) -> dict[str, Any]: + """A dictionary, and when it has been added its metadata (title, + edition, sources, figures of the build) and abbreviations.""" + catalogued = dictionary_of(did) + result = dictionary_entry(catalogued) + found = store.get(did) + if found is not None: + result["meta"] = found.meta + result["abbreviations"] = found.abbreviations + return result + + @router.post("/dictionaries/{did}", status_code=202) + def add_dictionary(did: str) -> dict[str, Any]: + """Download a dictionary; `job` shows the progress.""" + if not library: + raise HTTPException(403, "Adding is switched off on this server") + catalogued = dictionary_of(did) + if catalogued.url is None: + raise HTTPException(409, f"{catalogued.name} has not been published yet") + if not is_added(catalogued, lookups.root): + lookups.start(catalogued) + return dictionary_entry(catalogued) + + @router.delete("/dictionaries/{did}") + def remove_dictionary_route(did: str) -> dict[str, Any]: + if not library: + raise HTTPException(403, "Removing is switched off on this server") + catalogued = dictionary_of(did) + if not remove_dictionary(catalogued, lookups.root): + raise HTTPException(404, f"{catalogued.name} has not been added") + return dictionary_entry(catalogued) + + def linked_lexemes(keys: list[str], corpus: str | None) -> list[dict[str, Any]]: + """Lexeme keys (`Hebrew BR>[`) as lexemes of a corpus: node, form, + gloss; those the corpus doesn't have are left out.""" + lexicon = service.lexicon(corpus) if corpus and service.has(corpus) else None + if lexicon is None: + return [] + found = [] + for key in keys: + language, _, lex = key.partition(" ") + e = lexicon.find(lex, language) + if e is not None: + found.append( + {k: e.get(k) for k in ("node", "lex", "form", "gloss", "language")} + ) + return found + + def passages(text: str, corpus: str | None) -> dict[str, str]: + """The verses an article cites (`Gen.1.1`) as references in `corpus` + (`Genesis 1:1`), for those of its books the corpus has.""" + if not corpus or not service.has(corpus): + return {} + books = service.books(corpus) + found = {} + for osis in set(re.findall(r'data-osis="([^"]+)"', text)): + book, _, place = osis.partition(".") + if book in books and place: + chapter, _, verse = place.partition(".") + found[osis] = f"{books[book]} {chapter}" + ( + f":{verse}" if verse else "" + ) + return found + + @router.get("/dictionaries/{did}/entries/{eid}") + def dictionary_article( + did: str, eid: str, corpus: str | None = None + ) -> dict[str, Any]: + """An entry: its text (HTML with `` for verses + and `` for other entries), its neighbours, its root + family, and the lexemes of `corpus` linked to it and to its family.""" + dictionary_of(did) + found = store.get(did) + if found is None: + raise HTTPException(404, f"{did} has not been added") + article = found.article(eid) + if article is None: + raise HTTPException(404, f"No entry {eid}") + article["lexemes"] = linked_lexemes(article["lexemes"], corpus) + article["passages"] = passages(article["html"], corpus) + for member in article["family"]: + member["lexemes"] = linked_lexemes( + found.lexemes.get(member["id"], []), corpus + ) + return article + + # corpora that can be downloaded: the BHSA, the SP and the DSS, from their + # own releases (cfabric_browser.downloads) + + def download_of(org: str, repo: str) -> Download: + download = DOWNLOADS.get(f"{org}/{repo}") + if download is None or not allowed(download.id): + raise HTTPException( + 404, f"Not a corpus that can be downloaded: {org}/{repo}" + ) + return download + + def download_entry(download: Download) -> dict[str, Any]: + return { + "id": download.id, + "name": download.name, + "title": download.title, + "version": download.version, + "size": download.size, + "unpacked": download.unpacked, + "terms": download.terms, + "licence": download.licence, + # in any of the places the library looks + "present": any(is_present(download, root) for root in service.roots), + "job": fetches.status(download.id), + } + + @router.get("/downloads") + def list_downloads() -> dict[str, Any]: + if not library: + return {"enabled": False, "downloads": []} + return { + "enabled": True, + "downloads": [ + download_entry(d) for d in DOWNLOADS.values() if allowed(d.id) + ], + } + + @router.get("/downloads/{org}/{repo}") + def describe_download(org: str, repo: str) -> dict[str, Any]: + return download_entry(download_of(org, repo)) + + @router.post("/downloads/{org}/{repo}", status_code=202) + def start_download(org: str, repo: str) -> dict[str, Any]: + """Download a corpus in the background; `job` in the answer, and in + `GET /downloads/{org}/{repo}`, shows the progress.""" + if not library: + raise HTTPException(403, "Downloading is switched off on this server") + download = download_of(org, repo) + if not download_entry(download)["present"]: + + def prepare(progress) -> None: + # open it and load it once, which compiles it: minutes for the + # BHSA, after which it opens in about a second + progress("prepare", 0, 1) + name = open_from_library(download.id)["name"] + service.config(name) + progress("prepare", 1, 1) + + fetches.start(download, then=prepare) + return download_entry(download) + + @router.delete("/downloads/{org}/{repo}") + def cancel_download(org: str, repo: str) -> dict[str, Any]: + """Stop a download; a corpus being prepared is not stopped.""" + download = download_of(org, repo) + fetches.cancel(download.id) + return download_entry(download) + + return router diff --git a/libs/explorer/cfabric_browser/routes/parallel.py b/libs/explorer/cfabric_browser/routes/parallel.py new file mode 100644 index 0000000..998bcee --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/parallel.py @@ -0,0 +1,249 @@ +"""The Parallel interface: verse rows across corpora, also for search results.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Annotated, Any, Literal + +from fastapi import APIRouter, Depends, HTTPException, Query + +from cfabric_browser import formats +from cfabric_browser.concordance import Spans +from cfabric_browser.display import ( + RenderOptions, + node_from_section_ref, + render_node, + section_ref_for, +) +from cfabric_browser.routes.common import ( + MAX_PAGE, + Context, + render_options, + section_neighbours, + section_path, + split_items, + witness_row, +) +from cfabric_browser.witnesses import dress_signs, format_ref, parse_ref + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + service = ctx.service + corpus_api = ctx.corpus_api + search_job = ctx.search_job + + @router.get("/parallel") + def parallel( + ref: str, + lead: str, + opts: Annotated[RenderOptions, Depends(render_options)], + texts: str = "", + mode: Literal["pretty", "plain"] = "plain", + ) -> dict[str, Any]: + """A chapter in several corpora, as rows of verses with a cell per corpus. + + `ref` is a chapter of `lead` (`Genesis 1`), whose numbering the rows + follow; `texts` names the corpora of the columns (comma separated), + with or without `lead`. Books are matched through the book concordance, + chapters and verses by number. There is a row for every verse number + in any of the texts, in numeric order, and a cell only where a text has + that verse. A corpus read by biblical book (DSS) gets, per verse, the + text of each scroll that has it, as in `witnesses`. `fmt` applies to + the corpora that have that format; the others use their default. + + Per column: its `kind` (`text` or `witnesses`) and the `ref` of its + chapter, or None when it has no counterpart of `ref`. + """ + names = split_items(texts) or [] + lApi = corpus_api(lead) + for name in names: + corpus_api(name) + lConfig = service.config(lead) + try: + node = node_from_section_ref( + lApi, ref, lConfig.section_sep1, lConfig.section_sep2, opts.lang + ) + except ValueError as e: + raise HTTPException(404, str(e)) from None + chapterRef = section_ref_for(lApi, lConfig, node, lang=opts.lang) + + columns: list[dict[str, Any]] = [] + cells: dict[int, dict[str, Any]] = {} + for name in names: + api = service.api(name) + config = service.config(name) + own = formats.options(config, opts) + index = service.book_index(name) + if index is not None: + aligned = service.align(lead, chapterRef, name, opts.lang, "books") + parsed = parse_ref(aligned["ref"]) if aligned else None + if parsed is None or not index.has(parsed[0], parsed[1]): + columns.append({"name": name, "kind": "witnesses", "ref": None}) + continue + (book, chapter, _) = parsed + columns.append( + { + "name": name, + "kind": "witnesses", + "ref": format_ref(book, chapter), + } + ) + for v, witnesses in index.books[book][chapter].items(): + cells.setdefault(int(v), {})[name] = { + "witnesses": [ + witness_row(api, config, index, w, mode, own) + for w in witnesses + ] + } + continue + + sectionTypes = list(api.T.sectionTypes or ()) + if name == lead: + cNode: int | None = node + else: + aligned = service.align(lead, chapterRef, name, opts.lang) + exact = aligned is not None and aligned["exact"] + cNode = aligned["node"] if exact else None + if cNode is None or len(sectionTypes) < 3: + columns.append({"name": name, "kind": "text", "ref": None}) + continue + columns.append( + { + "name": name, + "kind": "text", + "ref": section_ref_for(api, config, cNode, lang=opts.lang), + } + ) + for v in api.L.d(cNode, otype=sectionTypes[2]): + number = api.T.sectionFromNode(int(v), lang="en")[2] + try: + key = int(number) + except (TypeError, ValueError): + continue + tree = render_node(api, config, int(v), mode, own).to_dict() + dress_signs(api, tree["fragments"]) + cell = cells.setdefault(key, {}).setdefault(name, {"trees": []}) + cell["trees"].append(tree) + # a translation: a note printed after the verse, and the verse's + # number in the print where it is divided differently + for extra in ("note", "printed"): + if extra in api.Fall() and (value := api.Fs(extra).v(int(v))): + cell[extra] = value + + rows = [ + { + "verse": v, + "ref": f"{chapterRef}{lConfig.section_sep2}{v}", + "cells": cells[v], + } + for v in sorted(cells) + ] + return { + "ref": chapterRef, + "path": section_path(lApi, lConfig, node, opts.lang), + **section_neighbours(lApi, lConfig, node, opts.lang), + "columns": columns, + "rows": rows, + } + + @router.get("/searches/{search_id}/parallel") + def search_parallel( + search_id: str, + opts: Annotated[RenderOptions, Depends(render_options)], + offset: int = Query(0, ge=0), + limit: int = Query(25, ge=1, le=MAX_PAGE), + texts: str = "", + mode: Literal["pretty", "plain"] = "plain", + ) -> dict[str, Any]: + """A page of results as rows of the parallel interface. + + Per result: the verse (the lowest section) its first node lies in, in + the searched corpus with the result's innermost nodes highlighted, and + the same verse in each of `texts`, as in `/parallel` (for the DSS, the + text of each scroll that has it). Cells are left out where a text has + no such verse. + """ + job = search_job(search_id) + corpus = job.corpus + api = corpus_api(corpus) + config = service.config(corpus) + names = [n for n in (split_items(texts) or []) if n != corpus] + for name in names: + corpus_api(name) + sectionTypes = list(api.T.sectionTypes or ()) + unitType = sectionTypes[-1] if sectionTypes else None + rank = api.N.otypeRank + spans = Spans(api) + + rows = [] + for i, tup in enumerate(job.page(offset, limit)): + nodes = [int(n) for n in tup] + if unitType is None: + break + # the first node that lies within a verse: not a book or chapter + # that the template starts with + first = next( + (n for n in nodes if rank[api.F.otype.v(n)] <= rank[unitType]), + nodes[0], + ) + if api.F.otype.v(first) == unitType: + unit = first + else: + ups = api.L.u(spans.span(first)[0], otype=unitType) + if not ups: + continue + unit = int(ups[0]) + # highlight the innermost nodes: those that contain no other node of the result + inner = [n for n in spans.innermost(nodes) if n != unit] + own = replace(opts, highlights={n: "" for n in inner}) + tree = render_node(api, config, unit, mode, own).to_dict() + dress_signs(api, tree["fragments"]) + ref = section_ref_for(api, config, unit, lang=opts.lang) + cells: dict[str, Any] = {corpus: {"trees": [tree]}} + for name in names: + cell = verse_cell(corpus, ref, name, opts, mode) + if cell is not None: + cells[name] = cell + rows.append( + {"index": offset + i, "nodes": nodes, "ref": ref, "cells": cells} + ) + return { + "search": job.snapshot(), + "offset": offset, + "columns": [corpus, *names], + "rows": rows, + } + + def verse_cell( + source: str, ref: str, target: str, opts: RenderOptions, mode: str + ) -> dict[str, Any] | None: + """The verse `ref` of `source` in `target`, as a cell of the parallel + interface, or None when `target` does not have it.""" + api = service.api(target) + config = service.config(target) + own = formats.options(config, opts) + own = replace(own, highlights=None) + index = service.book_index(target) + if index is not None: + aligned = service.align(source, ref, target, opts.lang, "books") + parsed = parse_ref(aligned["ref"]) if aligned and aligned["exact"] else None + if parsed is None or parsed[2] is None or not index.has(*parsed): + return None + (book, chapter, verse) = parsed + return { + "witnesses": [ + witness_row(api, config, index, w, mode, own) + for w in index.books[book][chapter][verse] + ] + } + aligned = service.align(source, ref, target, opts.lang) + if aligned is None or not aligned["exact"]: + return None + tree = render_node(api, config, aligned["node"], mode, own).to_dict() + dress_signs(api, tree["fragments"]) + return {"trees": [tree]} + + # the concordance: results as lines with the hit in context + + return router diff --git a/libs/explorer/cfabric_browser/routes/proposals.py b/libs/explorer/cfabric_browser/routes/proposals.py new file mode 100644 index 0000000..3d40c05 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/proposals.py @@ -0,0 +1,72 @@ +"""Searches proposed by a language model (`cfabric_browser.assistant`).""" + +from __future__ import annotations + +import json +from typing import Any + +from anyio import to_thread +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse + +from cfabric_browser.assistant.tools import ToolError +from cfabric_browser.routes.common import ( + Context, + ProposalRequest, + ProposalState, +) + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + tools = ctx.tools + + @router.get("/proposals") + def list_proposals() -> dict[str, Any]: + """The proposals, the newest first.""" + return tools.proposals.snapshot() + + @router.post("/proposals") + def add_proposal(body: ProposalRequest) -> dict[str, Any]: + """Propose a search, as the model's `propose_search` does (other web + pages are kept out by `cfabric_browser.guard`).""" + try: + return tools.propose_search( + body.corpus, body.template, body.request, body.explanation, body.model + ) + except ToolError as e: + raise HTTPException(404, str(e)) from None + + @router.post("/proposals/{pid}") + def set_proposal(pid: str, body: ProposalState) -> dict[str, Any]: + """The user ran or dismissed a proposal.""" + found = tools.proposals.set_state(pid, body.state) + if found is None: + raise HTTPException(404, f"No proposal {pid}") + return found + + @router.get("/proposals/events") + async def proposal_events(request: Request) -> StreamingResponse: + """Server-sent events with all proposals whenever they change.""" + store = tools.proposals + + async def stream(): + version = -1 + while True: + if await request.is_disconnected(): + return + now = await to_thread.run_sync(store.wait_change, version, 5.0) + if now != version: + version = now + yield f"event: proposals\ndata: {json.dumps(store.snapshot())}\n\n" + else: + # keeps the connection open through proxies + yield ": waiting\n\n" + + return StreamingResponse( + stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + return router diff --git a/libs/explorer/cfabric_browser/routes/reading.py b/libs/explorer/cfabric_browser/routes/reading.py new file mode 100644 index 0000000..8874e36 --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/reading.py @@ -0,0 +1,511 @@ +"""Reading: corpora, features, sections, passages, alignment, differences, +the book view, the Interlinear, and words.""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from fastapi import APIRouter, Depends, HTTPException, Query + +from cfabric_browser import formats +from cfabric_browser.compare import ( + compare_passages, + compare_three, + compare_witnesses, +) +from cfabric_browser.dictionaries.manage import DICTIONARIES +from cfabric_browser.display import ( + RenderOptions, + node_from_section_ref, + render_node, + section_ref_for, +) +from cfabric_browser.interlinear import interlinear_verse +from cfabric_browser.routes.common import ( + MAX_DOWN, + MAX_PASSAGE, + Context, + edge_item, + render_options, + section_item, + section_neighbours, + section_path, + split_items, + witness_row, +) +from cfabric_browser.summary import lexeme_of, summary +from cfabric_browser.witnesses import dress_signs, format_ref, parse_ref + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + service = ctx.service + store = ctx.store + corpus_api = ctx.corpus_api + check_node = ctx.check_node + allowed = ctx.allowed + + @router.get("/corpora") + def list_corpora() -> dict[str, Any]: + return { + "corpora": [ + { + "name": name, + # without loading: a corpus may be compiling for minutes + "title": service.title(name), + # translations are not compared with the other texts + "translation": service.translation(name) is not None, + } + for name in service.names() + ] + } + + @router.get("/corpora/{corpus}") + def describe_corpus(corpus: str) -> dict[str, Any]: + corpus_api(corpus) + return service.card(corpus) + + @router.get("/corpora/{corpus}/features") + def list_features(corpus: str) -> dict[str, Any]: + from cfabric.describe import list_features as listFeatures + + api = corpus_api(corpus) + excluded = service.config(corpus).excluded_features + features = [ + {**f.to_dict(), "excluded": f.name in excluded} for f in listFeatures(api) + ] + return {"features": features} + + @router.get("/corpora/{corpus}/features/{feature}") + def describe_feature( + corpus: str, feature: str, sample_limit: int = Query(20, ge=0, le=500) + ) -> dict[str, Any]: + from cfabric.describe import describe_feature as describeFeature + + api = corpus_api(corpus) + result: dict[str, Any] = describeFeature( + api, feature, sample_limit=sample_limit + ).to_dict() + if "error" in result: + raise HTTPException(404, f"Unknown feature: {feature}") + return result + + # sections and passages + + @router.get("/corpora/{corpus}/sections") + def sections(corpus: str, ref: str | None = None, lang: str = "en") -> dict: + """The section tree, one level at a time: books, then their chapters, ...""" + api = corpus_api(corpus) + config = service.config(corpus) + T = api.T + L = api.L + sectionTypes = list(T.sectionTypes or ()) + if not sectionTypes: + return {"node": None, "heading": "", "children": []} + if ref: + try: + node = node_from_section_ref( + api, ref, config.section_sep1, config.section_sep2, lang + ) + except ValueError as e: + raise HTTPException(404, str(e)) from None + level = sectionTypes.index(api.F.otype.v(node)) + if level + 1 >= len(sectionTypes): + childNodes: list[int] = [] + else: + childNodes = [int(n) for n in L.d(node, otype=sectionTypes[level + 1])] + heading = section_ref_for(api, config, node, lang=lang) + else: + node = None + childNodes = [int(n) for n in api.F.otype.s(sectionTypes[0])] + heading = "" + children = [section_item(api, config, n, lang) for n in childNodes] + return {"node": node, "heading": heading, "children": children} + + @router.get("/corpora/{corpus}/align") + def align( + corpus: str, + ref: str, + source: Annotated[str, Query(alias="from")], + lang: str = "en", + view: Literal["sections", "books"] = "sections", + ) -> dict[str, Any]: + """The passage of this corpus that corresponds to `ref` in `from`. + + Books are matched with a concordance of book names; chapters and + verses by number. `exact` is false when only an enclosing section + exists here (a verse missing in this corpus, for instance). + With `view=books`, the passage in this corpus' book view (DSS). + """ + corpus_api(corpus) + corpus_api(source) + try: + result = service.align(source, ref, corpus, lang, view) + except ValueError as e: + raise HTTPException(404, str(e)) from None + if result is None: + raise HTTPException(404, f"{corpus} has no counterpart of {ref}") + return result + + @router.get("/compare") + def compare( + a: str, + aref: str, + b: str, + bref: str, + lang: str = "en", + bview: Literal["sections", "books"] = "sections", + ) -> dict[str, Any]: + """Words that differ between passage `aref` of `a` and `bref` of `b`. + + `a` and `b` map word nodes of each corpus to `matres` (only matres + lectionis differ) or `variant`. With `bview=books`, `bref` is a passage + in the book view of `b` (DSS): each scroll is compared with `a`, and + `aWitnesses` names, per marked word of `a`, the scrolls that differ. + See `cfabric_browser.compare`. + """ + corpus_api(a) + corpus_api(b) + try: + if bview == "books": + return compare_witnesses(service, a, aref, b, bref, lang) + return compare_passages(service, a, aref, b, bref, lang) + except ValueError as e: + raise HTTPException(404, str(e)) from None + + # reading by biblical book (DSS) + + def book_index(corpus: str): + corpus_api(corpus) + index = service.book_index(corpus) + if index is None: + raise HTTPException(404, f"{corpus} cannot be read by biblical book") + return index + + @router.get("/corpora/{corpus}/books") + def books(corpus: str) -> dict[str, Any]: + """The biblical books in this corpus, with chapters, verses and scrolls.""" + return {"books": book_index(corpus).summary()} + + @router.get("/corpora/{corpus}/witnesses") + def witnesses( + corpus: str, + ref: str, + opts: Annotated[RenderOptions, Depends(render_options)], + mode: Literal["pretty", "plain"] = "plain", + ) -> dict[str, Any]: + """A chapter or verse in the book view: per verse, the text of each scroll. + + Each witness has the scroll, the place of its words (`f1i:1–2`), a + reference to open its fragment in the scroll, its words and a render + tree of them, in which reconstructed text is flagged `dim`. + `alsoNonBiblical` is set for text that the DSS also lists among + non-biblical material (`biblical=2`). + """ + index = book_index(corpus) + parsed = parse_ref(ref) + if parsed is None or not index.has(*parsed): + raise HTTPException(404, f"{corpus} has no {ref}") + (book, chapter, verse) = parsed + api = corpus_api(corpus) + config = service.config(corpus) + opts = formats.options(config, opts) + chapterVerses = index.books[book][chapter] + verses = [] + for v in [verse] if verse is not None else list(chapterVerses): + rows = [ + witness_row(api, config, index, w, mode, opts) for w in chapterVerses[v] + ] + verses.append( + {"verse": v, "ref": format_ref(book, chapter, v), "witnesses": rows} + ) + return { + "ref": format_ref(book, chapter, verse), + "book": book, + "chapter": chapter, + "chapterRef": format_ref(book, chapter), + "verses": verses, + **index.neighbours(book, chapter), + } + + # the interlinear: a passage word by word + + @router.get("/corpora/{corpus}/interlinear") + def interlinear( + corpus: str, + ref: str, + opts: Annotated[RenderOptions, Depends(render_options)], + features: str | None = Query( + None, description="Comma separated features to add as lines" + ), + under: str | None = Query( + None, description="Another corpus to show word by word under this one" + ), + ) -> dict[str, Any]: + """A passage (a chapter or a verse) word by word, for the interlinear. + + Per verse its words, each with its text and its lines in words + (`translit`, `lexeme`, `gloss`, `pos`, `morph`, read by role as on the + hover card), the values of `features`, and the indexes of its phrase + and clause in `phrases` and `clauses` (with the phrase function and + clause type). With `under`, also that corpus' verse, its words under + the words of this one, and the words that differ. See + `cfabric_browser.interlinear`. + """ + api = corpus_api(corpus) + if under: + corpus_api(under) + config = service.config(corpus) + opts = formats.options(config, opts) + try: + node = node_from_section_ref( + api, ref, config.section_sep1, config.section_sep2, opts.lang + ) + except ValueError as e: + raise HTTPException(404, str(e)) from None + verseType = (api.T.sectionTypes or [None])[-1] + if api.F.otype.v(node) == verseType: + verses = [node] + else: + verses = [int(v) for v in api.L.d(node, otype=verseType)] + names = split_items(features) or [] + return { + "ref": section_ref_for(api, config, node, lang=opts.lang), + "path": section_path(api, config, node, opts.lang), + **section_neighbours(api, config, node, opts.lang), + "truncated": len(verses) > MAX_PASSAGE, + "verses": [ + interlinear_verse( + service, + corpus, + v, + opts, + names, + under, + lambda w: word_short(corpus, w), + ) + for v in verses[:MAX_PASSAGE] + ], + } + + # the parallel interface: a chapter in several corpora, verse by verse + + @router.get("/agreement") + def agreement( + a: str, aref: str, b: str, bref: str, c: str, cref: str, lang: str = "en" + ) -> dict[str, Any]: + """Which scrolls of `c` (DSS, book view) side with `a` (MT) or `b` (SP). + + Per verse, every place where `a` and `b` differ, with the words of each + and, per scroll that preserves it, its reading there and whether it + `agrees` with `a`, `b` or neither (`other`). See + `cfabric_browser.compare.compare_three`. + """ + for corpus in (a, b, c): + corpus_api(corpus) + try: + return compare_three(service, a, aref, b, bref, c, cref, lang) + except ValueError as e: + raise HTTPException(404, str(e)) from None + + @router.get("/corpora/{corpus}/passage") + def passage( + corpus: str, + ref: str, + opts: Annotated[RenderOptions, Depends(render_options)], + mode: Literal["pretty", "plain"] = "pretty", + ) -> dict[str, Any]: + api = corpus_api(corpus) + config = service.config(corpus) + opts = formats.options(config, opts) + try: + node = node_from_section_ref( + api, ref, config.section_sep1, config.section_sep2, opts.lang + ) + except ValueError as e: + raise HTTPException(404, str(e)) from None + nType = api.F.otype.v(node) + # a section above the condense type (book, chapter) is shown as its + # condense-type parts, e.g. the verses of a chapter + condense = opts.condense_type or config.condense_type + rank = api.N.otypeRank + if not opts.full and condense and rank[nType] > rank[condense]: + nodes = [int(n) for n in api.L.d(node, otype=condense)] + else: + nodes = [node] + trees = [ + render_node(api, config, n, mode, opts).to_dict() + for n in nodes[:MAX_PASSAGE] + ] + for tree in trees: + dress_signs(api, tree["fragments"]) + # a translation: notes printed after verses, and verse numbers in the + # print where it divides differently (`cfabric_browser.translations`) + extras = { + extra: { + str(n): value + for n in nodes[:MAX_PASSAGE] + if (value := api.Fs(extra).v(n)) + } + for extra in ("note", "printed") + if service.translation(corpus) is not None and extra in api.Fall() + } + return { + "node": node, + "ref": section_ref_for(api, config, node, lang=opts.lang), + **{f"{extra}s": values for (extra, values) in extras.items()}, + "total": len(nodes), + "truncated": len(nodes) > MAX_PASSAGE, + "path": section_path(api, config, node, opts.lang), + **section_neighbours(api, config, node, opts.lang), + "trees": trees, + } + + # nodes + + @router.get("/corpora/{corpus}/nodes/{node}") + def node_render( + corpus: str, + node: int, + opts: Annotated[RenderOptions, Depends(render_options)], + mode: Literal["pretty", "plain"] = "pretty", + ) -> dict[str, Any]: + api = corpus_api(corpus) + check_node(api, node) + config = service.config(corpus) + tree: dict[str, Any] = render_node( + api, config, node, mode, formats.options(config, opts) + ).to_dict() + return tree + + def word_summary(corpus: str, node: int) -> list[dict[str, str]]: + """The essentials of a word; in a corpus without glosses (the DSS), + with the gloss from the lexicon of another open corpus (the BHSA).""" + api = corpus_api(corpus) + config = service.config(corpus) + empty = {str(v) for v in config.none_values if v is not None} + glosses = service.glosses(corpus) + if glosses is None: + return summary(api, node, empty) + (gloss_for, source) = glosses + return summary(api, node, empty, gloss_for, f"gloss in {source}") + + def word_bdb(corpus: str, node: int) -> dict[str, Any] | None: + """A word's entries in BDB, with its lexeme in the corpus whose + lexicon has it (the BHSA, also for the DSS): `{corpus, node, lex, + entries}`; None without BDB or a lexeme.""" + bdb = store.get("bdb") if allowed(DICTIONARIES["bdb"].corpus) else None + if bdb is None: + return None + api = corpus_api(corpus) + own = service.lexicon(corpus) + entry = None + if own is not None: + # the lexeme node the word is in + up = api.L.u(node, "lex") if "lex" in api.F.otype.all else () + entry = own.entry(int(up[0])) if len(up) else None + source = corpus + else: + # by its ETCBC lexeme, in the corpus it borrows glosses from (DSS) + glosses = service.glosses(corpus) + source = glosses[1] if glosses else None + config = service.config(corpus) + empty = {str(v) for v in config.none_values if v} + found = lexeme_of(api, node, empty) + lexicon = service.lexicon(source) if source else None + entry = lexicon.lookup(*found) if lexicon and found else None + if entry is None: + return None + link = bdb.for_lexeme(entry.get("language"), entry.get("lex")) + if not link["entries"]: + return None + return { + "corpus": source, + "node": entry["node"], + "lex": entry.get("lex"), + **link, + } + + def word_short(corpus: str, node: int) -> str | None: + """BDB's short definition of a word (its main entry), or None.""" + found = word_bdb(corpus, node) + if found is None: + return None + return found["entries"][0]["short"] or None + + @router.get("/corpora/{corpus}/nodes/{node}/summary") + def node_summary(corpus: str, node: int, lang: str = "en") -> dict[str, Any]: + """The essentials of a node, for the card shown when hovering over a + word: its text, and its gloss, lexeme, part of speech and morphology in + words (see `cfabric_browser.summary`).""" + api = corpus_api(corpus) + check_node(api, node) + config = service.config(corpus) + slots = api.E.oslots.s(node) + return { + "node": node, + "type": api.F.otype.v(node), + "ref": section_ref_for(api, config, node, lang=lang), + "text": api.T.text(node) if len(slots) <= 200 else None, + "summary": word_summary(corpus, node), + "bdb": word_bdb(corpus, node), + } + + @router.get("/corpora/{corpus}/nodes/{node}/info") + def node_info(corpus: str, node: int, lang: str = "en") -> dict[str, Any]: + """Everything about one node, for the node inspector.""" + api = corpus_api(corpus) + check_node(api, node) + config = service.config(corpus) + F, E, L, T = api.F, api.E, api.L, api.T + nType = F.otype.v(node) + slots = E.oslots.s(node) + + features = {} + for name in sorted(api.Fall()): + if name == "otype": + continue + value = api.Fs(name).v(node) + if value is not None: + features[name] = value if isinstance(value, (int, str)) else str(value) + + edges: dict[str, dict[str, list]] = {} + for name in sorted(api.Eall()): + # oslots is structure; omap@... maps nodes between corpus versions + if name == "oslots" or name.startswith("omap@"): + continue + es = api.Es(name) + out = [edge_item(x) for x in (es.f(node) or ())] + inc = [edge_item(x) for x in (es.t(node) or ())] + if out or inc: + edges[name] = {"from": out, "to": inc} + + up = [{"node": int(n), "type": F.otype.v(int(n))} for n in L.u(node)] + downCounts: dict[str, int] = {} + down: list[dict[str, Any]] = [] + for n in L.d(node): + t = F.otype.v(int(n)) + downCounts[t] = downCounts.get(t, 0) + 1 + if len(down) < MAX_DOWN: + down.append({"node": int(n), "type": t}) + + text = T.text(node) if len(slots) <= 200 else None + return { + "node": node, + "type": nType, + "ref": section_ref_for(api, config, node, lang=lang), + "slots": [int(slots[0]), int(slots[-1])], + "slotCount": len(slots), + "text": text, + "summary": word_summary(corpus, node), + "bdb": word_bdb(corpus, node), + "features": features, + "edges": edges, + "up": up, + "down": down, + "downCounts": downCounts, + } + + # search + + return router diff --git a/libs/explorer/cfabric_browser/routes/search.py b/libs/explorer/cfabric_browser/routes/search.py new file mode 100644 index 0000000..0cd9add --- /dev/null +++ b/libs/explorer/cfabric_browser/routes/search.py @@ -0,0 +1,530 @@ +"""Searching: starting, checking and following searches; results, the +Concordance, statistics and export.""" + +from __future__ import annotations + +import asyncio +import json +import threading +from collections import Counter, OrderedDict +from typing import Annotated, Any, Literal + +from anyio import to_thread +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import Response, StreamingResponse + +from cfabric_browser import formats +from cfabric_browser.concordance import MAX_SIDE, Concordance, parse_filters +from cfabric_browser.display import ( + RenderOptions, + render_tuple, + section_ref_for, +) +from cfabric_browser.helpers import tsvEsc +from cfabric_browser.queries import ( + LANGUAGES, + check_template, + template_constraints, + template_features, +) +from cfabric_browser.routes.common import ( + MAX_PAGE, + STATS_SAMPLE, + Context, + SearchRequest, + render_options, + section_item, + split_items, + tsv_response, +) +from cfabric_browser.workers import FINISHED, SearchJob + + +def build(ctx: Context) -> APIRouter: + router = APIRouter() + service = ctx.service + pool = ctx.pool + corpus_api = ctx.corpus_api + search_job = ctx.search_job + + @router.post("/corpora/{corpus}/search", status_code=202) + def start_search(corpus: str, request: SearchRequest) -> dict[str, Any]: + api = corpus_api(corpus) + if request.language not in LANGUAGES: + raise HTTPException( + 422, + f"Unsupported query language {request.language!r};" + f" supported: {', '.join(LANGUAGES)}", + ) + problems = check_template(api, request.template) + if problems: + raise HTTPException( + 400, + { + "message": "Invalid search template", + "problems": problems, + "template": request.template, + }, + ) + job = pool.submit( + corpus, request.template, limit=request.limit, timeout=request.timeout + ) + return job.snapshot() + + @router.post("/corpora/{corpus}/check") + def check(corpus: str, request: SearchRequest) -> dict[str, Any]: + """Check a template without searching, e.g. while typing.""" + api = corpus_api(corpus) + problems = check_template(api, request.template) + return {"ok": not problems, "problems": problems} + + @router.get("/searches/{search_id}") + def search_status(search_id: str) -> dict[str, Any]: + return search_job(search_id).snapshot() + + @router.delete("/searches/{search_id}") + def cancel_search(search_id: str) -> dict[str, Any]: + job = search_job(search_id) + pool.cancel(search_id) + return job.snapshot() + + # searches proposed by a language model (cfabric_browser.assistant) + + @router.get("/searches/{search_id}/events") + async def search_events(search_id: str, request: Request) -> StreamingResponse: + """Server-sent events with the status of a search until it finishes.""" + job = search_job(search_id) + + async def stream(): + version = -1 + lastSent = 0.0 + loop = asyncio.get_running_loop() + while True: + if await request.is_disconnected(): + return + version = await to_thread.run_sync(job.wait_change, version, 1.0) + now = loop.time() + snap = job.snapshot() + finished = snap["state"] in FINISHED + # at most ~10 updates per second while running + if finished or now - lastSent >= 0.1: + lastSent = now + yield f"event: status\ndata: {json.dumps(snap)}\n\n" + if finished: + return + + return StreamingResponse( + stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @router.get("/searches/{search_id}/results") + def search_results( + search_id: str, + offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=MAX_PAGE), + render: Literal["none", "plain"] = "none", + fmt: str | None = None, + lang: str = "en", + ) -> dict[str, Any]: + """A page of results: node tuples with their passage reference. + + With `render=plain`, each row also has a plain render tree per node. + While a search runs, rows come in the order they were found; once + it is done they are in canonical order (`sorted` is then true). + """ + job = search_job(search_id) + api = corpus_api(job.corpus) + config = service.config(job.corpus) + F = api.F + opts = RenderOptions(fmt=formats.chosen(config, fmt), lang=lang) + rows = [] + for i, tup in enumerate(job.page(offset, limit)): + row: dict[str, Any] = { + "index": offset + i, + "nodes": list(tup), + "types": [F.otype.v(n) for n in tup], + "ref": section_ref_for(api, config, tup[0], lang=lang), + } + if render == "plain": + row["plain"] = [ + t.to_dict() + for t in render_tuple(api, config, tup, mode="plain", options=opts) + ] + rows.append(row) + return {"search": job.snapshot(), "offset": offset, "rows": rows} + + @router.get("/searches/{search_id}/results/{index}") + def search_result( + search_id: str, + index: int, + opts: Annotated[RenderOptions, Depends(render_options)], + mode: Literal["pretty", "plain"] = "pretty", + ) -> dict[str, Any]: + """One result rendered in its context, with its nodes highlighted.""" + job = search_job(search_id) + page = job.page(index, 1) + if not page: + raise HTTPException(404, f"No result {index}") + api = corpus_api(job.corpus) + config = service.config(job.corpus) + trees = render_tuple(api, config, page[0], mode, formats.options(config, opts)) + return { + "index": index, + "nodes": list(page[0]), + "trees": [t.to_dict() for t in trees], + } + + concordances: OrderedDict[str, tuple[int, bool, Concordance]] = OrderedDict() + concordanceLock = threading.Lock() + + def concordance(job: SearchJob) -> Concordance: + """The concordance of a search's results. While the search runs, new + results are added to it; once they are sorted, it is built again.""" + snapshot = job.snapshot() + with concordanceLock: + cached = concordances.get(job.id) + if cached is not None: + concordances.move_to_end(job.id) + if cached and cached[0] == snapshot["version"]: + return cached[2] + if cached and cached[1] == snapshot["sorted"]: + # the same order: only results were added + built = cached[2] + with concordanceLock: + built.extend(job.page(len(built.hits), snapshot["count"])) + else: + api = corpus_api(job.corpus) + config = service.config(job.corpus) + built = Concordance(api, config, job.page(0, snapshot["count"])) + with concordanceLock: + concordances[job.id] = (snapshot["version"], snapshot["sorted"], built) + concordances.move_to_end(job.id) + # keep a few, the last used: they hold a list of words per result + while len(concordances) > 8: + concordances.popitem(last=False) + return built + + def concordance_selection( + conc: Concordance, + filters: str | None, + per: str, + sample: int | None, + seed: int, + ) -> list[int]: + return conc.select(parse_filters(filters), per == "verse", sample, seed) + + @router.get("/searches/{search_id}/kwic") + def kwic( + search_id: str, + offset: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=MAX_PAGE), + left: int = Query(6, ge=0, le=MAX_SIDE), + right: int = Query(6, ge=0, le=MAX_SIDE), + sort: str | None = Query( + None, description="`.`: hit.lex, L1.text, R2.sp, ..." + ), + filter: str | None = Query( + None, description="Features of the hit: vs:qal,book:Genesis" + ), + per: Literal["result", "verse"] = "result", + sample: int | None = Query(None, ge=1), + seed: int = 1, + features: str | None = Query( + None, description="Comma separated features to give per word" + ), + fmt: str | None = None, + lang: str = "en", + ) -> dict[str, Any]: + """A page of the results as concordance lines (KWIC): per result its + hit, the words before it (`left`) and after it (`right`), and its verse. + Sorting, filtering, one line per verse and sampling apply to all + results. See `cfabric_browser.concordance`.""" + job = search_job(search_id) + conc = concordance(job) + selection = concordance_selection(conc, filter, per, sample, seed) + try: + selection = conc.sort(selection, sort) + except ValueError as e: + raise HTTPException(400, str(e)) from None + api = conc.api + config = service.config(job.corpus) + fmt = formats.chosen(config, fmt) + names = split_items(features) or [] + lines = [] + for i in selection[offset : offset + limit]: + line = conc.line(i, left, right, names, fmt) + verse = line.pop("verse") + line["ref"] = ( + section_ref_for(api, config, verse, lang=lang) if verse else None + ) + lines.append(line) + return { + "search": job.snapshot(), + "total": len(selection), + "words": conc.last - conc.first + 1, + "offset": offset, + "lines": lines, + } + + @router.get("/searches/{search_id}/kwic/facets") + def kwic_facets( + search_id: str, + feature: str, + filter: str | None = None, + per: Literal["result", "verse"] = "result", + top: int = Query(50, ge=1, le=500), + ) -> dict[str, Any]: + """How often each value of a feature of the hit's first word occurs in + the (filtered) results, most frequent first; `book` gives the books in + their order.""" + job = search_job(search_id) + conc = concordance(job) + selection = concordance_selection(conc, filter, per, None, 1) + return { + "feature": feature, + "total": len(selection), + "values": conc.facet(selection, feature, top), + } + + @router.get("/searches/{search_id}/collocations") + def collocations( + search_id: str, + feature: str = "lex", + window: int = Query(3, ge=1, le=10), + filter: str | None = None, + per: Literal["result", "verse"] = "result", + min_count: int = Query(2, ge=1, alias="min"), + top: int = Query(40, ge=1, le=500), + ) -> dict[str, Any]: + """The values of a feature of the words within `window` words of the + hits, by their logDice score: how typical they are near the hit, + compared with the whole corpus.""" + job = search_job(search_id) + conc = concordance(job) + selection = concordance_selection(conc, filter, per, None, 1) + return { + "feature": feature, + "window": window, + "hits": len(selection), + "values": conc.collocations( + selection, feature, window, id(conc.api), min_count, top + ), + } + + # statistics over the results of a search + + def position_nodes(job: SearchJob, position: int) -> list[int]: + rows = job.page(0, job.count) + if rows and not 0 <= position < len(rows[0]): + raise HTTPException(404, f"Results have no position {position + 1}") + return [row[position] for row in rows] + + @router.get("/searches/{search_id}/stats/features") + def stats_features( + search_id: str, position: int = Query(0, ge=0) + ) -> dict[str, Any]: + """Features that have values on the nodes at one position of the results. + + Looks at a sample of the results; features the template mentions for + this position come first. + """ + job = search_job(search_id) + api = corpus_api(job.corpus) + config = service.config(job.corpus) + nodes = position_nodes(job, position) + if not nodes: + return { + "position": position, + "type": None, + "features": [], + "inTemplate": [], + "open": [], + } + sample = nodes[:: max(1, len(nodes) // STATS_SAMPLE)][:STATS_SAMPLE] + nType = api.F.otype.v(nodes[0]) + features = [] + for name in sorted(api.Fall()): + if name == "otype" or name in config.excluded_features: + continue + fObj = api.Fs(name) + if any(fObj.v(n) not in (None, "") for n in sample): + features.append(name) + perAtom = template_constraints(api, job.template) + mentioned = perAtom[position] if position < len(perAtom) else [] + inTemplate = [name for (name, _) in mentioned if name in features] + # features the template leaves open (`lex*`) are the interesting ones to count + openFeatures = [ + name for (name, fixed) in mentioned if not fixed and name in features + ] + return { + "position": position, + "type": nType, + "features": features, + "inTemplate": inTemplate, + "open": openFeatures, + } + + @router.get("/searches/{search_id}/stats/values") + def stats_values( + search_id: str, + feature: str, + position: int = Query(0, ge=0), + top: int = Query(25, ge=1, le=500), + ) -> dict[str, Any]: + """How often each value of a feature occurs at one position of the results. + + Counts distinct nodes, so a word that occurs in several results counts + once. Values the corpus marks as "no value" (NA, none, ...) are counted + separately. + """ + job = search_job(search_id) + api = corpus_api(job.corpus) + if feature not in api.Fall() or feature == "otype": + raise HTTPException(404, f"Unknown feature: {feature}") + config = service.config(job.corpus) + nodes = set(position_nodes(job, position)) + fObj = api.Fs(feature) + counts: Counter[str] = Counter() + empty = 0 + for n in nodes: + value = fObj.v(n) + if value is None or value == "" or value in config.none_values: + empty += 1 + else: + counts[str(value)] += 1 + ranked = counts.most_common() + shown = ranked[:top] + rest = ranked[top:] + return { + "search": job.snapshot(), + "position": position, + "type": api.F.otype.v(next(iter(nodes))) if nodes else None, + "feature": feature, + "nodes": len(nodes), + "withoutValue": empty, + "distinct": len(counts), + "values": [{"value": v, "count": c} for (v, c) in shown], + "other": {"values": len(rest), "count": sum(c for (_, c) in rest)}, + } + + @router.get("/searches/{search_id}/stats/sections") + def stats_sections( + search_id: str, + level: int = Query(0, ge=0, le=2), + lang: str = "en", + ) -> dict[str, Any]: + """Results per section (book, chapter, ...) of their first node, in text order.""" + job = search_job(search_id) + api = corpus_api(job.corpus) + config = service.config(job.corpus) + T, L = api.T, api.L + sectionTypes = list(T.sectionTypes or ()) + if level >= len(sectionTypes): + raise HTTPException(404, "This corpus has no sections at that level") + sType = sectionTypes[level] + slotType = api.F.otype.slotType + eOslots = api.E.oslots.s + counts: Counter[int] = Counter() + for row in job.page(0, job.count): + first = row[0] + if api.F.otype.v(first) == sType: + counts[first] += 1 + continue + # the section of the node's first slot, as in references; a node + # that crosses sections lies in none of them as a whole + slot = first if api.F.otype.v(first) == slotType else int(eOslots(first)[0]) + ups = L.u(slot, otype=sType) + if ups: + counts[int(ups[0])] += 1 + sections = [ + {**section_item(api, config, n, lang), "count": counts[n]} + for n in sorted(counts) + ] + return { + "search": job.snapshot(), + "level": level, + "type": sType, + "sections": sections, + } + + @router.get("/searches/{search_id}/export.tsv") + def export_tsv( + search_id: str, + fmt: str | None = None, + lang: str = "en", + excel: bool = Query( + False, + description="UTF-16 with byte-order mark, which Excel opens correctly", + ), + extra: Annotated[ + list[str] | None, + Query( + description="More feature columns for one part of the results:" + " `3:lex,gloss` adds lex and gloss for the third node of each" + " result. Can be given several times." + ), + ] = None, + ) -> Response: + """All results as tab-separated values, one row per result. + + Columns: result number, the section of the result's first node, and for + each node of the result its number, type and text, followed by the + features mentioned for it in the template (use `feature*` in a + template to add a column without restricting the search), and then + any `extra` features asked for. + """ + job = search_job(search_id) + if job.state not in FINISHED: + raise HTTPException(409, "The search has not finished yet") + api = corpus_api(job.corpus) + fmt = formats.chosen(service.config(job.corpus), fmt) + F, T, Fs = api.F, api.T, api.Fs + sectionTypes = list(T.sectionTypes or ()) + rows = job.page(0, job.count) + width = max((len(r) for r in rows), default=0) + perAtom = template_features(api, job.template) + features = [list(perAtom[i]) if i < len(perAtom) else [] for i in range(width)] + nodeFeatures = set(api.Fall()) - {"otype"} + for spec in extra or (): + (pos, _, names) = spec.partition(":") + if not pos.strip().isdigit() or not 1 <= int(pos) <= width: + raise HTTPException( + 422, f"Not a part of the results: {spec!r} (use 1 to {width})" + ) + for name in (n.strip() for n in names.split(",")): + if not name: + continue + if name not in nodeFeatures: + raise HTTPException(422, f"Unknown feature: {name}") + if name not in features[int(pos) - 1]: + features[int(pos) - 1].append(name) + + header = ["R"] + [f"S{i + 1}" for i in range(len(sectionTypes))] + for i in range(width): + header += [f"NODE{i + 1}", f"TYPE{i + 1}", f"TEXT{i + 1}"] + header += [f"{name}{i + 1}" for name in features[i]] + lines = ["\t".join(header)] + for r, tup in enumerate(rows, 1): + section = T.sectionFromNode(tup[0], lang=lang) if sectionTypes else () + cells = [str(r)] + cells += [tsvEsc("" if s is None else s) for s in (section or ())] + cells += [""] * (1 + len(sectionTypes) - len(cells)) + for i, n in enumerate(tup): + small = ( + F.otype.v(n) == F.otype.slotType or len(api.E.oslots.s(n)) <= 100 + ) + text = T.text(n, fmt=fmt) if small else "" + cells += [str(n), F.otype.v(n), tsvEsc(text)] + for name in features[i]: + value = Fs(name).v(n) + cells.append("" if value is None else tsvEsc(value)) + lines.append("\t".join(cells)) + content = "\n".join(lines) + "\n" + suffix = "-excel" if excel else "" + return tsv_response(content, f"{job.corpus}-results-{job.id}{suffix}", excel) + + # the lexicon (BHSA): its lexemes, and per lexeme the verses it occurs in + + return router diff --git a/libs/explorer/cfabric_browser/summary.py b/libs/explorer/cfabric_browser/summary.py new file mode 100644 index 0000000..8d93e3a --- /dev/null +++ b/libs/explorer/cfabric_browser/summary.py @@ -0,0 +1,202 @@ +"""The essentials of a word, for the card shown when hovering over it. + +Of all the features a word has (the BHSA has about 70), only these are shown: +its gloss, lexeme, part of speech and morphology, in words rather than codes. +Each is read from the first feature of its role that has a value, so that the +BHSA, the SP and the DSS all work. The DSS has its own analysis and the ETCBC's +(features ending in `_etcbc`, as in the BHSA): the ETCBC's comes first, and the +DSS's own where the ETCBC has nothing. Values that mean "not applicable" (`NA`, +`absent`, `n/a`) are left out. + +The DSS has no glosses; they come from the lexicon of another open corpus (the +BHSA), by the ETCBC lexeme (`lex_etcbc`) and language (`lang_etcbc`). +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +# label: candidate features, the first one with a value is used +ROLES: list[tuple[str, tuple[str, ...]]] = [ + ("gloss", ("gloss",)), + ("lexeme", ("voc_lex_utf8", "lex_utf8_etcbc", "glex", "lex_utf8")), + ("transliteration", ("lex_etcbc", "lex")), + ("part of speech", ("sp_etcbc", "sp")), + ("stem", ("vs_etcbc", "vs")), + ("tense", ("vt_etcbc", "vt")), + ("person", ("ps_etcbc", "ps")), + ("gender", ("gn_etcbc", "gn")), + ("number", ("nu_etcbc", "nu")), + ("state", ("st_etcbc", "st")), + ("language", ("lang_etcbc", "language")), + # a translation's word, as it is searched (`cfabric_browser.translations`) + ("word", ("lower",)), +] + +# a word's lexeme and language as in the BHSA, to find its gloss there +LEXEME = ("lex_etcbc", "lex") +LANGUAGE = ("lang_etcbc", "language") + +GlossFor = Callable[[str, str | None], str | None] + +# the pronominal suffix: its person, gender and number, shown together +SUFFIX = (("person", "prs_ps"), ("gender", "prs_gn"), ("number", "prs_nu")) + +EMPTY = {"", "NA", "na", "n/a", "absent", "none", "unknown", "0"} + +PARTS = { + "subs": "noun", + "nmpr": "proper noun", + "verb": "verb", + "adjv": "adjective", + "advb": "adverb", + "prep": "preposition", + "conj": "conjunction", + "prps": "personal pronoun", + "prde": "demonstrative pronoun", + "prin": "interrogative pronoun", + "intj": "interjection", + "nega": "negation", + "inrg": "interrogative", + "art": "article", + "ptcl": "particle", + "pron": "pronoun", + "numr": "numeral", + "suff": "suffix", +} +VALUES: dict[str, dict[str, str]] = { + "part of speech": PARTS, + "tense": { + "perf": "perfect", + "impf": "imperfect", + "wayq": "wayyiqtol", + "wayy": "wayyiqtol", + "impv": "imperative", + "infa": "infinitive absolute", + "infc": "infinitive construct", + "ptca": "participle", + "ptcp": "passive participle", + }, + "person": { + "p1": "1st", + "p2": "2nd", + "p3": "3rd", + "1": "1st", + "2": "2nd", + "3": "3rd", + }, + "gender": {"m": "masculine", "f": "feminine", "c": "common", "b": "both"}, + "number": { + "sg": "singular", + "pl": "plural", + "du": "dual", + "s": "singular", + "p": "plural", + "d": "dual", + }, + "state": {"a": "absolute", "c": "construct", "e": "emphatic"}, +} + +# the letters and vowel points of Hebrew; accents and other marks in a lexeme +# (the BHSA's homonym mark ֜, the SP's /) are left out +HEBREW = set(range(0x05B0, 0x05C8)) | set(range(0x05D0, 0x05EB)) + + +FINALS = {"כ": "ך", "מ": "ם", "נ": "ן", "פ": "ף", "צ": "ץ"} + + +def _clean_hebrew(text: str) -> str: + """Hebrew letters and points only, with a final form at the end (the SP + writes its lexemes without: אלהימ).""" + chars = [c for c in text if ord(c) in HEBREW and c not in "׀׃׆"] + letters = [i for i, c in enumerate(chars) if "א" <= c <= "ת"] + if letters and chars[letters[-1]] in FINALS: + chars[letters[-1]] = FINALS[chars[letters[-1]]] + return "".join(chars).strip() + + +def _value(api: Any, name: str, node: int) -> Any: + feature = getattr(api.F, name, None) + return None if feature is None else feature.v(node) + + +def _first( + api: Any, node: int, names: tuple[str, ...], empty: set[str] +) -> tuple[str, str] | None: + """The first of these features with a value: (feature, value).""" + for name in names: + value = _value(api, name, node) + if value is not None and str(value).strip() not in empty: + return (name, str(value).strip()) + return None + + +def lexeme_of( + api: Any, node: int, empty: set[str] | None = None +) -> tuple[str, str | None] | None: + """A word's ETCBC lexeme (`BR>[`) and language, as in the BHSA, or None.""" + empty = EMPTY | (empty or set()) + lex = _first(api, node, LEXEME, empty) + if not lex or not lex[1].isascii(): + return None + language = _first(api, node, LANGUAGE, empty) + return (lex[1], language[1] if language else None) + + +def summary( + api: Any, + node: int, + empty: set[str] | None = None, + gloss_for: GlossFor | None = None, + gloss_source: str = "lexicon", +) -> list[dict[str, str]]: + """The essentials of a node: a list of `{label, value, feature}`. + + `gloss_for(lex, language)` gives the gloss of an ETCBC lexeme from another + corpus (`gloss_source`), for a corpus without glosses (the DSS). + """ + empty = EMPTY | (empty or set()) + items = [] + for label, candidates in ROLES: + for name in candidates: + value = _value(api, name, node) + if value is None: + continue + text = str(value).strip() + if label == "lexeme": + text = _clean_hebrew(text) + elif label == "transliteration" and not text.isascii(): + # the DSS's `lex` is in Hebrew: the transliteration is `lex_etcbc` + continue + if text in empty: + # nothing here: try the next feature of this role + continue + if label == "language" and text == "Hebrew": + # the language only where it is not Hebrew + break + items.append( + { + "label": label, + "value": VALUES.get(label, {}).get(text, text), + "feature": name, + } + ) + break + if gloss_for and not any(item["label"] == "gloss" for item in items): + lex = _first(api, node, LEXEME, empty) + if lex and lex[1].isascii(): + language = _first(api, node, LANGUAGE, empty) + gloss = gloss_for(lex[1], language[1] if language else None) + if gloss: + items.insert( + 0, {"label": "gloss", "value": gloss, "feature": gloss_source} + ) + suffix = [] + for role, name in SUFFIX: + value = _value(api, name, node) + if value is not None and str(value) not in empty: + suffix.append(VALUES[role].get(str(value), str(value))) + if suffix: + items.append({"label": "suffix", "value": " ".join(suffix), "feature": "prs"}) + return items diff --git a/libs/explorer/cfabric_browser/translations/__init__.py b/libs/explorer/cfabric_browser/translations/__init__.py new file mode 100644 index 0000000..5452bdb --- /dev/null +++ b/libs/explorer/cfabric_browser/translations/__init__.py @@ -0,0 +1,12 @@ +"""Translations that can be added to the browser, such as the JPS 1917. + +A translation is downloaded once, checked and built into an ordinary corpus +in Text-Fabric layout, so that every interface can read, align and search it +like the Hebrew texts. See `catalog.py` for what is known about each source +and `build.py` for the steps. +""" + +from cfabric_browser.translations.build import BuildError, build +from cfabric_browser.translations.catalog import TRANSLATIONS, Split, Translation + +__all__ = ["TRANSLATIONS", "BuildError", "Split", "Translation", "build"] diff --git a/libs/explorer/cfabric_browser/translations/build.py b/libs/explorer/cfabric_browser/translations/build.py new file mode 100644 index 0000000..2ec8b93 --- /dev/null +++ b/libs/explorer/cfabric_browser/translations/build.py @@ -0,0 +1,505 @@ +"""Building a translation into a corpus. + +The steps, each of which stops the build with a `BuildError` when the source +is not what is expected: + +1. `fetch`: download the books, or take them from the cache when their + checksums still match. +2. `read_book`: check the title and licence the source gives, and get the + verses per chapter. +3. `clean`: remove the source's markup. A note in `` (a verse repeated + at the end of a book, as read in the synagogue) is kept apart; any other + markup stops the build, so that nothing slips in unseen. +4. `renumber`: divide verses as the BHSA does (`Split`), and check the + number of chapters and verses against the BHSA's. +5. `write_corpus`: words as slots, with `text`, `trailer` and `lower`; + books, chapters and verses as sections; metadata in every feature and in + `app/config.yaml`. + +The same source gives the same corpus: only the date each feature file was +written (`@dateWritten`) differs between builds. +""" + +from __future__ import annotations + +import html +import json +import re +import shutil +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from importlib import resources +from pathlib import Path +from typing import Any + +import yaml + +from cfabric_browser import __version__ +from cfabric_browser.display.config import read_app_config +from cfabric_browser.jobs import Progress +from cfabric_browser.net import CORPUS_ROOT, read_url, sha256_bytes +from cfabric_browser.translations.catalog import Split, Translation + +CACHE = "~/.cfabric/translations" +ROOT = CORPUS_ROOT + +# called with a step ("download", "build"), the number done and the total +Fetch = Callable[[str], bytes] + + +class BuildError(Exception): + """The source is not what the builder expects; the message says where.""" + + +@dataclass +class Verse: + chapter: int + verse: int + text: str + note: str | None = None + # the source's own reference, where the BHSA divides differently + printed: str | None = None + + +def hebrew_verses() -> dict[str, list[int]]: + """The number of verses per chapter of each book in the BHSA.""" + data = resources.files("cfabric_browser.translations").joinpath( + "hebrew_verses.json" + ) + return json.loads(data.read_text(encoding="utf-8")) + + +def fetch( + translation: Translation, + cache: str | Path = CACHE, + progress: Progress | None = None, + download: Fetch = read_url, +) -> dict[str, bytes]: + """The source file of every book, keyed by the BHSA's book name. + + Files are kept in `//` with their checksums in + `checksums.json`; a file whose checksum still matches is not downloaded + again. + """ + folder = Path(cache).expanduser() / translation.id + folder.mkdir(parents=True, exist_ok=True) + sums_file = folder / "checksums.json" + try: + sums: dict[str, str] = json.loads(sums_file.read_text(encoding="utf-8")) + except (OSError, ValueError): + sums = {} + + files: dict[str, bytes] = {} + total = len(translation.books) + for i, (name, path) in enumerate(translation.books): + target = folder / f"{name}.json" + data: bytes | None = None + if target.is_file() and name in sums: + data = target.read_bytes() + if sha256_bytes(data) != sums[name]: + data = None + if data is None: + url = translation.url.format(path=urllib.parse.quote(path)) + try: + data = download(url) + except OSError as e: + raise BuildError(f"Could not download {name} from {url}: {e}") from e + target.write_bytes(data) + sums[name] = sha256_bytes(data) + sums_file.write_text(json.dumps(sums, indent=1), encoding="utf-8") + files[name] = data + if progress is not None: + progress("download", i + 1, total) + return files + + +def read_book(translation: Translation, name: str, data: bytes) -> list[list[str]]: + """The verses per chapter of a book's source file, after checking that the + file is the expected version under the expected licence.""" + try: + source = json.loads(data.decode("utf-8")) + except ValueError as e: + raise BuildError(f"{name}: the source is not readable JSON ({e})") from e + title = source.get("versionTitle") + if title != translation.source_title: + raise BuildError( + f"{name}: the source is '{title}', not '{translation.source_title}'" + ) + licence = source.get("license") + if licence != translation.source_licence: + raise BuildError( + f"{name}: the source's licence is '{licence}', " + f"not '{translation.source_licence}'" + ) + text = source.get("text") + if not isinstance(text, list) or not all( + isinstance(ch, list) and all(isinstance(v, str) for v in ch) for ch in text + ): + raise BuildError(f"{name}: the source has no chapters with verses") + return text + + +NOTE = re.compile(r"\s*\s*\s*\[?(.*?)\]?\s*\s*$", re.DOTALL) +TAG = re.compile(r"<[^>]*>") + + +def clean(ref: str, raw: str) -> tuple[str, str | None]: + """A verse's text without markup, and its note in ``, if any.""" + note = None + match = NOTE.search(raw) + if match: + note = " ".join(html.unescape(match.group(1)).split()) + raw = raw[: match.start()] + tag = TAG.search(raw) + if tag: + raise BuildError(f"{ref}: unexpected markup {tag.group(0)!r}") + text = " ".join(html.unescape(raw).split()) + if not text: + raise BuildError(f"{ref}: the verse is empty") + return (text, note) + + +def _cut(ref: str, text: str, before: tuple[str, ...]) -> list[str]: + parts = [] + rest = text + for start in before: + at = rest.find(start) + if at <= 0: + raise BuildError(f"{ref}: cannot divide the verse before {start!r}") + parts.append(rest[:at].rstrip()) + rest = rest[at:] + parts.append(rest) + return parts + + +def renumber( + book: str, + chapters: list[list[tuple[str, str | None]]], + splits: tuple[Split, ...] = (), +) -> list[list[Verse]]: + """Verses per chapter, divided as in the BHSA. + + `chapters` holds (text, note) per verse in the source's numbering. + """ + mine = {(s.chapter, s.verse): s for s in splits if s.book == book} + result: list[list[Verse]] = [] + for c, verses in enumerate(chapters, start=1): + current: list[Verse] = [] + for v, (text, note) in enumerate(verses, start=1): + printed = f"{c}:{v}" + split = mine.pop((c, v), None) + if split is None: + current.append(Verse(c, 0, text, note, printed)) + continue + parts = _cut(f"{book} {printed}", text, split.before) + pieces = [Verse(c, 0, part, None, printed) for part in parts] + pieces[-1].note = note + if split.to_previous_chapter: + if not result: + raise BuildError(f"{book} {printed}: there is no previous chapter") + first = pieces.pop(0) + first.chapter = c - 1 + result[-1].append(first) + current.extend(pieces) + result.append(current) + if mine: + missing = ", ".join(f"{c}:{v}" for (c, v) in mine) + raise BuildError(f"{book}: no verse {missing} to divide") + + divided = {(s.chapter, s.verse) for s in splits if s.book == book} + for verses in result: + for v, verse in enumerate(verses, start=1): + verse.verse = v + own = f"{verse.chapter}:{v}" + source = tuple(int(x) for x in verse.printed.split(":")) # type: ignore[union-attr] + if verse.printed == own and source not in divided: + verse.printed = None + return result + + +def check_counts(book: str, chapters: list[list[Verse]], expected: list[int]) -> None: + """Stop when the chapters or verses differ from the BHSA's.""" + counts = [len(verses) for verses in chapters] + if len(counts) != len(expected): + raise BuildError( + f"{book}: {len(counts)} chapters, but the BHSA has {len(expected)}" + ) + wrong = [ + f"{c} has {n} verses, not {e}" + for (c, (n, e)) in enumerate(zip(counts, expected), start=1) + if n != e + ] + if wrong: + raise BuildError(f"{book}: chapter " + "; ".join(wrong)) + + +WORD = re.compile(r"([^\s—]+)([\s—]*)") +LAST_LETTER = re.compile(r"^(.*[^\W_])(.*)$", re.DOTALL) + + +def tokenize(text: str) -> list[tuple[str, str]]: + """Words with what follows them (punctuation and space), so that joining + them gives the text again. A word keeps punctuation before it (`‘Speak`, + `[into`); punctuation after it goes into its trailer.""" + words: list[tuple[str, str]] = [] + prefix = "" + for match in WORD.finditer(text): + (chunk, space) = match.groups() + letters = LAST_LETTER.match(chunk) + if letters is None: + # punctuation on its own: with the word before it, else the next + if words: + (w, t) = words[-1] + words[-1] = (w, t + chunk + space) + else: + prefix += chunk + space + continue + words.append((prefix + letters.group(1), letters.group(2) + space)) + prefix = "" + if prefix: + if not words: + raise ValueError(f"no words in {text!r}") + (w, t) = words[-1] + words[-1] = (w, t + prefix) + return words + + +def lower(word: str) -> str: + """A word for searching: lower case, without punctuation (`‘Speak` → + `speak`), keeping apostrophes and hyphens inside it (`jacob's`, + `man-servant`).""" + plain = re.sub(r"[^\w'-]", "", word.replace("’", "'")).strip("'-") + return plain.lower() + + +def write_corpus( + translation: Translation, + books: dict[str, list[list[Verse]]], + repo: Path, + checksum: str, +) -> Path: + """Write the corpus in Text-Fabric layout under `repo`; returns the data + directory (`/tf/`).""" + from cfabric import Fabric + + otype: dict[int, str] = {} + oslots: dict[int, set[int]] = {} + feats: dict[str, dict[int, Any]] = { + name: {} + for name in ( + "text", + "trailer", + "lower", + "book", + "chapter", + "verse", + "note", + "printed", + ) + } + + slot = 0 + spans: list[tuple[str, Any, int, int, Verse | None]] = [] + for book, chapters in books.items(): + bookStart = slot + 1 + for c, verses in enumerate(chapters, start=1): + chapterStart = slot + 1 + for verse in verses: + verseStart = slot + 1 + words = tokenize(verse.text) + for i, (w, t) in enumerate(words): + slot += 1 + otype[slot] = "word" + feats["text"][slot] = w + # a space between verses, for reading them as running text + feats["trailer"][slot] = ( + t if (i < len(words) - 1 or t.endswith(" ")) else t + " " + ) + feats["lower"][slot] = lower(w) + spans.append(("verse", (book, c, verse.verse), verseStart, slot, verse)) + spans.append(("chapter", (book, c), chapterStart, slot, None)) + spans.append(("book", (book,), bookStart, slot, None)) + + maxSlot = slot + node = maxSlot + for kind in ("book", "chapter", "verse"): + for k, ref, first, last, verse in spans: + if k != kind: + continue + node += 1 + otype[node] = kind + oslots[node] = set(range(first, last + 1)) + feats["book"][node] = ref[0] + if kind != "book": + feats["chapter"][node] = ref[1] + if kind == "verse": + feats["verse"][node] = ref[2] + assert verse is not None + if verse.note: + feats["note"][node] = verse.note + if verse.printed: + feats["printed"][node] = verse.printed + + generic = { + "name": translation.title, + "title": translation.cite, + "language": translation.language, + "licence": translation.licence, + "source": translation.source, + "sourceChecksum": checksum, + "createdBy": f"cfabric-browser {__version__}", + } + described = { + "otype": "the type of a node: book, chapter, verse or word", + "oslots": "the words of a book, chapter or verse", + "text": "the word as printed, with any punctuation before it", + "trailer": "punctuation and space after the word", + "lower": "the word in lower case without punctuation, for searching", + "book": "the book, by its English name in the BHSA", + "chapter": "the chapter, as in the BHSA", + "verse": "the verse, as in the BHSA", + "note": "a note printed after the verse in small type, such as the verse " + "repeated at the end of Malachi", + "printed": "the chapter and verse in the 1917 print, where the BHSA " + "divides the verses differently", + } + metaData: dict[str, dict[str, Any]] = {"": generic} + for name, description in described.items(): + isInt = name in ("chapter", "verse") + metaData[name] = { + "description": description, + "valueType": "int" if isInt else "str", + } + metaData["otext"] = { + "fmt:text-orig-full": "{text}{trailer}", + "sectionTypes": "book,chapter,verse", + "sectionFeatures": "book,chapter,verse", + } + + data = repo / "tf" / translation.version + data.mkdir(parents=True, exist_ok=True) + CF = Fabric(locations=str(data), silent="deep") + good = CF.save( + nodeFeatures={"otype": otype, **{k: v for k, v in feats.items() if v}}, + edgeFeatures={"oslots": oslots}, + metaData=metaData, + location=str(data), + module="", + silent="deep", + ) + if not good: + raise BuildError("Could not write the corpus") + + app = repo / "app" + app.mkdir(parents=True, exist_ok=True) + config = { + "apiVersion": 3, + "writing": translation.language, + "provenanceSpec": { + "corpus": f"{translation.title} = {translation.name}", + "org": translation.org, + "repo": translation.repo, + "version": translation.version, + }, + "translation": { + "id": translation.id, + "licence": translation.licence, + "licenceNote": translation.licence_note, + "source": translation.source, + "cite": translation.cite, + "builder": f"cfabric-browser {__version__}", + "sourceChecksum": checksum, + }, + } + (app / "config.yaml").write_text( + yaml.safe_dump(config, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + return data + + +def prepare( + translation: Translation, files: dict[str, bytes] +) -> dict[str, list[list[Verse]]]: + """Check and clean the source files and divide them as in the BHSA.""" + expected = hebrew_verses() + books: dict[str, list[list[Verse]]] = {} + for name, _ in translation.books: + if name not in files: + raise BuildError(f"{name} is missing from the source") + chapters = read_book(translation, name, files[name]) + cleaned = [ + [clean(f"{name} {c}:{v}", raw) for v, raw in enumerate(verses, start=1)] + for c, verses in enumerate(chapters, start=1) + ] + divided = renumber(name, cleaned, translation.splits) + if name in expected: + check_counts(name, divided, expected[name]) + books[name] = divided + return books + + +def is_ours(repo: Path, translation: Translation) -> bool: + """Whether `repo` was built as this translation (its app config says so).""" + config = repo / "app" / "config.yaml" + if not config.is_file(): + return False + try: + own = read_app_config(config).get("translation") or {} + except Exception: # noqa: BLE001 - not readable, so not ours to replace + return False + return own.get("id") == translation.id + + +def build( + translation: Translation, + root: str | Path = ROOT, + cache: str | Path = CACHE, + progress: Progress | None = None, + download: Fetch = read_url, +) -> Path: + """Download, check and build a translation; returns its data directory. + + The corpus is written next to the place it goes, then moved there, so an + interrupted build leaves an earlier one intact. + """ + files = fetch(translation, cache=cache, progress=progress, download=download) + books = prepare(translation, files) + if progress is not None: + progress("build", 0, 1) + checksum = sha256_bytes( + "".join( + f"{name}:{sha256_bytes(files[name])}\n" for name, _ in translation.books + ).encode() + ) + + org = Path(root).expanduser() / translation.org + org.mkdir(parents=True, exist_ok=True) + final = org / translation.repo + building = org / f".{translation.repo}.building" + old = org / f".{translation.repo}.old" + # an earlier build that stopped between its two renames + if old.exists() and not final.exists() and is_ours(old, translation): + old.rename(final) + if final.exists() and not is_ours(final, translation): + raise BuildError( + f"{final} exists but was not built as {translation.id}; move it away first" + ) + for leftover in (building, old): + if leftover.exists(): + shutil.rmtree(leftover) + write_corpus(translation, books, building, checksum) + if final.exists(): + final.rename(old) + try: + building.rename(final) + except OSError: + if old.exists() and not final.exists(): + old.rename(final) + raise + if old.exists(): + shutil.rmtree(old, ignore_errors=True) + if progress is not None: + progress("build", 1, 1) + return final / "tf" / translation.version diff --git a/libs/explorer/cfabric_browser/translations/catalog.py b/libs/explorer/cfabric_browser/translations/catalog.py new file mode 100644 index 0000000..e5400fe --- /dev/null +++ b/libs/explorer/cfabric_browser/translations/catalog.py @@ -0,0 +1,164 @@ +"""The translations the browser can add, and what is known about their sources. + +A translation is described by a `Translation`: where its books are, what the +source must say about itself (title, licence), and the few places where its +verse division differs from the Hebrew Bible's in the BHSA (`Split`). The +builder (`build.py`) turns it into an ordinary corpus. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class Split: + """A source verse that the BHSA divides into several verses. + + The verse `chapter:verse` (in the source's numbering) is cut before each + text in `before`; the following verses of the chapter move up. With + `to_previous_chapter`, the first part is the last verse of the previous + chapter instead (Numbers 25:19, which the source joins to 26:1). + """ + + book: str + chapter: int + verse: int + before: tuple[str, ...] + to_previous_chapter: bool = False + + +@dataclass(frozen=True) +class Translation: + """A translation that can be downloaded and built into a corpus.""" + + id: str + # short, as a column heading, and in full + title: str + name: str + language: str + # where the corpus goes, in Text-Fabric layout: //tf/ + org: str + repo: str + version: str + # what the source must say about itself + source_title: str + source_licence: str + # the source's address per book, with {path} for a book's path + url: str + # (BHSA English name, path at the source), in the BHSA's order + books: tuple[tuple[str, str], ...] + licence: str + licence_note: str + source: str + cite: str + splits: tuple[Split, ...] = field(default_factory=tuple) + + @property + def corpus_id(self) -> str: + return f"{self.org}/{self.repo}" + + +SEFARIA = "https://storage.googleapis.com/sefaria-export/json/Tanakh/{path}" + +JPS_FILE = "English/The Holy Scriptures A New Translation JPS 1917.json" + +JPS_BOOKS = tuple( + (name, f"{section}/{book}/{JPS_FILE}") + for (name, section, book) in ( + ("Genesis", "Torah", "Genesis"), + ("Exodus", "Torah", "Exodus"), + ("Leviticus", "Torah", "Leviticus"), + ("Numbers", "Torah", "Numbers"), + ("Deuteronomy", "Torah", "Deuteronomy"), + ("Joshua", "Prophets", "Joshua"), + ("Judges", "Prophets", "Judges"), + ("1_Samuel", "Prophets", "I Samuel"), + ("2_Samuel", "Prophets", "II Samuel"), + ("1_Kings", "Prophets", "I Kings"), + ("2_Kings", "Prophets", "II Kings"), + ("Isaiah", "Prophets", "Isaiah"), + ("Jeremiah", "Prophets", "Jeremiah"), + ("Ezekiel", "Prophets", "Ezekiel"), + ("Hosea", "Prophets", "Hosea"), + ("Joel", "Prophets", "Joel"), + ("Amos", "Prophets", "Amos"), + ("Obadiah", "Prophets", "Obadiah"), + ("Jonah", "Prophets", "Jonah"), + ("Micah", "Prophets", "Micah"), + ("Nahum", "Prophets", "Nahum"), + ("Habakkuk", "Prophets", "Habakkuk"), + ("Zephaniah", "Prophets", "Zephaniah"), + ("Haggai", "Prophets", "Haggai"), + ("Zechariah", "Prophets", "Zechariah"), + ("Malachi", "Prophets", "Malachi"), + ("Psalms", "Writings", "Psalms"), + ("Job", "Writings", "Job"), + ("Proverbs", "Writings", "Proverbs"), + ("Ruth", "Writings", "Ruth"), + ("Song_of_songs", "Writings", "Song of Songs"), + ("Ecclesiastes", "Writings", "Ecclesiastes"), + ("Lamentations", "Writings", "Lamentations"), + ("Esther", "Writings", "Esther"), + ("Daniel", "Writings", "Daniel"), + ("Ezra", "Writings", "Ezra"), + ("Nehemiah", "Writings", "Nehemiah"), + ("1_Chronicles", "Writings", "I Chronicles"), + ("2_Chronicles", "Writings", "II Chronicles"), + ) +) + +JPS1917 = Translation( + id="jps1917", + title="JPS 1917", + name="The Holy Scriptures: A New Translation (Jewish Publication Society, 1917)", + language="eng", + org="Context-Fabric", + repo="jps1917", + version="1917", + source_title="The Holy Scriptures: A New Translation (JPS 1917)", + source_licence="Public Domain", + url=SEFARIA, + books=JPS_BOOKS, + licence="Public domain in the United States", + licence_note=( + "Published in 1917, so in the public domain in the United States. " + "Elsewhere it depends on the translators' dates; check the rules in " + "your country." + ), + source=( + "Sefaria's export (sefaria.org), from the Open Siddur Project's " + "scans of the 1917 print, which JPS shared" + ), + cite=( + "The Holy Scriptures According to the Masoretic Text: A New " + "Translation (Philadelphia: Jewish Publication Society, 1917), " + "from Sefaria's export" + ), + splits=( + # the Ten Commandments: the BHSA has one verse per commandment + Split( + "Exodus", + 20, + 13, + ( + "Thou shalt not commit adultery.", + "Thou shalt not steal.", + "Thou shalt not bear false witness", + ), + ), + Split( + "Deuteronomy", + 5, + 17, + ( + "Neither shalt thou commit adultery.", + "Neither shalt thou steal.", + "Neither shalt thou bear false witness", + ), + ), + Split("Numbers", 26, 1, ("that the LORD spoke",), to_previous_chapter=True), + ), +) + +TRANSLATIONS: dict[str, Translation] = {t.id: t for t in (JPS1917,)} diff --git a/libs/explorer/cfabric_browser/translations/hebrew_verses.json b/libs/explorer/cfabric_browser/translations/hebrew_verses.json new file mode 100644 index 0000000..f02757d --- /dev/null +++ b/libs/explorer/cfabric_browser/translations/hebrew_verses.json @@ -0,0 +1,41 @@ +{ + "Genesis": [31,25,24,26,32,22,24,22,29,32,32,20,18,24,21,16,27,33,38,18,34,24,20,67,34,35,46,22,35,43,54,33,20,31,29,43,36,30,23,23,57,38,34,34,28,34,31,22,33,26], + "Exodus": [22,25,22,31,23,30,29,28,35,29,10,51,22,31,27,36,16,27,25,26,37,30,33,18,40,37,21,43,46,38,18,35,23,35,35,38,29,31,43,38], + "Leviticus": [17,16,17,35,26,23,38,36,24,20,47,8,59,57,33,34,16,30,37,27,24,33,44,23,55,46,34], + "Numbers": [54,34,51,49,31,27,89,26,23,36,35,16,33,45,41,35,28,32,22,29,35,41,30,25,19,65,23,31,39,17,54,42,56,29,34,13], + "Deuteronomy": [46,37,29,49,33,25,26,20,29,22,32,31,19,29,23,22,20,22,21,20,23,29,26,22,19,19,26,69,28,20,30,52,29,12], + "Joshua": [18,24,17,24,15,27,26,35,27,43,23,24,33,15,63,10,18,28,51,9,45,34,16,33], + "Judges": [36,23,31,24,31,40,25,35,57,18,40,15,25,20,20,31,13,31,30,48,25], + "1_Samuel": [28,36,21,22,12,21,17,22,27,27,15,25,23,52,35,23,58,30,24,42,16,23,28,23,44,25,12,25,11,31,13], + "2_Samuel": [27,32,39,12,25,23,29,18,13,19,27,31,39,33,37,23,29,32,44,26,22,51,39,25], + "1_Kings": [53,46,28,20,32,38,51,66,28,29,43,33,34,31,34,34,24,46,21,43,29,54], + "2_Kings": [18,25,27,44,27,33,20,29,37,36,20,22,25,29,38,20,41,37,37,21,26,20,37,20,30], + "Isaiah": [31,22,26,6,30,13,25,23,20,34,16,6,22,32,9,14,14,7,25,6,17,25,18,23,12,21,13,29,24,33,9,20,24,17,10,22,38,22,8,31,29,25,28,28,25,13,15,22,26,11,23,15,12,17,13,12,21,14,21,22,11,12,19,11,25,24], + "Jeremiah": [19,37,25,31,31,30,34,23,25,25,23,17,27,22,21,21,27,23,15,18,14,30,40,10,38,24,22,17,32,24,40,44,26,22,19,32,21,28,18,16,18,22,13,30,5,28,7,47,39,46,64,34], + "Ezekiel": [28,10,27,17,17,14,27,18,11,22,25,28,23,23,8,63,24,32,14,44,37,31,49,27,17,21,36,26,21,26,18,32,33,31,15,38,28,23,29,49,26,20,27,31,25,24,23,35], + "Hosea": [9,25,5,19,15,11,16,14,17,15,11,15,15,10], + "Joel": [20,27,5,21], + "Amos": [15,16,15,13,27,14,17,14,15], + "Obadiah": [21], + "Jonah": [16,11,10,11], + "Micah": [16,13,12,14,14,16,20], + "Nahum": [14,14,19], + "Habakkuk": [17,20,19], + "Zephaniah": [18,15,20], + "Haggai": [15,23], + "Zechariah": [17,17,10,14,11,15,14,23,17,12,17,14,9,21], + "Malachi": [14,17,24], + "Psalms": [6,12,9,9,13,11,18,10,21,18,7,9,6,7,5,11,15,51,15,10,14,32,6,10,22,12,14,9,11,13,25,11,22,23,28,13,40,23,14,18,14,12,5,27,18,12,10,15,21,23,21,11,7,9,24,14,12,12,18,14,9,13,12,11,14,20,8,36,37,6,24,20,28,23,11,13,21,72,13,20,17,8,19,13,14,17,7,19,53,17,16,16,5,23,11,13,12,9,9,5,8,29,22,35,45,48,43,14,31,7,10,10,9,8,18,19,2,29,176,7,8,9,4,8,5,6,5,6,8,8,3,18,3,3,21,26,9,8,24,14,10,8,12,15,21,10,20,14,9,6], + "Job": [22,13,26,21,27,30,21,22,35,22,20,25,28,22,35,22,16,21,29,29,34,30,17,25,6,14,23,28,25,31,40,22,33,37,16,33,24,41,30,32,26,17], + "Proverbs": [33,22,35,27,23,35,27,36,18,32,31,28,25,35,33,33,28,24,29,30,31,29,35,34,28,28,27,28,27,33,31], + "Ruth": [22,23,18,22], + "Song_of_songs": [17,17,11,16,16,12,14,14], + "Ecclesiastes": [18,26,22,17,19,12,29,17,18,20,10,14], + "Lamentations": [22,22,66,22,22], + "Esther": [22,23,15,17,14,14,10,17,32,3], + "Daniel": [21,49,33,34,30,29,28,27,27,21,45,13], + "Ezra": [11,70,13,24,17,22,28,36,15,44], + "Nehemiah": [11,20,38,17,19,19,72,18,37,40,36,47,31], + "1_Chronicles": [54,55,24,43,41,66,40,40,44,14,47,41,14,17,29,43,27,17,19,8,30,19,32,31,31,32,34,21,30], + "2_Chronicles": [18,17,17,22,14,42,22,18,31,19,23,16,23,14,19,14,19,34,11,37,20,12,21,27,28,23,9,27,36,27,21,33,25,33,27,23] +} diff --git a/libs/explorer/cfabric_browser/translations/manage.py b/libs/explorer/cfabric_browser/translations/manage.py new file mode 100644 index 0000000..ce0724e --- /dev/null +++ b/libs/explorer/cfabric_browser/translations/manage.py @@ -0,0 +1,123 @@ +"""Adding and removing translations, from the library or the command line. + +A translation is added by building it (`build.py`) into the first corpus root +(`~/text-fabric-data/github`), after which the library lists it like any other +corpus. Removing deletes only a directory the builder made (its +`app/config.yaml` names the translation). A corpus that is open cannot be +deleted while the browser runs (on Windows its files are in use), so its +removal waits until the next start (`remove_pending`). +""" + +from __future__ import annotations + +import json +import logging +import shutil +from pathlib import Path +from typing import Any + +from cfabric_browser.jobs import BackgroundJobs +from cfabric_browser.translations.build import CACHE, BuildError, build, is_ours +from cfabric_browser.translations.catalog import TRANSLATIONS, Translation + +logger = logging.getLogger(__name__) + +PENDING = "~/.cfabric/translations/remove.json" + + +def repo_of(translation: Translation, root: str | Path) -> Path: + return Path(root).expanduser() / translation.org / translation.repo + + +def is_built(translation: Translation, root: str | Path) -> bool: + """Whether the translation has been built under `root` (by this builder).""" + repo = repo_of(translation, root) + data = repo / "tf" / translation.version + return (data / "otype.tf").is_file() and is_ours(repo, translation) + + +def _pending(path: str | Path) -> list[str]: + try: + ids = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + return [i for i in ids if isinstance(i, str)] if isinstance(ids, list) else [] + + +def _write_pending(path: str | Path, ids: list[str]) -> None: + target = Path(path).expanduser() + if ids: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(sorted(set(ids))), encoding="utf-8") + elif target.exists(): + target.unlink() + + +def is_pending(translation: Translation, pending: str | Path = PENDING) -> bool: + return translation.id in _pending(pending) + + +def remove(translation: Translation, root: str | Path) -> bool: + """Delete a built translation; False if there is none (made by the builder).""" + repo = repo_of(translation, root) + if not is_ours(repo, translation): + return False + shutil.rmtree(repo) + return True + + +def schedule_removal(translation: Translation, pending: str | Path = PENDING) -> None: + _write_pending(pending, [*_pending(pending), translation.id]) + + +def cancel_removal(translation: Translation, pending: str | Path = PENDING) -> None: + _write_pending(pending, [i for i in _pending(pending) if i != translation.id]) + + +def remove_pending(root: str | Path, pending: str | Path = PENDING) -> list[str]: + """Remove the translations whose removal waited for a restart; returns their + ids. Run at startup, before any corpus is opened.""" + done = [] + left = [] + for tid in _pending(pending): + translation = TRANSLATIONS.get(tid) + if translation is None: + continue + try: + remove(translation, root) + done.append(tid) + except OSError as e: + logger.warning("Could not remove %s: %s", tid, e) + left.append(tid) + _write_pending(pending, left) + return done + + +class Jobs(BackgroundJobs): + """Builds running in the background, one at a time per translation, with + their progress for the library to show.""" + + def __init__( + self, + root: str | Path, + cache: str | Path = CACHE, + pending: str | Path = PENDING, + **build_options: Any, + ) -> None: + super().__init__(expected=(BuildError,)) + self.root = root + self.cache = cache + self.pending = pending # removals waiting for a restart + self.build_options = build_options # e.g. download=, for tests + + def start(self, translation: Translation) -> dict[str, Any]: + return self.run( + translation.id, + lambda progress: build( + translation, + root=self.root, + cache=self.cache, + progress=progress, + **self.build_options, + ), + ) diff --git a/libs/explorer/cfabric_browser/window.py b/libs/explorer/cfabric_browser/window.py new file mode 100644 index 0000000..f99f005 --- /dev/null +++ b/libs/explorer/cfabric_browser/window.py @@ -0,0 +1,81 @@ +"""The browser in a window of its own (pywebview), instead of a web browser.""" + +from __future__ import annotations + +import socket +import sys +import threading +import time +import webbrowser +from pathlib import Path +from typing import Any + +# Web storage of the window (settings, search history, saved workspaces). It is +# tied to the address, so the window uses a fixed port where it can. +STORAGE = Path.home() / ".cfabric" / "window" + + +def free_port(host: str, start: int, tries: int = 20) -> int: + """The first port from `start` on which nothing listens yet.""" + for port in range(start, start + tries): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind((host, port)) + except OSError: + continue + return port + raise OSError(f"No free port in {start}-{start + tries - 1}") + + +def run_window(app: Any, host: str, port: int, url: str) -> int: + """Serve `app` in a background thread and show it in a window until closed.""" + try: + import webview + except ImportError: + print( + "cfabric-browser: --window needs pywebview:" + " pip install cfabric-browser[window]", + file=sys.stderr, + ) + return 2 + import uvicorn + + config = uvicorn.Config( + app, + host=host, + port=port, + log_level="warning", + # without a console (the standalone program) there is nowhere to log to + log_config=None if sys.stdout is None else uvicorn.config.LOGGING_CONFIG, + ) + server = uvicorn.Server(config) + thread = threading.Thread(target=server.run, name="cfabric-server", daemon=True) + thread.start() + while not server.started: + if not thread.is_alive(): + return 1 + time.sleep(0.05) + + webview.settings["ALLOW_DOWNLOADS"] = True # TSV export, saved workspaces + webview.create_window( + "CACCHT Explorer", url, width=1400, height=900, min_size=(640, 480) + ) + STORAGE.mkdir(parents=True, exist_ok=True) + try: + webview.start(private_mode=False, storage_path=str(STORAGE)) + except Exception as e: # noqa: BLE001 + # no window toolkit (on Linux pywebview needs GTK or Qt; on Windows .NET + # may refuse to load it), or the window failed: use a web browser + print(f"No window possible ({e}); opening a web browser instead.", flush=True) + print(f"CACCHT Explorer at {url} (Ctrl+C to stop)", flush=True) + webbrowser.open(url) + try: + while thread.is_alive(): + thread.join(timeout=0.5) + except KeyboardInterrupt: + pass + + # the window is closed: stop the server, which stops the search workers + server.should_exit = True + thread.join(timeout=15) + return 0 diff --git a/libs/explorer/cfabric_browser/witnesses.py b/libs/explorer/cfabric_browser/witnesses.py new file mode 100644 index 0000000..b192cf0 --- /dev/null +++ b/libs/explorer/cfabric_browser/witnesses.py @@ -0,0 +1,254 @@ +"""Reading a corpus of manuscripts by biblical book. + +In the Dead Sea Scrolls the sections are scroll, fragment and line. Biblical +text is identified at the word level instead: words carry the features +`book`, `chapter` and `verse`. A `BookIndex` turns those into a second way to +read the corpus: per verse, the scrolls that contain it (its *witnesses*), +each with the words of that verse. + +A corpus gets a book index when its words have these features with values +that are biblical books (`Gen`, `Is`, …; see `books.py`) and its own top +sections are not books. Words whose `book` is not a biblical book (in the +DSS: words of biblical scrolls not identified with a verse, which carry the +scroll name) are left out. + +Book view references use the English book names of `books.py` +(`Genesis 1`, `Genesis 1:1`), so they match those of MT and SP; any name the +concordance knows is accepted (`Gen 1:1`). +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +from cfabric_browser.books import BOOKS, canonical_book + +WORD = "word" +BOOK, CHAPTER, VERSE = "book", "chapter", "verse" +RECONSTRUCTED = "rec" # on signs: reconstructed by a modern editor +BIBLICAL = "biblical" # 2: also in Abegg's non-biblical file (DSS) + +ORDER = {book: i for (i, book) in enumerate(BOOKS)} +SIGLUM = re.compile(r"^(\d+)Q(\d+)(.*)$") +REF = re.compile(r"^(.+?)\s+(\d+)(?::(\d+))?$") + + +def book_name(canonical: str) -> str: + """The English name of a book: `Genesis` for `Gen`, `Ruth` for `Ruth`.""" + names = [n for n in BOOKS.get(canonical, ()) if len(n) > len(canonical)] + return names[0] if names else canonical + + +def siglum_key(name: str) -> tuple: + """Sort key for scroll names: by cave, then number (1Q1, 4Q2, 4Q10, …).""" + match = SIGLUM.match(name) + if match: + return (0, int(match[1]), int(match[2]), match[3].lower()) + return (1, name.lower()) + + +def parse_ref(ref: str) -> tuple[str, int, int | None] | None: + """`Genesis 1:3` → (`Gen`, 1, 3); None if this is no book reference.""" + match = REF.match(ref.strip().replace("_", " ")) + if not match: + return None + canonical = canonical_book(match[1]) + if canonical is None: + return None + return (canonical, int(match[2]), int(match[3]) if match[3] else None) + + +def format_ref(book: str, chapter: int, verse: int | None = None) -> str: + ref = f"{book_name(book)} {chapter}" + return ref if verse is None else f"{ref}:{verse}" + + +@dataclass +class Witness: + """The words of one verse in one scroll.""" + + scroll: str + words: list[int] = field(default_factory=list) + also_non_biblical: bool = False + + +@dataclass +class BookIndex: + """Book → chapter → verse → witnesses, sorted by scroll.""" + + word_type: str + books: dict[str, dict[int, dict[int, list[Witness]]]] + + @classmethod + def build(cls, api) -> BookIndex | None: + """The book index of a corpus, or None if it has no biblical words.""" + F, T = api.F, api.T + if not {BOOK, CHAPTER, VERSE} <= set(api.Fall()): + return None + if WORD not in F.otype.all: + return None + fBook, fChapter, fVerse = api.Fs(BOOK).v, api.Fs(CHAPTER).v, api.Fs(VERSE).v + fBiblical = api.Fs(BIBLICAL).v if BIBLICAL in api.Fall() else None + top = T.sectionTypes[0] if T.sectionTypes else None + + # the words per scroll: its name asked once, not for each word + scrolls: Iterable[tuple[str, Any]] + if top: + scrolls = ( + (str(T.sectionFromNode(int(s), lang="en")[0]), api.L.d(int(s), WORD)) + for s in F.otype.s(top) + ) + else: + scrolls = [("", F.otype.s(WORD))] + canonical: dict[str, str | None] = {} + verses: dict[tuple[str, int, int], dict[str, Witness]] = {} + for scroll, words in scrolls: + for w in words: + w = int(w) + name = fBook(w) + if not name: + continue + if name not in canonical: + canonical[name] = canonical_book(str(name)) + book = canonical[name] + if book is None: + continue + (chapter, verse) = (str(fChapter(w)), str(fVerse(w))) + if not (chapter.isdigit() and verse.isdigit()): + continue + witnesses = verses.setdefault((book, int(chapter), int(verse)), {}) + witness = witnesses.get(scroll) + if witness is None: + witness = witnesses[scroll] = Witness(scroll) + witness.words.append(w) + if fBiblical and fBiblical(w) == 2: + witness.also_non_biblical = True + if not verses: + return None + + books: dict[str, dict[int, dict[int, list[Witness]]]] = {} + for key in sorted( + verses, key=lambda k: (ORDER.get(k[0], len(ORDER)), k[1], k[2]) + ): + witnesses = verses[key] + books.setdefault(key[0], {}).setdefault(key[1], {})[key[2]] = [ + witnesses[s] for s in sorted(witnesses, key=siglum_key) + ] + return cls(WORD, books) + + def has(self, book: str, chapter: int, verse: int | None = None) -> bool: + chapters = self.books.get(book, {}) + if verse is None: + return chapter in chapters + return verse in chapters.get(chapter, {}) + + def summary(self) -> list[dict[str, Any]]: + """The books with their chapters, verses and scrolls.""" + result = [] + for book, chapters in self.books.items(): + scrolls = { + w.scroll for c in chapters.values() for v in c.values() for w in v + } + result.append( + { + "book": book, + "label": book_name(book), + "ref": format_ref(book, next(iter(chapters))), + "scrolls": len(scrolls), + "verses": sum(len(c) for c in chapters.values()), + "chapters": [ + { + "chapter": chapter, + "ref": format_ref(book, chapter), + "verses": len(verses), + "witnesses": sum(len(ws) for ws in verses.values()), + } + for (chapter, verses) in chapters.items() + ], + } + ) + return result + + def neighbours(self, book: str, chapter: int) -> dict[str, str | None]: + """References of the chapters before and after, across books.""" + keys = [(b, c) for (b, chapters) in self.books.items() for c in chapters] + i = keys.index((book, chapter)) + prev = keys[i - 1] if i > 0 else None + nxt = keys[i + 1] if i + 1 < len(keys) else None + return { + "prev": format_ref(*prev) if prev else None, + "next": format_ref(*nxt) if nxt else None, + } + + +def place(api, words: list[int]) -> str: + """Where words are in their scroll: `f1i:1` or `f1i:1–2` or `f1:9–f2:3`.""" + T = api.T + first = T.sectionFromNode(words[0], lang="en") + last = T.sectionFromNode(words[-1], lang="en") + start = f"{first[1]}:{first[2]}" if len(first) > 2 else str(first[-1]) + if first == last: + return start + if len(last) > 2 and first[1] == last[1]: + return f"{start}–{last[2]}" + end = f"{last[1]}:{last[2]}" if len(last) > 2 else str(last[-1]) + return f"{start}–{end}" + + +# Placeholders in the DSS transcription (sign `type`, see the feature +# documentation of ETCBC/dss): `ε` (`missing`) is text that is lost and not +# reconstructed, ` # ` and ` ? ` (`unc`) a letter too damaged to identify. +# They are shown as in the editions: [ … ] for a gap, ◦ for an illegible letter. +# The transcription also marks morpheme breaks (`sep` `׳`, as in עמ׳י), which +# are not in the scrolls: they are left out. +SIGN_TYPE = "type" +PLACEHOLDERS = {"ε": "[…]", " # ": "◦", " ? ": "◦", "׳": ""} +PLACEHOLDER_TYPES = {"missing", "unc"} +SHOWN_TYPES = PLACEHOLDER_TYPES | {"sep"} + + +def show_placeholders(text: str) -> str: + for glyph, shown in PLACEHOLDERS.items(): + text = text.replace(glyph, shown) + return text + + +def dress_signs(api, fragments: list[dict[str, Any]]) -> None: + """Prepare the signs of a manuscript (DSS) in render tree fragments. + + Parts reconstructed by an editor are flagged `dim`: slots (signs) with + `rec`, and fragments shown as one text (a word in the structure view) + whose slots are all reconstructed. Placeholders for lost and illegible + text are shown as in the editions, and morpheme breaks are left out (see + `PLACEHOLDERS`). + """ + if RECONSTRUCTED not in api.Fall(): + return + rec = api.Fs(RECONSTRUCTED).v + signType = api.Fs(SIGN_TYPE).v if SIGN_TYPE in api.Fall() else None + slotType = api.F.otype.slotType + + def visit(frags: list[dict[str, Any]]) -> None: + for frag in frags: + children = frag.get("children") + if children: + visit(children) + elif "text" in frag: + (begin, end) = frag["range"] + if frag["type"] == slotType: + reconstructed = bool(rec(frag["node"])) + placeholder = signType and signType(frag["node"]) in SHOWN_TYPES + else: + reconstructed = all(rec(s) for s in range(begin, end + 1)) + placeholder = signType and any( + signType(s) in SHOWN_TYPES for s in range(begin, end + 1) + ) + if reconstructed: + frag["dim"] = True + if placeholder: + frag["text"] = show_placeholders(frag["text"]) + + visit(fragments) diff --git a/libs/explorer/cfabric_browser/words.py b/libs/explorer/cfabric_browser/words.py new file mode 100644 index 0000000..75883ef --- /dev/null +++ b/libs/explorer/cfabric_browser/words.py @@ -0,0 +1,68 @@ +"""The words of a corpus: their node type, the words of a node, the format of +their text, and the book each lies in. + +Shared by the comparison, the Concordance, the Interlinear and the lexicon. +""" + +from __future__ import annotations + +from bisect import bisect_right + +from cfabric_browser.display import DisplayConfig + + +def word_type(api, config: DisplayConfig) -> str: + """The node type of words: `word`, or else a base type, or else the slot.""" + types = api.F.otype.all + if "word" in types: + return "word" + bases = [t for t in types if t in config.base_types and t != config.slot_type] + return str(bases[0]) if bases else config.slot_type + + +def orig_format(config: DisplayConfig) -> str | None: + """The first text format in the original script, or else the default.""" + return next( + (f for f in config.text_formats if f.startswith("text-orig")), + config.default_format, + ) + + +def words_of(api, wType: str, node: int) -> list[int]: + """The words of a node, in order; for a part of a word (a sign), its word.""" + if api.F.otype.v(node) == wType: + return [node] + down = [int(w) for w in api.L.d(node, otype=wType)] + if down: + return down + up = api.L.u(node, otype=wType) + return [int(up[0])] if up else [] + + +class BookSpans: + """The books as runs of nodes of one type (words, or slots), to find the + book of such a node quickly (asking `L.u` for each is slow for many).""" + + def __init__(self, api, unit: str) -> None: + self._starts: list[int] = [] + self._ends: list[int] = [] + self._books: list[int] = [] + types = api.T.sectionTypes or () + if not types: + return + spans = [] + for b in api.F.otype.s(types[0]): + units = api.L.d(int(b), otype=unit) + if len(units): + spans.append((int(units[0]), int(units[-1]), int(b))) + for first, last, b in sorted(spans): + self._starts.append(first) + self._ends.append(last) + self._books.append(b) + + def book(self, node: int) -> int | None: + """The book a node of the unit type lies in, if any.""" + k = bisect_right(self._starts, node) - 1 + if k < 0 or node > self._ends[k]: + return None + return self._books[k] diff --git a/libs/explorer/cfabric_browser/workers.py b/libs/explorer/cfabric_browser/workers.py new file mode 100644 index 0000000..39cfd37 --- /dev/null +++ b/libs/explorer/cfabric_browser/workers.py @@ -0,0 +1,627 @@ +"""Searching in worker processes. + +Searches run in separate processes so that a heavy query never blocks the +browser, several searches can run at once, and a runaway search can always be +stopped. Each worker opens the corpora itself; because corpora are memory +mapped, the workers share most of their memory with each other and with the +server process. + +Protocol, per worker, over two one-way pipes: + +* server to worker: `("search", job_id, corpus, template, limit, timeout, + sets)`, `("cancel", job_id)`, `("stop",)` +* worker to server: `("ready",)`, `("batch", job_id, [tuples])`, + `("done", job_id, count, truncated)`, `("failed", job_id, problems)`, + `("stopped", job_id, reason, count)` with reason `timeout` or `cancelled` + +Cancellation is cooperative: the worker checks its `CancelToken` (cancelled, +or past its timeout) between results. A search that doesn't get to its +results in time (its engine is still matching) is stopped by the watchdog, +which kills the worker and starts another. Searches use only the core's +public `S.search`, so this works with the Python core and the Rust one. +""" + +from __future__ import annotations + +import logging +import multiprocessing as mp +import queue +import threading +import time +import uuid +from collections import OrderedDict +from collections.abc import Callable +from dataclasses import dataclass, field +from multiprocessing.connection import Connection +from typing import Any + +logger = logging.getLogger(__name__) + +QUEUED = "queued" +RUNNING = "running" +DONE = "done" +FAILED = "failed" +TIMEOUT = "timeout" +CANCELLED = "cancelled" +FINISHED = frozenset({DONE, FAILED, TIMEOUT, CANCELLED}) + +BATCH_SIZE = 2000 +BATCH_INTERVAL = 0.15 +GRACE = 3.0 +# results between checks of the cancel token (cheap, but not for every one) +CHECK_EVERY = 256 +# workers in a row that may die before they are ready, before giving up +MAX_FAILED_STARTS = 3 + + +@dataclass +class SearchJob: + """A search and its (growing) results, shared between threads.""" + + id: str + corpus: str + template: str + limit: int + timeout: float + state: str = QUEUED + results: list[tuple[int, ...]] = field(default_factory=list) + truncated: bool = False + sorted: bool = False + problems: list[dict[str, Any]] = field(default_factory=list) + created: float = field(default_factory=time.monotonic) + started: float | None = None + finished: float | None = None + version: int = 0 + sets: dict[str, set[int]] | None = field(default=None, repr=False) + _cond: threading.Condition = field(default_factory=threading.Condition, repr=False) + + @property + def count(self) -> int: + return len(self.results) + + @property + def elapsed(self) -> float: + if self.started is None: + return 0.0 + return (self.finished or time.monotonic()) - self.started + + def update(self, **changes: Any) -> None: + with self._cond: + for key, value in changes.items(): + setattr(self, key, value) + self.version += 1 + self._cond.notify_all() + + def extend(self, batch: list[tuple[int, ...]]) -> None: + with self._cond: + self.results.extend(batch) + self.version += 1 + self._cond.notify_all() + + def wait_change(self, version: int, timeout: float) -> int: + """Block until the job changes after `version`, or the timeout passes.""" + with self._cond: + if self.version == version and self.state not in FINISHED: + self._cond.wait(timeout) + return self.version + + def page(self, offset: int, limit: int) -> list[tuple[int, ...]]: + with self._cond: + return self.results[offset : offset + limit] + + def snapshot(self) -> dict[str, Any]: + with self._cond: + return { + "id": self.id, + "corpus": self.corpus, + "template": self.template, + "state": self.state, + "count": len(self.results), + "truncated": self.truncated, + "sorted": self.sorted, + "limit": self.limit, + "timeout": self.timeout, + "elapsed": round(self.elapsed, 3), + "problems": list(self.problems), + "version": self.version, + } + + +# the worker process + + +def _worker_main( + specs: list[tuple[str, str]], + commands: Connection, + output: Connection, + batchSize: int, + batchInterval: float, +) -> None: + """Entry point of a worker process.""" + from cfabric_browser.cancel import CancelToken, SearchCancelled, SearchTimeout + from cfabric_browser.queries import ( + capture_engine_messages, + check_template, + quantifier_problems, + ) + from cfabric_browser.registry import CorpusRegistry + + registry = CorpusRegistry() + for name, path in specs: + registry.register(path, name) + try: + registry.get_api(name) + except Exception: # its searches report the problem + logger.exception("Search worker could not load corpus %s", name) + output.send(("ready",)) + + jobs: queue.Queue = queue.Queue() + current: dict[str, Any] = {"id": None, "token": None} + lock = threading.Lock() + + def listen() -> None: + while True: + try: + msg = commands.recv() + except (EOFError, OSError): + jobs.put(None) + return + if msg[0] == "search": + jobs.put(msg) + elif msg[0] == "add": + # a corpus opened while the browser runs; loaded on first search + registry.register(msg[2], msg[1]) + elif msg[0] == "cancel": + with lock: + if current["id"] == msg[1] and current["token"] is not None: + current["token"].cancel() + elif msg[0] == "stop": + jobs.put(None) + return + + threading.Thread(target=listen, daemon=True).start() + + while True: + msg = jobs.get() + if msg is None: + break + (_, jobId, corpus, template, limit, timeout, sets) = msg + token = CancelToken(timeout=timeout) + with lock: + current["id"] = jobId + current["token"] = token + + count = 0 + batch: list[tuple[int, ...]] = [] + lastSend = time.monotonic() + + def flush(jobId: str) -> None: + nonlocal batch, lastSend + if batch: + output.send(("batch", jobId, batch)) + batch = [] + lastSend = time.monotonic() + + try: + api = registry.get_api(corpus) + problems = check_template(api, template, sets) + if problems: + output.send(("failed", jobId, problems)) + continue + token.check() + nested: list[dict[str, Any]] = [] + truncated = False + with capture_engine_messages() as messages: + results = api.S.search(template, sets=sets, silent="deep") + for result in results or (): + if count == 0: + # the Python engine only logs problems inside + # quantifiers, once it has studied the template + nested = quantifier_problems(messages) + if nested: + break + if count >= limit: + truncated = True + break + batch.append(tuple(int(n) for n in result)) + count += 1 + if count % CHECK_EVERY == 0: + token.check() + if ( + len(batch) >= batchSize + or time.monotonic() - lastSend > batchInterval + ): + flush(jobId) + if count == 0 and not nested: + nested = quantifier_problems(messages) + if nested: + output.send(("failed", jobId, nested)) + continue + flush(jobId) + output.send(("done", jobId, count, truncated)) + except SearchTimeout: + flush(jobId) + output.send(("stopped", jobId, TIMEOUT, count)) + except SearchCancelled: + flush(jobId) + output.send(("stopped", jobId, CANCELLED, count)) + except Exception as e: # noqa: BLE001 - report anything to the server + flush(jobId) + output.send( + ("failed", jobId, [{"kind": "error", "line": None, "message": repr(e)}]) + ) + finally: + with lock: + current["id"] = None + current["token"] = None + + +# the server side + + +class _Worker: + def __init__( + self, + index: int, + context: Any, + specs: list[tuple[str, str]], + batchSize: int, + batchInterval: float, + ) -> None: + self.index = index + (cmdRecv, self.commands) = context.Pipe(duplex=False) + (self.output, outSend) = context.Pipe(duplex=False) + self.process = context.Process( + target=_worker_main, + args=(specs, cmdRecv, outSend, batchSize, batchInterval), + name=f"cfabric-search-{index}", + daemon=True, + ) + self.process.start() + cmdRecv.close() + outSend.close() + self.ready = threading.Event() + # ready, or gone before it got there + self.settled = threading.Event() + self.job: SearchJob | None = None + self.dead = False + + def send(self, msg: tuple) -> bool: + try: + self.commands.send(msg) + return True + except (OSError, EOFError, BrokenPipeError): + return False + + def kill(self) -> None: + self.dead = True + if self.process.is_alive(): + self.process.kill() + self.process.join(timeout=5) + for conn in (self.commands, self.output): + try: + conn.close() + except OSError: + pass + + +class SearchPool: + """A pool of worker processes that run searches. + + Parameters + ---------- + specs: list of (name, path) + The corpora the workers must be able to search. + sort_key: callable + Given a corpus name, the key function that puts result tuples in + canonical order (`N.sortKeyTuple`), used when a search has finished. + workers: int + Number of worker processes, i.e. searches that can run at once. + max_results: int + Results kept per search; searches stop (truncated) beyond this. + timeout: float + Default time limit per search, in seconds. + keep: int + Number of finished searches kept for paging and export. + """ + + def __init__( + self, + specs: list[tuple[str, str]], + sort_key: Callable[[str], Callable[[tuple[int, ...]], Any]] | None = None, + workers: int = 2, + max_results: int = 500_000, + timeout: float = 60.0, + keep: int = 100, + batch_size: int = BATCH_SIZE, + batch_interval: float = BATCH_INTERVAL, + ) -> None: + self.specs = list(specs) + self.sort_key = sort_key + self.max_results = max_results + self.timeout = timeout + self.keep = keep + self.batch_size = batch_size + self.batch_interval = batch_interval + self._context = mp.get_context("spawn") + self._lock = threading.RLock() + self._workers: list[_Worker] = [] + self._queue: list[SearchJob] = [] + self._jobs: OrderedDict[str, SearchJob] = OrderedDict() + self._cancelDeadline: dict[str, float] = {} + # jobs being finished (their results sorted), by one thread only + self._finishing: set[str] = set() + # workers in a row that died before they were ready + self._failedStarts = 0 + self._closing = False + self._size = max(1, workers) + self._watchdog: threading.Thread | None = None + + # lifecycle + + def start(self, wait: bool = True, timeout: float = 600.0) -> None: + """Start the workers; by default wait until they have loaded the corpora.""" + with self._lock: + for i in range(self._size): + self._workers.append(self._spawn(i)) + self._watchdog = threading.Thread( + target=self._watch, name="cfabric-search-watchdog", daemon=True + ) + self._watchdog.start() + if wait: + deadline = time.monotonic() + timeout + for worker in list(self._workers): + worker.settled.wait(max(0.0, deadline - time.monotonic())) + + def shutdown(self) -> None: + with self._lock: + self._closing = True + workers = list(self._workers) + for worker in workers: + worker.send(("stop",)) + for worker in workers: + worker.process.join(timeout=2) + worker.kill() + + @property + def ready(self) -> bool: + return any(w.ready.is_set() for w in self._workers) + + def _spawn(self, index: int) -> _Worker: + worker = _Worker( + index, self._context, self.specs, self.batch_size, self.batch_interval + ) + threading.Thread( + target=self._read, + args=(worker,), + name=f"cfabric-search-reader-{index}", + daemon=True, + ).start() + return worker + + def _replace(self, worker: _Worker) -> None: + """Kill a worker and start a fresh one in its place.""" + worker.kill() + with self._lock: + if self._closing or worker not in self._workers: + return + i = self._workers.index(worker) + self._workers[i] = self._spawn(worker.index) + self._dispatch() + + # jobs + + def submit( + self, + corpus: str, + template: str, + limit: int | None = None, + timeout: float | None = None, + sets: dict[str, set[int]] | None = None, + ) -> SearchJob: + """Queue a search; it starts as soon as a worker is free.""" + job = SearchJob( + id=uuid.uuid4().hex[:12], + corpus=corpus, + template=template, + limit=min(limit or self.max_results, self.max_results), + timeout=min(timeout or self.timeout, self.timeout), + sets=sets, + ) + with self._lock: + self._jobs[job.id] = job + self._queue.append(job) + self._evict() + self._dispatch() + return job + + def get(self, job_id: str) -> SearchJob | None: + with self._lock: + job = self._jobs.get(job_id) + if job is not None: + self._jobs.move_to_end(job_id) + return job + + def add_corpus(self, name: str, path: str) -> None: + """Make a corpus searchable by all workers, including ones started later.""" + with self._lock: + if any(n == name for (n, _) in self.specs): + return + self.specs.append((name, path)) + workers = list(self._workers) + for worker in workers: + worker.send(("add", name, path)) + + def cancel(self, job_id: str) -> bool: + """Stop a search. Returns False if there is no such search.""" + with self._lock: + job = self._jobs.get(job_id) + if job is None: + return False + if job.state == QUEUED: + self._queue.remove(job) + job.update(state=CANCELLED, finished=time.monotonic()) + return True + if job.state != RUNNING: + return True + worker = next((w for w in self._workers if w.job is job), None) + self._cancelDeadline[job.id] = time.monotonic() + GRACE + if worker is not None: + worker.send(("cancel", job.id)) + return True + + def _evict(self) -> None: + finished = [j for j in self._jobs.values() if j.state in FINISHED] + excess = len(self._jobs) - self.keep + for job in finished[: max(0, excess)]: + del self._jobs[job.id] + + def _dispatch(self) -> None: + with self._lock: + while self._queue: + worker = next( + ( + w + for w in self._workers + if w.ready.is_set() and w.job is None and not w.dead + ), + None, + ) + if worker is None: + return + job = self._queue.pop(0) + worker.job = job + job.update(state=RUNNING, started=time.monotonic()) + sent = worker.send( + ( + "search", + job.id, + job.corpus, + job.template, + job.limit, + job.timeout, + job.sets, + ) + ) + if not sent: + worker.job = None + job.update(state=QUEUED, started=None) + self._queue.insert(0, job) + return + + def _claim(self, job: SearchJob) -> bool: + """Whether this thread may finish the job (call with the lock held).""" + if job.state in FINISHED or job.id in self._finishing: + return False + self._finishing.add(job.id) + return True + + def _finish( + self, worker: _Worker, job: SearchJob, claimed: bool = False, **changes: Any + ) -> bool: + """Finish a job once: the first of the reader and the watchdog wins.""" + if not claimed: + with self._lock: + if not self._claim(job): + return False + if changes.get("state") == DONE and self.sort_key is not None: + try: + key = self.sort_key(job.corpus) + with job._cond: + job.results.sort(key=key) + changes["sorted"] = True + except Exception: + logger.exception("Could not sort results of search %s", job.id) + job.update(finished=time.monotonic(), **changes) + with self._lock: + self._finishing.discard(job.id) + if worker.job is job: + worker.job = None + self._cancelDeadline.pop(job.id, None) + self._dispatch() + return True + + def _read(self, worker: _Worker) -> None: + """Receive the messages of one worker.""" + while True: + try: + msg = worker.output.recv() + except (EOFError, OSError): + break + kind = msg[0] + if kind == "ready": + self._failedStarts = 0 + worker.ready.set() + worker.settled.set() + self._dispatch() + continue + job = worker.job + if job is None or msg[1] != job.id: + continue + if kind == "batch": + job.extend(msg[2]) + elif kind == "done": + self._finish(worker, job, state=DONE, truncated=msg[3]) + elif kind == "failed": + self._finish(worker, job, state=FAILED, problems=msg[2]) + elif kind == "stopped": + self._finish(worker, job, state=msg[2]) + + # the worker is gone; no new search may go to it + with self._lock: + unexpected = not worker.dead + worker.dead = True + worker.settled.set() + job = worker.job + if job is not None: + self._finish( + worker, + job, + state=FAILED, + problems=[ + { + "kind": "error", + "line": None, + "message": "The search process stopped unexpectedly", + } + ], + ) + if not unexpected or self._closing: + return + if not worker.ready.is_set(): + self._failedStarts += 1 + if self._failedStarts >= MAX_FAILED_STARTS: + logger.error("Search workers keep failing to start; searching is off") + return + logger.warning("Search worker %d died; starting a new one", worker.index) + self._replace(worker) + + def _watch(self) -> None: + """Kill workers whose search ignores its timeout or a cancel request.""" + while not self._closing: + time.sleep(0.5) + self._stop_stuck(time.monotonic()) + + def _stop_stuck(self, now: float) -> None: + """One round of the watchdog.""" + stuck: list[tuple[_Worker, SearchJob, str]] = [] + with self._lock: + for worker in self._workers: + job = worker.job + if job is None or job.started is None: + continue + deadline = self._cancelDeadline.get(job.id) + if deadline is not None and now > deadline: + stuck.append((worker, job, CANCELLED)) + elif now > job.started + job.timeout + GRACE: + stuck.append((worker, job, TIMEOUT)) + for worker, job, reason in stuck: + with self._lock: + # finished meanwhile: the worker is fine + if worker.job is not job or not self._claim(job): + continue + # so that the job queued next doesn't go to this worker + worker.dead = True + logger.warning("Search %s did not stop; restarting its worker", job.id) + self._finish(worker, job, claimed=True, state=reason) + self._replace(worker) diff --git a/libs/explorer/frontend/.gitignore b/libs/explorer/frontend/.gitignore new file mode 100644 index 0000000..ff2c585 --- /dev/null +++ b/libs/explorer/frontend/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +*.tsbuildinfo diff --git a/libs/explorer/frontend/index.html b/libs/explorer/frontend/index.html new file mode 100644 index 0000000..202f7f1 --- /dev/null +++ b/libs/explorer/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + CACCHT Explorer + + + +
+ + + diff --git a/libs/explorer/frontend/package-lock.json b/libs/explorer/frontend/package-lock.json new file mode 100644 index 0000000..ec3689f --- /dev/null +++ b/libs/explorer/frontend/package-lock.json @@ -0,0 +1,3255 @@ +{ + "name": "cfabric-browser-frontend", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cfabric-browser-frontend", + "version": "0.1.0", + "dependencies": { + "@codemirror/autocomplete": "6.20.3", + "@codemirror/commands": "6.11.1", + "@codemirror/language": "6.13.1", + "@codemirror/lint": "6.9.7", + "@codemirror/state": "6.7.6", + "@codemirror/view": "6.43.14", + "@fontsource-variable/jetbrains-mono": "5.3.0", + "@fontsource-variable/source-serif-4": "5.3.1", + "@fontsource/amiri": "5.3.0", + "@fontsource/gentium-plus": "5.3.0", + "@fontsource/noto-sans-syriac": "5.3.0", + "@fontsource/noto-serif-hebrew": "5.3.0", + "@lezer/highlight": "1.2.5", + "@tanstack/react-query": "5.104.2", + "@tanstack/react-virtual": "3.14.18", + "react": "19.3.0", + "react-dom": "19.3.0" + }, + "devDependencies": { + "@tailwindcss/vite": "4.3.3", + "@types/react": "19.3.0", + "@types/react-dom": "19.3.0", + "@vitejs/plugin-react": "4.7.0", + "marked": "18.0.14", + "playwright-core": "1.64.0", + "tailwindcss": "4.3.3", + "typescript": "5.9.3", + "vite": "6.4.4", + "vitest": "3.2.4" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.10.tgz", + "integrity": "sha512-EMCx3AGZ3WfAl6ht7AyQoPm2Lh+BGYEMdc4Q6Hf7uSCDOZFuLq44qNwL+iaim8VlyYhRAo4X43+weadK8ZB4Yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.10.tgz", + "integrity": "sha512-B3lYAs2nkKskbgQ1nfy8Ma9ao9Q1rntTAQHrzzyDuj+TSO+X3le6PfWwqDvodsb+OumE10eMFfuMCTQSQ5iuIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.9", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@codemirror/autocomplete": { + "version": "6.20.3", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.3.tgz", + "integrity": "sha512-tlosUqb+3BbxCxZdu4tKeRghPFC+QM7q4X5YhKV2eCmPG+1r2F3f4AaSz5sCrFqUtX4Jh20VFTKecl16MgiV9g==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.11.1", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.11.1.tgz", + "integrity": "sha512-O/4hG3SC1YwcmQ0d2UVNDs+AsaNWd1iHVxbTeEBuqH+6bExAiPK3iS/BvpY6rZGURALv4ZD3sIgcCmRvw3ehBg==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.7.0", + "@codemirror/view": "^6.27.0", + "@lezer/common": "^1.1.0" + } + }, + "node_modules/@codemirror/language": { + "version": "6.13.1", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.13.1.tgz", + "integrity": "sha512-zfvU2r3TyylQLMj/te+5CnQ0rSqZi8B1oTCBbQkagcLW4Xc6imab6qN+pEJhsuNOLaJqiRyBXp9UhWA1DhVvDw==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/streamparser": "^6.0.0", + "@codemirror/view": "^6.23.0", + "@lezer/common": "^1.5.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.9.7.tgz", + "integrity": "sha512-28/+iWLYxKxsvGYhSYL7zaCZqLz5+FFFDq9tVsvGv9kv8RY4fFAchJ5WX9M3YrrRlTIsECjsXPqeNgnSmNP2dg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.42.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.7.6", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.7.6.tgz", + "integrity": "sha512-kAz+AncRtKuIknedxT1bq4XwXv4UowhbkHU1myPrtVb/jZtImWuV5BXzv5vK6i3kYACsdiZiQKFQQ5Mq7elW8w==", + "license": "MIT", + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/streamparser": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@codemirror/streamparser/-/streamparser-6.0.0.tgz", + "integrity": "sha512-JTFPF6xrfjt0mnfGPgzcLiMwYmvIyNaoizp8BYzv8FE8KrT6GBGDZ190rtt+O7Wke3gG/MFHyiY9YGNeimof4w==", + "license": "MIT" + }, + "node_modules/@codemirror/view": { + "version": "6.43.14", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.43.14.tgz", + "integrity": "sha512-hC45YnmJNR1rz42OF57BLoQNFMV12/13drnvibDsR2quNDmldcb4WvYVV81hFQg56i9lcqQMEDqilhX2QDcGQQ==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.7.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@fontsource-variable/jetbrains-mono": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/jetbrains-mono/-/jetbrains-mono-5.3.0.tgz", + "integrity": "sha512-F32xpS2NsGYoQi2ADSkKTgpJj7ozajsGgDJ8woTnqjmIB+dxDIqImjl4pXZVEExu8UFZ2ndhmX18EBS/hdz3Lw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource-variable/source-serif-4": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@fontsource-variable/source-serif-4/-/source-serif-4-5.3.1.tgz", + "integrity": "sha512-4Rtay22SC4lMvw11r/+9kxANB6P3M0X8s1NudBC2v0J1KU2y3DcMoNcT3lBoZ/CqLirzf+StlO6UlJilyrvaUg==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/amiri": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/amiri/-/amiri-5.3.0.tgz", + "integrity": "sha512-i4SWJnVIphj8556EdVzNrEG/Qu82k6JXjXTtSUIohx3tHmo2GHw1Kv60TWo+xjk7Yf1Etb2Gskh0lUFclTKyUw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/gentium-plus": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/gentium-plus/-/gentium-plus-5.3.0.tgz", + "integrity": "sha512-b66LAeYwBdHu0NL9ESFiq9NBIn4vIbl+L1kFKbSv5rjg2fRqzPFMtzAaQnqvgfdIK12LbMJHvDtybMk9N5lsJw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/noto-sans-syriac": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/noto-sans-syriac/-/noto-sans-syriac-5.3.0.tgz", + "integrity": "sha512-9GSaPnxN9rmYGhvyGyWUKlNPvncWZDFwGc5VwrJu7Gik1aovVudrqQ49wwtYxUJ6fxzvCEDhX36j6DKO8osceg==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/noto-serif-hebrew": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/noto-serif-hebrew/-/noto-serif-hebrew-5.3.0.tgz", + "integrity": "sha512-UbTqdsCCQRYAxQ1ZNjWTcL+tbqYvCG0nhCSD0Y2CmWEfl32e71/DAlRw9IvhjEdCJnKJm0t9AOfiNiut1mHKeA==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@lezer/common": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.3.tgz", + "integrity": "sha512-H0iErY4e43LpXbYDyBci5W4v/RwTgGV3YzYOlJPiGZ8RY8w51kVE+xLn1tg3Z6UmSfyETvtsUK3r7YsgDQEI7Q==", + "license": "MIT" + }, + "node_modules/@lezer/highlight": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.5.tgz", + "integrity": "sha512-O1GMVKgtf5YspFaRzpmqkVgMtIk2HG9uMAbH6wdAIdNh3TVzCsch/+cRi+6l2//UGDKe6NB9BW3cvuR5ezMQ0w==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.3.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.11.tgz", + "integrity": "sha512-UwgZcl1R66H2hDzBrA7kWyYmX6s8p5TQ+e/t7CuvgCtBEPxbEPaHYHu+pJBvTvDotEJpuQkJjbXuHiw5Cixxgw==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.4.tgz", + "integrity": "sha512-Wy0V7+SGUjnF9/TkiM1hKVDPj7jKXduPNboMVtHTA8dySMURWqfg/JZ9E2Sq8JgSJmkl7k7Qe9FLeMSrSraWmQ==", + "license": "MIT" + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.64.6.tgz", + "integrity": "sha512-FkO19Fowv/OQeJZPVfdetJaI3/VpbZmBsnmAlR/rQ+b3QwfiGQiOpVegpSRH331kelvHTVw9hXYG3DwjAaRzyQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.64.6.tgz", + "integrity": "sha512-si5/Etm6MphPc2xYHnk455Shls/hoMjTy3mhXge4OPmvCw7TwC4KP4fKokAlKLV4XaQvLvgSHfafPcblLMYYdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.64.6.tgz", + "integrity": "sha512-PQCl3q7b0BgvVt72i2xrWM6I5KliwwVe4GpWW5QDGyUQawDS9aPZss/aGyFqBmxdRxlE8wDc9CaxiQ5DHRSp/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.64.6.tgz", + "integrity": "sha512-l9DFl1Itb1YYdSrP1K3U8ih44AOHa/p+Q4WMMDfGvH5qVWp+ajfgS1DlYo4yCRh8wJ2ZJv0Fqccf+ULAmMFjmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.64.6.tgz", + "integrity": "sha512-bR4GR3205N9+od0+l5Dd9/iZTDpJkIe6d6Ydp2qbcQZJLmKBdYQQRp8gbWpbMt92twKJY66o1vK4NkoKQODIrA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.64.6.tgz", + "integrity": "sha512-XTVNaDzcFl2TeBglvxdGgsCv9JGtAGRAoJENmGBR/XvDH2mA2K0IXc2aMfP0dR8mexSXSKifJ85FixoXX1wfTw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.64.6.tgz", + "integrity": "sha512-qWc6qbc+25ugG9vqPR9P422KJBbVfZ05mjcMggoq8tsfWP+eRfC+/tiDdIlRF5VL2pDVZWN9zAhWsmu7R1+7Dw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.64.6.tgz", + "integrity": "sha512-V4Nwkp8844X/Xnww3PI1LYS0YPei8XQEVG70R+5MPZGO7AhYoun4hCN7Svs4U3wbQLZbwiz+9xRFhUZZdRl8Ww==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.64.6.tgz", + "integrity": "sha512-xb72A1aPaaIsp+a88VtRae/ovIEXislbUMyp7b9AQQU7ZkP2m/+ik6DW9TQcJyw5R8/RIailF9zExYwkjR++UA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.64.6.tgz", + "integrity": "sha512-pbcs1BVBwC43hZkjhjlF8Rx8m5G0KnhLZs9Azy/gqccP6UsNSzJr0hKGIAVPqzAH1O/NdPxxC9GVGb4msQg/aw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.64.6.tgz", + "integrity": "sha512-EKOC8os6Vsg487ZLKUqzLF99Jsmnq6tu7O7mAELakNpcA+8KadktFf/JYdRuHe/DzCUXmN38pUKu4XBcKlqGbQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.64.6.tgz", + "integrity": "sha512-v/SQ68GnSlr/HyjB79sNXesGDwkOPu9JjbHfmWaymCiM+nZsQ9TstG1KfA4MuEvTr2d3PoY8PhFxj9bt0e32JA==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.64.6.tgz", + "integrity": "sha512-G3pmliMQ7r15zFFNwBlaT/xTdSLi4lfgI7CUrQueyKYheNVhrd8JrWCE6Ll8xf4AA1RZFqq+mNJYntiElsVOyA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.64.6.tgz", + "integrity": "sha512-XQTB/IPoB/wG6x/wu9uomAt8YH61IAr5iZTubMiad/lu88sotq2ocz9hJd556+Jz2kIkF7ZQZIsUuWE3OWqtcA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.64.6.tgz", + "integrity": "sha512-O5mpokth6+ydMuwiGtY8fIjvOBfH7u6SzmDNmZ3k6GK5C6iEAC2q2ZN/T5L9jYILx0IcCT0KnfMDihQ5yQFSBw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.64.6.tgz", + "integrity": "sha512-aIFxbAD9v3TvIaEDsKmukjcDg1JS4/tFXw8SHyQQgzOwzWCQ4cyDK7Ju1pQKqUXDFkEu/BZCFDCDATsKhQcB/Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.64.6.tgz", + "integrity": "sha512-fuNo1UwBc6BxnHDOxxguQL/InhheMN3RJcJMb0b4xP8r5Q3CNCrJbVg0eCIFuJ4e/zAfjR2gYbOfbzO/G7CIyw==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.64.6.tgz", + "integrity": "sha512-05v6wFKDKJPZs9xlytVrrRu6IbRuoSvlH9xNwEOafXsKyjBFZPvAcZz2VIAB/M+lblYeKa9iOYC4RDVdfcAAaw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.64.6.tgz", + "integrity": "sha512-wJXKcdMjvkc/FaavZXNP1yW4i0pfOmhZAIWRSLG2yJ0PB0TWi7dkLErhX0u0vptDcXKKO/m1Y/YTeuyvb2MXwg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.64.6.tgz", + "integrity": "sha512-cwg7/h0ThvJwy8CBkmACdiHAxRWpbsCmIsjOt+Zzx0hTnIcSUmKYEV03bGg+hD827OnwUsbKh1ZZDW754xWyBA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.64.6.tgz", + "integrity": "sha512-T0RkYIJbdYXJEQFw+tMBO09KDFdgmNxkHsof3wWAABrvnx15Yh8RytmrgYy9+q9cBeBNZF66MMQZC+5nzEz6vg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.64.6.tgz", + "integrity": "sha512-KlfpmF1AAIdAWxt8QQNClbpLFlAUfYOzqIuJ7IXEStMHpFRuH78RG6FjqPb3Rqhh0K89F5npvG+/i3Fjl6IveQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.64.6.tgz", + "integrity": "sha512-MrTLjd7Jlt9/ZavJZYG61KQv9Iy9bzbY+yXy73Xvwoa2uP66wvDShkyBr5Chz/yrE6lbD/zHMuTmKQVNQqBApg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.64.6.tgz", + "integrity": "sha512-8fWDH1CkPI8GdBKAxzHDza7UmgficsKo2+MVbvUgOGhB0629w5sHMg3Qu5y7TZsrYKHHWpz8fla4KvoT7WsO3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.64.6.tgz", + "integrity": "sha512-9/c9PWdr5GCIHkkxiHM1HjNBQMVdftP4O8QTcoUAaRza2FSqJ4axbf8QAnBuE6BIGQDl71SaRPeDdDYid5ecTw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", + "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.24.1", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", + "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-x64": "4.3.3", + "@tailwindcss/oxide-freebsd-x64": "4.3.3", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-x64-musl": "4.3.3", + "@tailwindcss/oxide-wasm32-wasi": "4.3.3", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", + "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", + "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", + "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", + "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", + "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", + "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", + "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", + "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", + "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", + "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", + "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", + "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", + "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.3", + "@tailwindcss/oxide": "4.3.3", + "tailwindcss": "4.3.3" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.104.2", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.104.2.tgz", + "integrity": "sha512-/qbsxJK5c3Ve1aFIULWYNcT2jeRUAiD4HME6Eu2ZdpofHoVgAjpkQhLg5xHuOoBkTPQUJLfP7zOxy0qsBcqROA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.104.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.104.2.tgz", + "integrity": "sha512-xKmzWGhadbE5wGS/ZmXShRITnI9oItox5GO14KvFZco3+vhTAyXS1erYzJWdiY280zapAGJXFUL9nHgwP6SV/g==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.104.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@tanstack/react-virtual": { + "version": "3.14.18", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.18.tgz", + "integrity": "sha512-zuSuxQTfeLhvdS0LOi1MjQ/FrG4MVeVT2w+Iy33fviGFqjpWV9kbyNQOrNJXckyO0iNko2NlW3TaThcg2Cz9qA==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.19.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@tanstack/virtual-core": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.19.3.tgz", + "integrity": "sha512-jPxPiyBaUn+IChc8sfYNKn7B2SCOxYO1mCJrh7U4ldmrMxun19q/iw+jHRYk2cR/KNEjW8ABJrufX41m342Sqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/@vitest/expect": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", + "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", + "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", + "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.4", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", + "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot/node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils/node_modules/@vitest/pretty-format": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", + "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.28", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.28.tgz", + "integrity": "sha512-br0LOmjJqr6Rt4AEkadbcNf67Zpo7EPwKsWbjnRrJIa1QYFHQBqM4C7F2WTbV8ahTEcg1Eo+Y0aWjjORInYFyA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001815", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001815.tgz", + "integrity": "sha512-uDybUEyfgbyJJ93J1vikTylBX31AkdVYeVPeffMQpf8XN7hSfr0oZOn4VFOIMNvtU5+rHvC5Udg2TgtJKHm23Q==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/crelt": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", + "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.453", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.453.tgz", + "integrity": "sha512-9r/nz9akllr3SZsHUhfK10ktUH6j0Dbo3cKdh91khw0FvR6HhXGnATmOq5X8ena2aq4GXLQ1koh3nvKaQJaNmQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/enhanced-resolve": { + "version": "5.26.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.26.0.tgz", + "integrity": "sha512-9vhedylFonb2YGogzUKX6+Ja72gOJbN1QHAqdrvqLwhdl/QWbKopzoUC9EbQNVsAns/bx/4uyqalrQAoy1IByw==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/marked": { + "version": "18.0.14", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.14.tgz", + "integrity": "sha512-mBHK6FBHuBAlhgRe88w9F0O1AbwwXJUcQibUbC/QcdTbVGAD7aWza+xt3N6oT/jCZx3/OMeS+8rnuiHZcQ9s7A==", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.58", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.58.tgz", + "integrity": "sha512-BdiKAgbfRlvfMq/XzO/AF9NPmaA8a+YpLAyAFTVFBj3gARrAIfWvNq7Q7NE4se3KgFu0KkrvQMxTS0XVnsGzHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright-core": { + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postcss": { + "version": "8.5.30", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz", + "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.64.6", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.64.6.tgz", + "integrity": "sha512-fX1Vy+UNKdXn+xKDnu/HBXbHVr1AY8cSdQj9BU4kLv5aFCMqGpvrVbyUMmjN3DdzWhSAuTbcX8PCNEVaumthYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.64.6", + "@rollup/rollup-android-arm64": "4.64.6", + "@rollup/rollup-darwin-arm64": "4.64.6", + "@rollup/rollup-darwin-x64": "4.64.6", + "@rollup/rollup-freebsd-arm64": "4.64.6", + "@rollup/rollup-freebsd-x64": "4.64.6", + "@rollup/rollup-linux-arm-gnueabihf": "4.64.6", + "@rollup/rollup-linux-arm-musleabihf": "4.64.6", + "@rollup/rollup-linux-arm64-gnu": "4.64.6", + "@rollup/rollup-linux-arm64-musl": "4.64.6", + "@rollup/rollup-linux-loong64-gnu": "4.64.6", + "@rollup/rollup-linux-loong64-musl": "4.64.6", + "@rollup/rollup-linux-ppc64-gnu": "4.64.6", + "@rollup/rollup-linux-ppc64-musl": "4.64.6", + "@rollup/rollup-linux-riscv64-gnu": "4.64.6", + "@rollup/rollup-linux-riscv64-musl": "4.64.6", + "@rollup/rollup-linux-s390x-gnu": "4.64.6", + "@rollup/rollup-linux-x64-gnu": "4.64.6", + "@rollup/rollup-linux-x64-musl": "4.64.6", + "@rollup/rollup-openbsd-x64": "4.64.6", + "@rollup/rollup-openharmony-arm64": "4.64.6", + "@rollup/rollup-win32-arm64-msvc": "4.64.6", + "@rollup/rollup-win32-ia32-msvc": "4.64.6", + "@rollup/rollup-win32-x64-gnu": "4.64.6", + "@rollup/rollup-win32-x64-msvc": "4.64.6", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/strip-literal/node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/style-mod": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.4.tgz", + "integrity": "sha512-XXWIQt633/EpAFx8aZDOTjBzrCaGmhvEQlQo6MVPfa2OzO2cWo+4hV9h+6UkHYlXGfy+ODXKUdP7Pthmcu5ATw==", + "license": "MIT" + }, + "node_modules/tailwindcss": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", + "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.6.tgz", + "integrity": "sha512-u8KszXvGfU68hVcZpRHKG28T0krMuv2G5nDhiHaMLen/gIuFEgIJhaJuO69qjnXg5paSrbPMFfx3brNuN8eVSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.4.tgz", + "integrity": "sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.4.tgz", + "integrity": "sha512-xGjU/SyjeNyb0E4KxXBvYz/1C+3JXuujvf2hXNOtnlcE+58o5ajd9N2/n2VhkiJvpCP2qQasnUpx5PXNeI+p1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", + "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.4", + "@vitest/mocker": "3.2.4", + "@vitest/pretty-format": "^3.2.4", + "@vitest/runner": "3.2.4", + "@vitest/snapshot": "3.2.4", + "@vitest/spy": "3.2.4", + "@vitest/utils": "3.2.4", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.4", + "@vitest/ui": "3.2.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/libs/explorer/frontend/package.json b/libs/explorer/frontend/package.json new file mode 100644 index 0000000..182fcfb --- /dev/null +++ b/libs/explorer/frontend/package.json @@ -0,0 +1,46 @@ +{ + "name": "cfabric-browser-frontend", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build && node scripts/guide.mjs", + "guide": "node scripts/guide.mjs", + "screenshots": "node scripts/screenshots.mjs", + "preview": "vite preview", + "typecheck": "tsc -b", + "test": "vitest run" + }, + "dependencies": { + "@codemirror/autocomplete": "6.20.3", + "@codemirror/commands": "6.11.1", + "@codemirror/language": "6.13.1", + "@codemirror/lint": "6.9.7", + "@codemirror/state": "6.7.6", + "@codemirror/view": "6.43.14", + "@fontsource-variable/jetbrains-mono": "5.3.0", + "@fontsource-variable/source-serif-4": "5.3.1", + "@fontsource/amiri": "5.3.0", + "@fontsource/gentium-plus": "5.3.0", + "@fontsource/noto-sans-syriac": "5.3.0", + "@fontsource/noto-serif-hebrew": "5.3.0", + "@lezer/highlight": "1.2.5", + "@tanstack/react-query": "5.104.2", + "@tanstack/react-virtual": "3.14.18", + "react": "19.3.0", + "react-dom": "19.3.0" + }, + "devDependencies": { + "@tailwindcss/vite": "4.3.3", + "@types/react": "19.3.0", + "@types/react-dom": "19.3.0", + "@vitejs/plugin-react": "4.7.0", + "marked": "18.0.14", + "playwright-core": "1.64.0", + "tailwindcss": "4.3.3", + "typescript": "5.9.3", + "vite": "6.4.4", + "vitest": "3.2.4" + } +} diff --git a/libs/explorer/frontend/public/favicon.svg b/libs/explorer/frontend/public/favicon.svg new file mode 100644 index 0000000..080feb5 --- /dev/null +++ b/libs/explorer/frontend/public/favicon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/libs/explorer/frontend/scripts/guide.mjs b/libs/explorer/frontend/scripts/guide.mjs new file mode 100644 index 0000000..70e998e --- /dev/null +++ b/libs/explorer/frontend/scripts/guide.mjs @@ -0,0 +1,285 @@ +// Builds the user guide (libs/explorer/guide/*.md) into HTML pages: for the app +// (cfabric_browser/static/guide/, served at /guide/, so the guide works offline) +// and, with `--site DIR`, for caccht.github.io (DIR/explorer/), in the style of +// that site's own pages (data-use.html). Run by `npm run build`, after Vite, +// which empties the static folder first. +// +// node scripts/guide.mjs # the app's pages +// node scripts/guide.mjs --site ../site # also the website's pages + +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Marked } from "marked"; + +const here = dirname(fileURLToPath(import.meta.url)); +export const SOURCE = resolve(here, "../../guide"); +export const APP_OUT = resolve(here, "../../cfabric_browser/static/guide"); + +// the pages in reading order; index is the contents +export const PAGES = [ + "index", + "getting-started", + "datasets", + "interfaces", + "reading", + "words", + "searching", + "concordance", + "asking-claude", + "adding-texts", + "notes-and-citing", + "problems", +]; + +/** A heading's id, as in links like `interfaces.md#panes`. */ +export function slug(text) { + return text + .toLowerCase() + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .replace(/<[^>]*>/g, "") + .replace(/[^\w\s-]/g, "") + .trim() + .replace(/\s+/g, "-"); +} + +/** Links between pages are written to the Markdown files (`words.md#bdb`); the pages are HTML. */ +export function pageLink(href) { + const m = /^([\w-]+)\.md(#.*)?$/.exec(href); + return m ? `${m[1]}.html${m[2] ?? ""}` : href; +} + +function escape(text) { + return text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} + +/** A page's title (its `#` heading), the rest as HTML, and the ids of its headings. */ +export function render(markdown) { + const ids = []; + const marked = new Marked({ + renderer: { + heading({ tokens, depth }) { + const inner = this.parser.parseInline(tokens); + const id = slug(inner); + ids.push(id); + return `${inner}\n`; + }, + link({ href, title, tokens }) { + const inner = this.parser.parseInline(tokens); + const target = pageLink(href); + const external = /^https?:/.test(target) ? ' rel="noopener"' : ""; + return `${inner}`; + }, + // screenshots open at full size when clicked + image({ href, text }) { + return `${escape(text)}`; + }, + }, + }); + const tokens = marked.lexer(markdown); + const first = tokens.findIndex((t) => t.type !== "space"); + let title = ""; + if (first >= 0 && tokens[first].type === "heading" && tokens[first].depth === 1) { + title = tokens[first].text; + tokens.splice(first, 1); + } + return { title, html: marked.parser(tokens), ids }; +} + +function readPages(source) { + return PAGES.map((name) => ({ name, ...render(readFileSync(join(source, `${name}.md`), "utf8")) })); +} + +function neighbours(pages, i) { + const prev = i > 0 ? pages[i - 1] : null; + const next = i < pages.length - 1 ? pages[i + 1] : null; + const link = (p, rel, label) => + p ? `${label} ${escape(p.title)}` : ""; + // laid out here, so that it needs nothing from the website's stylesheet + return ``; +} + +// the app's pages: the app's colours, its light or dark choice, a way back +const APP_STYLE = ` +:root{--vellum:#faf9f6;--raised:#fff;--ink:#1c1b19;--stone:#6e6a63;--rule:#e4e1da;--woad:#34477a;--woad-soft:#e6e9f2;color-scheme:light} +[data-theme=dark]{--vellum:#15171d;--raised:#1c1f27;--ink:#ece9e2;--stone:#a19c92;--rule:#2c303b;--woad:#8ea2d8;--woad-soft:#232a3d;color-scheme:dark} +*{box-sizing:border-box} +body{margin:0;background:var(--vellum);color:var(--ink);font:16px/1.6 "Source Serif 4",Georgia,serif} +a{color:var(--woad)} +header{display:flex;align-items:center;gap:.6rem;padding:.5rem 1rem;border-bottom:1px solid var(--rule);background:var(--raised);font-size:15px} +header img{width:20px;height:20px} +header .name{font-weight:600} +header .back{margin-inline-start:auto;text-decoration:none} +.layout{display:grid;grid-template-columns:15rem minmax(0,44rem);gap:2.5rem;justify-content:center;padding:1.5rem 1rem 4rem} +.toc ol{list-style:none;margin:0;padding:0;font-size:14px;position:sticky;top:1rem} +.toc a{display:block;padding:.2rem .5rem;border-radius:4px;color:var(--ink);text-decoration:none} +.toc a:hover{background:var(--woad-soft)} +.toc a[aria-current=page]{background:var(--woad-soft);font-weight:600} +h1{font-size:2rem;line-height:1.2;margin:.2rem 0 1rem} +h2{font-size:1.35rem;margin:2rem 0 .5rem} +h3{font-size:1.1rem;margin:1.5rem 0 .4rem} +code{font:.9em "JetBrains Mono",ui-monospace,monospace;background:var(--woad-soft);padding:.05em .3em;border-radius:3px} +pre{background:var(--raised);border:1px solid var(--rule);padding:.8rem 1rem;overflow-x:auto;border-radius:4px} +pre code{background:none;padding:0} +table{border-collapse:collapse;margin:1rem 0;font-size:15px} +th,td{border:1px solid var(--rule);padding:.35rem .6rem;text-align:start;vertical-align:top} +img{max-width:100%;height:auto;border:1px solid var(--rule);border-radius:4px} +.guide-pn{display:flex;justify-content:space-between;gap:1rem;margin-top:3rem;padding-top:1rem;border-top:1px solid var(--rule);font-size:15px} +.guide-pn span{color:var(--stone)} +@media (max-width:760px){.layout{grid-template-columns:minmax(0,1fr);gap:1rem}.toc ol{position:static}} +`; + +// the theme the app uses; and where the reader came from, so that the way back +// returns to the same passage (the app keeps its state in the URL) +const APP_SCRIPT = ` +try{var t=localStorage.getItem("cfabric-browser/theme");document.documentElement.dataset.theme=t==="light"||t==="dark"?t:matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light"}catch(e){} +addEventListener("DOMContentLoaded",function(){var k="cfabric-browser/guide-from",b=document.querySelector("a.back");try{var r=document.referrer&&new URL(document.referrer);if(r&&r.origin===location.origin&&!r.pathname.startsWith("/guide"))sessionStorage.setItem(k,r.href);var f=sessionStorage.getItem(k);if(f&&b)b.href=f}catch(e){}}); +`; + +export function appPage(pages, i) { + const page = pages[i]; + const toc = pages + .map( + (p) => `
  • ${escape(p.title)}
    • `, + ) + .join(""); + return ` + + + + +${escape(page.title)}: CACCHT Explorer + + + + + +
    CACCHT ExplorerGuide← Back to the Explorer
    +
    + +
    +

    ${escape(page.title)}

    +${page.html}${neighbours(pages, i)} +
    +
    + + +`; +} + +// the website's pages, as its data-use.html: the same head, bar and footer +export function sitePage(pages, i) { + const page = pages[i]; + const url = `https://caccht.github.io/explorer/${page.name === "index" ? "" : `${page.name}.html`}`; + const toc = pages + .map((p) => (p.name === page.name ? `
  • ${escape(p.title)}
    • ` : `
  • ${escape(p.title)}
    • `)) + .join(""); + const index = page.name === "index"; + const title = index ? `${escape(page.title)} — CACCHT` : `${escape(page.title)} — CACCHT Explorer`; + return ` + + + + +${title} + + + + + + + + + + + + + + + + + + + +
    +
    + + + CACCHT + + +
    +
    + +
    + +
    +
    +
    +

    CACCHT Explorer · Guide

    +

    ${escape(page.title)}

    +
    + +
    +${page.html}${index ? "" : ` +

    Contents of the guide

    +
      ${toc}
    `} +${neighbours(pages, i)} +
    +
    +
    + +
    + + + + + + +`; +} + +function write(dir, pages, page) { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + pages.forEach((p, i) => writeFileSync(join(dir, `${p.name}.html`), page(pages, i))); + // the screenshots (made by scripts/screenshots.mjs) + const images = join(SOURCE, "images"); + if (existsSync(images)) cpSync(images, join(dir, "images"), { recursive: true }); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const pages = readPages(SOURCE); + write(APP_OUT, pages, appPage); + console.log(`guide: ${pages.length} pages in ${APP_OUT}`); + const at = process.argv.indexOf("--site"); + if (at > 0 && process.argv[at + 1]) { + const out = join(resolve(process.argv[at + 1]), "explorer"); + write(out, pages, sitePage); + console.log(`guide: ${pages.length} pages in ${out}`); + } +} diff --git a/libs/explorer/frontend/scripts/guide.test.mjs b/libs/explorer/frontend/scripts/guide.test.mjs new file mode 100644 index 0000000..9973f89 --- /dev/null +++ b/libs/explorer/frontend/scripts/guide.test.mjs @@ -0,0 +1,143 @@ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { guideUrl } from "../src/guide"; +import { INTERFACES } from "../src/ui"; +import { appPage, pageLink, PAGES, render, sitePage, slug, SOURCE } from "./guide.mjs"; +import { SHOTS } from "./screenshots.mjs"; + +const pages = PAGES.map((name) => ({ name, ...render(readFileSync(join(SOURCE, `${name}.md`), "utf8")) })); + +describe("the guide's pages", () => { + it("has a title on every page", () => { + for (const p of pages) expect(p.title, p.name).not.toBe(""); + }); + + it("links only to pages and headings that exist", () => { + for (const p of pages) { + for (const [, page, anchor] of p.html.matchAll(/href="([\w-]+)\.html(?:#([\w-]+))?"/g)) { + const target = pages.find((q) => q.name === page); + expect(target, `${p.name} links to ${page}.html`).toBeDefined(); + if (anchor) expect(target.ids, `${p.name} links to ${page}.html#${anchor}`).toContain(anchor); + } + } + }); + + it("shows only pictures that exist, and uses every one", () => { + const used = new Set(); + for (const p of pages) { + for (const [, file] of p.html.matchAll(/ { + for (const ui of INTERFACES) { + const [, page, anchor] = /^\/guide\/([\w-]+)\.html(?:#([\w-]+))?$/.exec(guideUrl(ui.id)); + const target = pages.find((q) => q.name === page); + expect(target, ui.id).toBeDefined(); + if (anchor) expect(target.ids, ui.id).toContain(anchor); + } + }); + + it("builds both kinds of page", () => { + const i = PAGES.indexOf("interfaces"); + expect(appPage(pages, i)).toContain('

    Panes

    '); + expect(appPage(pages, i)).toContain('aria-current="page">The five ways of working'); + expect(sitePage(pages, i)).toContain('href="../assets/css/site.css"'); + expect(sitePage(pages, i)).toContain('href="https://caccht.github.io/explorer/interfaces.html"'); + }); +}); + +// Italics in the guide that are not the Explorer's own labels: other programs' +// buttons and menus, examples of requests and results, glosses, and emphasis. +const NOT_LABELS = new Set([ + // Windows, macOS, GitHub and Claude Desktop + "Windows protected your PC", "More info", "Run anyway", "Properties", "Unblock", "About This Mac", + "Chip Apple M…", "Applications", "System Settings → Privacy & Security", "Open Anyway", "Cmd+Shift+.", + "View → Show → Hidden items", "Assets", "Settings → Extensions", "Install Extension…", "Advanced settings", + "Extensions", "Settings → Developer → Edit Config", "Install", "Open", + // the ChatGPT desktop app + "Settings → MCP servers", "Add server", "Streamable HTTP", "Restart", "Go → Go to Folder…", + // a release's name, requests to Claude, glosses and other examples + "0.1.0-test.13", "find all cases of >LHJM/ in the BHSA", "clauses where the object comes before the verb", + "Find all cases of >LHJM/ in the BHSA.", "All infinitives construct of HLK[.", + "Clauses in Genesis where the subject comes before the verb.", "Lord", "lord", "cut down", "create", "speak", + "word", "perish", "verses with both אלהים and ברא", "Searching… 812 results so far", "4Q7 = bhsa", "4Q7 other", + "Subj", "Pred", "Objc", "Qal", "Niph.", "Pi.", "right before", "before", "petuhah", "setumah", "matres lectionis", + "transliteration", "· bhsa", + // the program's messages as the user sees them, with a number in them + "Download (68 MB)", "Parallel ▾", "Stopped after 60s with … results. Narrow the template", + "First … results (stopped at the limit)", +]); + +/** The guide's italics, outside code: what it names in the program. */ +function italics(markdown) { + const text = markdown.replace(/```[\s\S]*?```/g, "").replace(/`[^`\n]*`/g, "code"); + return [...text.matchAll(/(? m[1].trim()); +} + +/** The texts of the program's interface: its frontend, the guide's own pages, and the server's messages. */ +function programText() { + const roots = [ + [join(SOURCE, "../frontend/src"), /\.tsx?$/], + [join(SOURCE, "../frontend/scripts"), /^guide\.mjs$/], + [join(SOURCE, "../cfabric_browser"), /\.py$/], + [join(SOURCE, "../packaging"), /\.py$/], + ]; + let all = ""; + for (const [root, wanted] of roots) { + for (const file of readdirSync(root, { recursive: true })) { + const name = String(file).split(/[\\/]/).pop(); + if (wanted.test(name) && !/\.test\./.test(name)) all += readFileSync(join(root, String(file)), "utf8") + "\n"; + } + } + return all.replace(/\s+/g, " "); +} + +/** Whether a label is in the program; `…` in the guide stands for a value, `→` separates menu and item. */ +export function inProgram(label, text) { + return label.split("→").every((part) => { + const pieces = part.trim().split("…").map((p) => p.trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + return new RegExp(pieces.join("[\\s\\S]{0,80}?")).test(text); + }); +} + +describe("the guide's names of things in the program", () => { + it("names only labels that the program has", () => { + const text = programText(); + const missing = []; + for (const p of pages) { + const markdown = readFileSync(join(SOURCE, `${p.name}.md`), "utf8"); + for (const label of italics(markdown)) { + if (!NOT_LABELS.has(label) && !inProgram(label, text)) missing.push(`${p.name}: *${label}*`); + } + } + expect(missing).toEqual([]); + }); + + it("matches labels with values and menus", () => { + const text = "In {books.length} books title: 'Choose an interface' Display Text"; + expect(inProgram("In … books", text)).toBe(true); + expect(inProgram("Display → Text", text)).toBe(true); + expect(inProgram("Hide all", text)).toBe(false); + }); +}); + +describe("helpers", () => { + it("makes ids from headings", () => { + expect(slug("Switching and hiding")).toBe("switching-and-hiding"); + expect(slug("Notes, workspaces and citing")).toBe("notes-workspaces-and-citing"); + expect(slug("The JPS 1917")).toBe("the-jps-1917"); + }); + + it("turns links to Markdown pages into links to HTML pages", () => { + expect(pageLink("words.md#bdb")).toBe("words.html#bdb"); + expect(pageLink("datasets.md")).toBe("datasets.html"); + expect(pageLink("https://caccht.github.io/#corpora")).toBe("https://caccht.github.io/#corpora"); + }); +}); diff --git a/libs/explorer/frontend/scripts/screenshots.mjs b/libs/explorer/frontend/scripts/screenshots.mjs new file mode 100644 index 0000000..95c1554 --- /dev/null +++ b/libs/explorer/frontend/scripts/screenshots.mjs @@ -0,0 +1,258 @@ +// Takes the guide's screenshots (libs/explorer/guide/images/) from a running +// Explorer, so that they can be redone for each release instead of by hand. +// Start the Explorer on the real corpora first, with an empty home folder apart +// from the corpora and BDB, so that the pictures show what users see: +// +// cfabric-browser bhsa "sp=DT-UCPH/sp@7.2.4" "dss=ETCBC/dss@2.0.1" \ +// "jps1917=Context-Fabric/jps1917" --port 8799 --no-open +// node scripts/screenshots.mjs http://127.0.0.1:8799 [name ...] +// +// Uses the Chrome installed on this computer (playwright-core downloads no +// browser of its own). + +import { mkdirSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium } from "playwright-core"; + +const here = dirname(fileURLToPath(import.meta.url)); +const OUT = resolve(here, "../../guide/images"); +const SIZE = { width: 1280, height: 760 }; + +// each shot: a URL, what to wait for, what to do before taking it, and the theme +export const SHOTS = [ + { + name: "parallel", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + }, + { + name: "panes", + url: "/?c0=bhsa&ref0=Genesis+1&c1=dss&by1=books&sync=1", + ready: "text=4Q2", + }, + { + name: "reader", + url: "/read/bhsa/Genesis/1?v=3&panel=parallels&texts=sp,jps1917", + ready: "text=Parallels", + }, + { + name: "reader-dark", + url: "/read/bhsa/Genesis/1?mode=running", + ready: "text=Running text", + theme: "dark", + }, + { + name: "interlinear", + url: "/interlinear/bhsa/Genesis/1?under=sp&struct=1", + ready: "text=Lines", + }, + { + name: "menu", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: (page) => page.getByTitle("Choose an interface").click(), + clip: { x: 760, y: 0, width: 520, height: 420 }, + }, + { + name: "differences", + url: "/Exodus/20?texts=bhsa,sp,dss&diff=1&v=17", + ready: "text=words differ", + }, + { + name: "navigate", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.getByRole("button", { name: "Choose a book and chapter" }).click(); + await page.getByRole("button", { name: "Exodus", exact: true }).click(); + }, + }, + { + name: "hover", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.locator(".plain-word").nth(2).hover(); + await page.waitForSelector("text=Click for all features"); + }, + clip: { x: 0, y: 40, width: 760, height: 420 }, + }, + { + name: "inspector", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.locator(".plain-word").nth(2).click(); + await page.waitForSelector("text=Add as search term"); + await page.mouse.move(1270, 750); + }, + }, + { + name: "lexicon", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.getByRole("button", { name: "Lexicon" }).click(); + await page.getByLabel("Find a lexeme").fill("create"); + await page.getByText("BR>[").first().click(); + await page.waitForSelector("text=occurrences in"); + }, + }, + { + name: "bdb", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.getByRole("button", { name: "Lexicon" }).click(); + await page.getByLabel("Find a lexeme").fill("create"); + await page.getByText("BR>[").first().click(); + await page.getByRole("tab", { name: "BDB" }).click(); + await page.waitForSelector("text=Root family"); + }, + }, + { + name: "search", + url: `/Genesis/1?texts=bhsa,sp,jps1917&search=1&sc=bhsa&q=${encodeURIComponent("word lex=HLK[ vt=infc")}`, + ready: "text=Search in", + act: async (page) => { + await runSearch(page); + }, + }, + { + name: "search-terms", + url: "/Genesis/12?texts=bhsa,sp", + ready: ".plain-word", + act: async (page) => { + // לֶךְ in Genesis 12:1, an imperative of HLK[: its stem counts too + await page.locator(".plain-word").nth(5).click(); + await page.getByText("Add as search term").click(); + await page.getByRole("checkbox", { name: /qal/ }).check(); + await page.mouse.move(1270, 750); + }, + }, + { + name: "statistics", + url: `/Genesis/1?texts=bhsa,sp&search=1&sc=bhsa&q=${encodeURIComponent("word lex=HLK[ vt*")}`, + ready: "text=Search in", + act: async (page) => { + await runSearch(page); + await page.getByRole("button", { name: "Statistics", exact: true }).click(); + await page.waitForSelector("text=Values of"); + await page.waitForLoadState("networkidle"); + }, + }, + { + name: "concordance", + url: `/concordance/bhsa?q=${encodeURIComponent("word lex=HLK[ vt=infc")}&sort=L1.text`, + ready: "text=Concordance", + act: async (page) => { + await runSearch(page); + await page.waitForSelector("text=per 10,000 words"); + }, + }, + { + name: "collocations", + url: `/concordance/bhsa?q=${encodeURIComponent("word lex=HLK[ vt=infc")}&tab=collocations`, + ready: "text=Concordance", + act: async (page) => { + await runSearch(page); + await page.waitForSelector("text=logDice"); + await page.waitForLoadState("networkidle"); + }, + }, + { + name: "library", + url: "/Genesis/1?texts=bhsa,sp,dss", + ready: "text=4Q2", + act: async (page) => { + await page.getByRole("button", { name: "Texts", exact: true }).click(); + await page.getByText("Add a text…").click(); + await page.waitForSelector("text=Dictionaries"); + await page.waitForLoadState("networkidle"); + }, + size: { width: 1280, height: 1100 }, + }, + { + name: "workspaces", + url: "/read/bhsa/Genesis/1?v=3&panel=notes", + ready: "text=Running text", + act: async (page) => { + await page.getByRole("button", { name: "Workspaces", exact: true }).click(); + await page.getByPlaceholder("Name this workspace").fill("Genesis 1, with notes"); + await page.getByRole("button", { name: "Save", exact: true }).click(); + await page.waitForSelector("text=Export to a file"); + }, + clip: { x: 640, y: 0, width: 640, height: 420 }, + }, + { + name: "proposal", + url: "/Genesis/1?texts=bhsa,sp&search=1&sc=bhsa", + prepare: (base) => + fetch(`${base}/api/proposals`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + corpus: "bhsa", + template: "clause\n p1:phrase function=Subj\n p2:phrase function=Pred\np1 < p2", + request: "clauses where the subject comes before the verb", + explanation: + "Clauses with a subject phrase and a predicate phrase, the subject first. The predicate phrase holds the verb (or, in a nominal clause, the predicate).", + model: "Claude", + }), + }).then((r) => { + if (!r.ok) throw new Error(`proposing: ${r.status}`); + }), + ready: "text=Proposed by Claude", + // so that it doesn't show in the other shots + after: (page) => page.getByRole("button", { name: "Dismiss", exact: true }).click(), + }, +]; + +/** Clicks *Search* in the search form (not the top bar's) and waits for the results. */ +async function runSearch(page) { + await page.getByRole("button", { name: "Search", exact: true }).last().click(); + await page.waitForSelector("text=/results in /", { timeout: 60_000 }); +} + +async function take(browser, base, shot) { + const context = await browser.newContext({ viewport: shot.size ?? SIZE, deviceScaleFactor: 1 }); + await context.addInitScript((theme) => { + try { + localStorage.setItem("cfabric-browser/theme", theme); + } catch {} + }, shot.theme ?? "light"); + if (shot.prepare) await shot.prepare(base); + const page = await context.newPage(); + await page.goto(base + shot.url); + if (shot.ready) await page.waitForSelector(shot.ready, { timeout: 30_000 }); + await page.waitForLoadState("networkidle"); + if (shot.act) await shot.act(page); + await page.waitForTimeout(400); + const file = join(OUT, `${shot.name}.png`); + await page.screenshot({ path: file, clip: shot.clip }); + if (shot.after) await shot.after(page); + await context.close(); + console.log(`screenshot: ${file}`); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [base = "http://127.0.0.1:8799", ...names] = process.argv.slice(2); + mkdirSync(OUT, { recursive: true }); + const browser = await chromium.launch({ channel: "chrome" }); + let failed = 0; + try { + for (const shot of SHOTS) { + if (names.length && !names.includes(shot.name)) continue; + try { + await take(browser, base, shot); + } catch (e) { + failed += 1; + console.error(`screenshot ${shot.name} failed: ${e.message.split("\n").slice(0, 8).join("\n")}`); + } + } + } finally { + await browser.close(); + } + process.exitCode = failed ? 1 : 0; +} diff --git a/libs/explorer/frontend/src/App.tsx b/libs/explorer/frontend/src/App.tsx new file mode 100644 index 0000000..5f29b52 --- /dev/null +++ b/libs/explorer/frontend/src/App.tsx @@ -0,0 +1,579 @@ +import { useQuery } from "@tanstack/react-query"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { api } from "./api"; +import { corpusCard } from "./shared/corpora"; +import { DisplayProvider, displayRef } from "./display/Display"; +import { useDifferences, type PaneDiff } from "./differences"; +import { HoverCard, Hovering, useHover } from "./HoverCard"; +import { Inspector } from "./Inspector"; +import { LexiconView } from "./lexicon/LexiconView"; +import { Library } from "./Library"; +import { Workspaces } from "./Workspaces"; +import { DisplayOptions } from "./DisplayOptions"; +import { BookView } from "./read/BookView"; +import { ReadView } from "./read/ReadView"; +import { SearchView } from "./search/SearchView"; +import { InterfaceMenu } from "./InterfaceMenu"; +import { describeWorkspace } from "./switch"; +import { + emptyPane, + MAX_PANES, + useSettings, + useTheme, + useWorkspace, + type Pane, + type View, +} from "./state"; +import type { CorpusCard, ReadBy, ViewSettings } from "./types"; + +const VIEW_NAMES: Record = { read: "Read", search: "Search", lexicon: "Lexicon" }; + +/** The views a corpus offers: the lexicon only where it has one (BHSA). */ +function viewsOf(card: CorpusCard | undefined): View[] { + return card?.lexicon ? ["read", "search", "lexicon"] : ["read", "search"]; +} + +function Mark() { + // the Context-Fabric woven mark + return ; +} + + +interface PaneProps { + index: number; + pane: Pane; + card: CorpusCard | undefined; + cardError: unknown; + corpora: string[]; + paneCount: number; + settings: ViewSettings; + note: string | null; + diff: PaneDiff | null; + onChange: (change: Partial, push?: boolean) => void; + onOpenPassage: (ref: string) => void; + onBy: (by: ReadBy) => void; + onOpenPane: (pane: Pane) => void; + onClose: () => void; + onInspect: (corpus: string, node: number) => void; +} + +function PaneView({ + index, + pane, + card, + cardError, + corpora, + paneCount, + settings, + note, + diff, + onChange, + onOpenPassage, + onBy, + onOpenPane, + onClose, + onInspect, +}: PaneProps) { + const compact = paneCount > 1; + const setView = (view: View) => onChange({ view }, true); + // a corpus without a lexicon (after switching corpus) is read instead + const view: View = pane.view === "lexicon" && card && !card.lexicon ? "read" : pane.view; + const [hover, onHover] = useHover(); + // the word under the card goes when the pane shows something else + useEffect(() => { + onHover(null); + }, [onHover, pane.corpus, view, pane.ref, pane.by, pane.lex, pane.search, pane.result]); + // a corpus' own default format applies unless the reader chose one it has + const paneSettings = + card && settings.fmt && !card.display.textFormats.includes(settings.fmt) ? { ...settings, fmt: null } : settings; + + return ( +
    + {paneCount > 1 || corpora.length > 1 ? ( +
    + +
    + {viewsOf(card).map((v) => ( + + ))} +
    + {paneCount > 1 ? ( + + ) : null} +
    + ) : null} +
    + {cardError ? ( +

    + Could not open {pane.corpus}: {String((cardError as Error).message)} +

    + ) : !card ? ( +

    + Opening {pane.corpus}… A corpus opened for the first time is prepared once, which can take a minute. +

    + ) : ( + onInspect(card.name, n)} + onHover={onHover} + marks={view === "read" ? diff?.marks : undefined} + markTitles={view === "read" ? diff?.titles : undefined} + > + {view === "read" && pane.by === "books" && card.bookView ? ( + onChange({ by: "sections", ref }, true)} + compact={compact} + note={note} + diff={diff?.message ?? null} + places={diff?.places} + /> + ) : view === "read" ? ( + + ) : view === "lexicon" ? ( + onChange({ lex }, true)} + // read the verse here; with sync, the other reading panes follow + onOpenPassage={onOpenPassage} + onSearch={async (query) => { + // start the search, so that its results are there straight away + try { + const started = await api.search(card.name, query); + onChange({ view: "search", query, search: started.id, result: null }, true); + } catch { + onChange({ view: "search", query, search: null, result: null }, true); + } + }} + compact={compact} + /> + ) : ( + onChange({ query })} + onSearch={(search) => onChange({ search, result: null }, true)} + ask={pane.ask} + onProposal={(query, ask) => onChange({ query, ask })} + onExpand={(result) => onChange({ result })} + onOpenPassage={(ref) => onChange({ view: "read", ref }, true)} + otherCorpora={corpora.filter((c) => c !== card.name)} + onOpenPane={onOpenPane} + compact={compact} + /> + )} + + {(hovered) => ( + + )} + + + )} +
    +
    + ); +} + +export function App() { + const [ws, actions] = useWorkspace(); + const [settings, setSettings] = useSettings(); + const [theme, toggleTheme] = useTheme(); + const [inspected, setInspected] = useState<{ corpus: string; node: number } | null>(null); + const [library, setLibrary] = useState(false); + const [notes, setNotes] = useState>({}); + const differences = useDifferences(ws.panes, ws.diff); + + // an identity per pane, for React, so that closing one doesn't hand its neighbour's + // props (and state) to it; panes added or taken away by the URL get new ones at the end + const paneIds = useRef([]); + const nextPaneId = useRef(0); + while (paneIds.current.length < ws.panes.length) paneIds.current.push(nextPaneId.current++); + paneIds.current.length = ws.panes.length; + const closePane = (index: number) => { + if (ws.panes.length <= 1) return; + paneIds.current.splice(index, 1); + // the notes of the panes after it move up with them + setNotes((n) => { + const moved: Record = {}; + for (const [k, v] of Object.entries(n)) { + const i = Number(k); + if (i !== index) moved[i > index ? i - 1 : i] = v; + } + return moved; + }); + actions.removePane(index); + }; + + const corporaQuery = useQuery({ queryKey: ["corpora"], queryFn: api.corpora }); + const corpora = corporaQuery.data?.corpora.map((c) => c.name) ?? []; + const firstCorpus = corpora[0] ?? null; + + // panes without a corpus get the first one + useEffect(() => { + if (!firstCorpus) return; + const missing = new Map>(); + ws.panes.forEach((p, i) => { + if (!p.corpus) missing.set(i, { corpus: firstCorpus }); + }); + if (missing.size) actions.updatePanes(missing); + }, [ws.panes, firstCorpus, actions]); + + // started without corpora: offer the library straight away, once + const noCorpora = corporaQuery.data !== undefined && corpora.length === 0; + const offeredLibrary = useRef(false); + useEffect(() => { + if (noCorpora && !offeredLibrary.current) { + offeredLibrary.current = true; + setLibrary(true); + } + }, [noCorpora]); + + const names = [...new Set(ws.panes.map((p) => p.corpus).filter((c): c is string => Boolean(c)))]; + const cards = useQuery({ + queryKey: ["cards", ...names], + queryFn: async () => { + const entries = await Promise.allSettled(names.map(corpusCard)); + const result: Record = {}; + entries.forEach((e, i) => { + result[names[i]] = e.status === "fulfilled" ? { card: e.value } : { error: e.reason }; + }); + return result; + }, + enabled: names.length > 0, + placeholderData: (previous) => previous, + }); + const cardOf = (name: string | null) => (name ? cards.data?.[name] : undefined); + const nameList = names.join(" + "); + + useEffect(() => { + document.title = nameList ? `${nameList}: CACCHT Explorer` : "CACCHT Explorer"; + }, [nameList]); + + // With synchronised reading, reading panes without a passage (a short link, + // a new pane) open the passage of the first pane that has one. + useEffect(() => { + if (!ws.sync) return; + const source = ws.panes.find((p) => p.view === "read" && p.corpus && p.ref); + const missing = ws.panes + .map((p, i) => (p.view === "read" && p.corpus && !p.ref ? i : -1)) + .filter((i) => i >= 0); + if (!source || !missing.length) return; + let current = true; + (async () => { + const changes = new Map>(); + const newNotes: Record = {}; + await Promise.all( + missing.map(async (i) => { + const p = ws.panes[i]; + try { + const target = await api.align(p.corpus!, source.corpus!, source.ref!, p.by); + changes.set(i, { ref: target.ref }); + newNotes[i] = target.exact + ? null + : `${displayRef(source.ref!)} is not in ${p.corpus}; showing ${displayRef(target.ref)}.`; + } catch { + newNotes[i] = `${p.corpus} has no counterpart of ${displayRef(source.ref!)}.`; + } + }), + ); + if (!current) return; + if (changes.size) actions.updatePanes(changes); + setNotes((n) => ({ ...n, ...newNotes })); + })(); + return () => { + current = false; + }; + }, [ws.panes, ws.sync, actions]); + + /** Open a passage in a pane; with synchronised reading, the other reading panes follow. */ + const openPassage = useCallback( + async (index: number, ref: string) => { + const source = ws.panes[index]; + if (!ws.sync || !source.corpus) { + setNotes((n) => ({ ...n, [index]: null })); + actions.updatePane(index, { view: "read", ref }, true); + return; + } + const sourceCorpus = source.corpus; + const changes = new Map>([[index, { view: "read", ref }]]); + const newNotes: Record = { [index]: null }; + await Promise.all( + ws.panes.map(async (p, i) => { + if (i === index || p.view !== "read" || !p.corpus) return; + try { + const target = await api.align(p.corpus, sourceCorpus, ref, p.by); + changes.set(i, { ref: target.ref }); + newNotes[i] = target.exact + ? null + : `${displayRef(ref)} is not in ${p.corpus}; showing ${displayRef(target.ref)}.`; + } catch { + newNotes[i] = `${p.corpus} has no counterpart of ${displayRef(ref)}.`; + } + }), + ); + actions.updatePanes(changes, true); + setNotes((n) => ({ ...n, ...newNotes })); + }, + [ws.panes, ws.sync, actions], + ); + + /** Read a pane by its own sections or by biblical book; with sync, at the passage of the other panes. */ + const setBy = async (index: number, by: ReadBy) => { + const pane = ws.panes[index]; + const other = ws.panes.find((p, i) => i !== index && p.view === "read" && p.corpus && p.ref); + let ref: string | null = null; + let note: string | null = null; + if (ws.sync && other && pane.corpus) { + try { + const target = await api.align(pane.corpus, other.corpus!, other.ref!, by); + ref = target.ref; + note = target.exact ? null : `${displayRef(other.ref!)} is not in ${pane.corpus}; showing ${displayRef(target.ref)}.`; + } catch { + note = `${pane.corpus} has no counterpart of ${displayRef(other.ref!)}.`; + } + } + setNotes((n) => ({ ...n, [index]: note })); + actions.updatePane(index, { by, ref }, true); + }; + + const addPane = (pane: Pane) => { + if (ws.panes.length < MAX_PANES) actions.addPane(pane); + }; + + if (corporaQuery.error) { + return ( +
    +

    The browser cannot reach its server

    +

    + Check that cfabric-browser is still running, then reload this page. +

    +
    + ); + } + if (!corporaQuery.data) return
    Opening…
    ; + + const firstCard = cardOf(ws.panes[0]?.corpus ?? null)?.card; + const inspectedCard = inspected ? cardOf(inspected.corpus)?.card : undefined; + const single = ws.panes.length === 1 && corpora.length === 1; + + return ( +
    +
    +
    + + CACCHT Explorer +
    + {single && firstCard ? ( + <> + + {firstCard.name} {firstCard.version} + + + + ) : null} +
    + {ws.panes.length > 1 ? ( + + ) : null} + {ws.panes.length > 1 ? ( + + ) : null} + + + + + describeWorkspace(ws)} /> + +
    +
    +
    +
    + {noCorpora ? ( +
    +

    No corpus is open

    +

    + Open one of the corpora on this computer to read, search and compare it. +

    + +
    + ) : null} + {noCorpora ? null : ws.panes.map((pane, i) => { + const entry = cardOf(pane.corpus); + return ( + actions.updatePane(i, change, push)} + onOpenPassage={(ref) => openPassage(i, ref)} + onBy={(by) => setBy(i, by)} + onOpenPane={addPane} + onClose={() => closePane(i)} + onInspect={(corpus, node) => setInspected({ corpus, node })} + /> + ); + })} +
    + {inspected && inspectedCard ? ( +
    + setInspected({ corpus: inspected.corpus, node })} + onClose={() => setInspected(null)} + onOpenPassage={(ref) => { + const i = ws.panes.findIndex((p) => p.corpus === inspected.corpus); + setInspected(null); + if (i >= 0) openPassage(i, ref); + }} + /> +
    + ) : null} +
    + {library ? ( + setLibrary(false)} + onOpen={(corpus) => { + setLibrary(false); + if (ws.panes.some((p) => p.corpus === corpus)) return; + // a pane still without a corpus (the browser started empty) gets it + const empty = ws.panes.findIndex((p) => !p.corpus); + if (empty >= 0) actions.updatePane(empty, { corpus }, true); + else addPane(emptyPane(corpus)); + }} + /> + ) : null} +
    + ); +} diff --git a/libs/explorer/frontend/src/Cite.tsx b/libs/explorer/frontend/src/Cite.tsx new file mode 100644 index 0000000..3dfb093 --- /dev/null +++ b/libs/explorer/frontend/src/Cite.tsx @@ -0,0 +1,110 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { displayRef } from "./display/Display"; +import { describeAsked, type Asked } from "./shared/asked"; +import { useHealth } from "./shared/health"; +import type { CorpusCard } from "./types"; +import { useCloseOutside } from "./shared/popover"; + +// A citation for what you are looking at: the passage, or the search with its +// template and number of results, plus the corpus (title, version, DOI) and the +// software that produced it. Copied as plain text. + +const useVersions = () => useHealth(); + +function corpusLine(corpus: CorpusCard): string { + // a translation: its edition and where the text came from + if (corpus.translation) return `${corpus.translation.cite}.`; + const title = corpus.title || corpus.name; + const doi = corpus.doi ? ` https://doi.org/${corpus.doi}.` : ""; + return `${title}, version ${corpus.version}.${doi}`; +} + +export function citePassage(corpus: CorpusCard, ref: string, software: string, date: string): string { + return `${displayRef(ref)}. ${corpusLine(corpus)} Read with ${software}, ${date}.`; +} + +export function citeSearch( + corpus: CorpusCard, + template: string, + count: number, + truncated: boolean, + software: string, + date: string, + asked: Asked | null = null, +): string { + const results = `${count.toLocaleString("en-US")}${truncated ? " (stopped at the limit)" : ""} results`; + const proposed = asked ? `\n${describeAsked(asked, template)}.` : ""; + return `Search in ${corpusLine(corpus)}\nTemplate:\n${template.trim()}\n${results}, found with ${software}, ${date}.${proposed}`; +} + +interface Props { + /** Builds the citation from the software description and today's date. */ + text: (software: string, date: string) => string; + label?: string; + /** Which edge of the button the panel lines up with: the end (default), or the start near the left of a bar. */ + align?: "start" | "end"; +} + +export function Cite({ text, label = "Cite", align = "end" }: Props) { + const [open, setOpen] = useState(false); + const [copied, setCopied] = useState(false); + const panel = useRef(null); + const { data } = useVersions(); + const software = data + ? `CACCHT Explorer ${data.version}${data.coreVersion ? ` (context-fabric ${data.coreVersion})` : ""}` + : "CACCHT Explorer"; + const date = new Date().toISOString().slice(0, 10); + const citation = text(software, date); + + const close = useCallback(() => setOpen(false), []); + useEffect(() => { + if (!open) return; + setCopied(false); + }, [open]); + useCloseOutside(open, close, panel); + + const copy = async () => { + try { + await navigator.clipboard.writeText(citation); + setCopied(true); + } catch { + // the text is selected in the box below for copying by hand + setCopied(false); + } + }; + + return ( +
    + + {open ? ( +
    +