Regenerate R parity cache #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regenerate R parity cache | |
| on: | |
| repository_dispatch: | |
| types: [nns-r-package-updated] | |
| workflow_dispatch: | |
| inputs: | |
| r_commit: | |
| description: Exact commit in OVVO-Financial/NNS containing the package archives | |
| required: true | |
| type: string | |
| r_version: | |
| description: NNS package version (13.0 or newer) | |
| required: true | |
| type: string | |
| source_tarball: | |
| description: Source package filename at the R repository root | |
| required: true | |
| type: string | |
| windows_binary: | |
| description: Windows binary filename at the R repository root | |
| required: true | |
| type: string | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: r-parity-cache-${{ github.event.client_payload.r_version || inputs.r_version }} | |
| cancel-in-progress: true | |
| jobs: | |
| regenerate-cache: | |
| runs-on: ubuntu-latest | |
| env: | |
| RGL_USE_NULL: 'true' | |
| R_KEEP_PKG_SOURCE: 'yes' | |
| steps: | |
| - name: Check out NNS-python | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Resolve and validate package payload | |
| id: package | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" = "repository_dispatch" ]; then | |
| r_repo="${{ github.event.client_payload.r_repo }}" | |
| r_commit="${{ github.event.client_payload.r_commit }}" | |
| r_version="${{ github.event.client_payload.r_version }}" | |
| source_tarball="${{ github.event.client_payload.source_tarball }}" | |
| windows_binary="${{ github.event.client_payload.windows_binary }}" | |
| else | |
| r_repo="OVVO-Financial/NNS" | |
| r_commit="${{ inputs.r_commit }}" | |
| r_version="${{ inputs.r_version }}" | |
| source_tarball="${{ inputs.source_tarball }}" | |
| windows_binary="${{ inputs.windows_binary }}" | |
| fi | |
| test "${r_repo}" = "OVVO-Financial/NNS" || { | |
| echo "Unsupported upstream repository: ${r_repo}" | |
| exit 1 | |
| } | |
| printf '%s' "${r_commit}" | grep -Eq '^[0-9a-fA-F]{40}$' || { | |
| echo "r_commit must be a full 40-character commit SHA." | |
| exit 1 | |
| } | |
| printf '%s' "${source_tarball}" | grep -Eq '^NNS_[0-9][0-9A-Za-z.-]*\.(tar\.gz|tgz)$' || { | |
| echo "Invalid source package filename: ${source_tarball}" | |
| exit 1 | |
| } | |
| printf '%s' "${windows_binary}" | grep -Eq '^NNS_[0-9][0-9A-Za-z.-]*\.zip$' || { | |
| echo "Invalid Windows package filename: ${windows_binary}" | |
| exit 1 | |
| } | |
| python - "${r_version}" "${source_tarball}" "${windows_binary}" <<'PY' | |
| import re | |
| import sys | |
| version, source, binary = sys.argv[1:] | |
| match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?(?:[-+].*)?", version) | |
| if match is None: | |
| raise SystemExit(f"Unsupported NNS package version: {version!r}") | |
| if (int(match.group(1)), int(match.group(2))) < (13, 0): | |
| raise SystemExit(f"NNS {version} is below the supported minimum 13.0") | |
| expected = f"NNS_{version}" | |
| if not source.startswith(expected + ".") or binary != expected + ".zip": | |
| raise SystemExit("Package filenames do not match the dispatched NNS version") | |
| PY | |
| { | |
| echo "r_repo=${r_repo}" | |
| echo "r_commit=${r_commit}" | |
| echo "r_version=${r_version}" | |
| echo "source_tarball=${source_tarball}" | |
| echo "windows_binary=${windows_binary}" | |
| } >> "${GITHUB_OUTPUT}" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Set up R | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: release | |
| use-public-rspm: true | |
| - name: Install Linux system dependencies | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| libgsl-dev libjpeg-dev libpng-dev libtiff5-dev libfreetype6-dev \ | |
| libharfbuzz-dev libfribidi-dev xorg-dev | |
| - name: Download exact R package archives | |
| shell: bash | |
| env: | |
| R_REPO: ${{ steps.package.outputs.r_repo }} | |
| R_COMMIT: ${{ steps.package.outputs.r_commit }} | |
| SOURCE_TARBALL: ${{ steps.package.outputs.source_tarball }} | |
| WINDOWS_BINARY: ${{ steps.package.outputs.windows_binary }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p upstream/package | |
| base_url="https://raw.githubusercontent.com/${R_REPO}/${R_COMMIT}" | |
| curl --fail --location --retry 3 \ | |
| "${base_url}/${SOURCE_TARBALL}" \ | |
| --output "upstream/${SOURCE_TARBALL}" | |
| curl --fail --location --retry 3 \ | |
| "${base_url}/${WINDOWS_BINARY}" \ | |
| --output "upstream/${WINDOWS_BINARY}" | |
| tar -xzf "upstream/${SOURCE_TARBALL}" -C upstream/package --strip-components=1 | |
| test -f upstream/package/DESCRIPTION | |
| - name: Install R package dependencies | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| Rscript -e "options(repos=c(CRAN='https://cloud.r-project.org')); install.packages(c('remotes','jsonlite')); remotes::install_deps('upstream/package', dependencies=NA, upgrade='never')" | |
| - name: Install exact NNS source package | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| python scripts/install_local_r_nns.py \ | |
| --source "upstream/${{ steps.package.outputs.source_tarball }}" \ | |
| --expected-version "${{ steps.package.outputs.r_version }}" | |
| - name: Install Python package and test tools | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| python -m pip install -U pip | |
| python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy | |
| python -m pip install hypothesis pytest-benchmark pytest-xdist | |
| python -m pip install -e . --force-reinstall --no-deps | |
| - name: Regenerate complete live-R parity cache | |
| id: regenerate | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| python scripts/regenerate_r_cache.py --fresh --allow-ci 2>&1 | tee parity-regeneration.log | |
| - name: Confirm generated cache version | |
| if: always() | |
| shell: bash | |
| env: | |
| EXPECTED_VERSION: ${{ steps.package.outputs.r_version }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| cache_dir = Path('tests/_r_cache') | |
| shards = sorted(cache_dir.glob('*.json')) if cache_dir.is_dir() else [] | |
| if not shards: | |
| raise SystemExit('tests/_r_cache/ contains no shard files') | |
| expected = os.environ['EXPECTED_VERSION'] | |
| total = 0 | |
| for shard_path in shards: | |
| shard = json.loads(shard_path.read_text(encoding='utf-8')) | |
| actual = shard.get('nns_version') | |
| if actual != expected: | |
| raise SystemExit( | |
| f'{shard_path.name}: cache version {actual!r} ' | |
| f'!= dispatched version {expected!r}' | |
| ) | |
| entries = shard.get('entries') | |
| if not isinstance(entries, dict) or not entries: | |
| raise SystemExit(f'{shard_path.name} contains no entries') | |
| total += len(entries) | |
| print( | |
| f'Validated {total} cache entries across {len(shards)} ' | |
| f'shards for NNS {expected}.' | |
| ) | |
| PY | |
| - name: Remove transient cache files | |
| if: always() | |
| shell: bash | |
| run: rm -rf tests/_r_cache.bak && rm -f tests/_r_cache.lock | |
| - name: Record cache provenance in sync manifest | |
| if: always() | |
| shell: bash | |
| env: | |
| R_COMMIT: ${{ steps.package.outputs.r_commit }} | |
| R_VERSION: ${{ steps.package.outputs.r_version }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| path = Path('sync/nns_source.json') | |
| manifest = json.loads(path.read_text(encoding='utf-8')) | |
| manifest['r_commit'] = os.environ['R_COMMIT'] | |
| manifest['r_version'] = os.environ['R_VERSION'] | |
| path.write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8') | |
| print(f"Manifest now records {manifest['r_repo']}@{manifest['r_commit']} " | |
| f"(NNS {manifest['r_version']}).") | |
| PY | |
| - name: Summarize cache changes by function | |
| id: cachediff | |
| if: always() | |
| shell: bash | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import subprocess | |
| from pathlib import Path | |
| def committed(name: str) -> dict: | |
| proc = subprocess.run( | |
| ['git', 'show', f'HEAD:tests/_r_cache/{name}'], | |
| capture_output=True, text=True, | |
| ) | |
| if proc.returncode != 0: | |
| return {} | |
| return json.loads(proc.stdout).get('entries', {}) | |
| cache_dir = Path('tests/_r_cache') | |
| lines = ['### Cache changes by function', ''] | |
| if not cache_dir.is_dir(): | |
| lines.append('_No cache directory was generated; see the regeneration log._') | |
| else: | |
| proc = subprocess.run( | |
| ['git', 'ls-tree', '--name-only', 'HEAD', 'tests/_r_cache/'], | |
| capture_output=True, text=True, | |
| ) | |
| old_names = {Path(p).name for p in proc.stdout.split() if p.endswith('.json')} | |
| new_names = {p.name for p in cache_dir.glob('*.json')} | |
| rows = [] | |
| for name in sorted(old_names | new_names): | |
| before = committed(name) if name in old_names else {} | |
| after = ( | |
| json.loads((cache_dir / name).read_text()).get('entries', {}) | |
| if name in new_names else {} | |
| ) | |
| added = len(set(after) - set(before)) | |
| removed = len(set(before) - set(after)) | |
| changed = sum( | |
| 1 for k in set(before) & set(after) if before[k] != after[k] | |
| ) | |
| if added or removed or changed: | |
| label = name[: -len('.json')] | |
| rows.append( | |
| f'| `{label}` | {len(before)} | {len(after)} ' | |
| f'| {added} | {removed} | {changed} |' | |
| ) | |
| if rows: | |
| lines += [ | |
| '| function | before | after | added | removed | changed |', | |
| '| --- | --- | --- | --- | --- | --- |', | |
| *rows, | |
| ] | |
| else: | |
| lines.append('_No cache entries changed._') | |
| Path('cache-diff.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') | |
| print('\n'.join(lines)) | |
| PY | |
| { | |
| echo 'summary<<CACHE_DIFF_EOF' | |
| cat cache-diff.md | |
| echo 'CACHE_DIFF_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Verify committed-cache mode | |
| id: verify | |
| if: always() | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| { | |
| python -m pytest -q tests/invariants | |
| NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity | |
| } 2>&1 | tee committed-cache-verification.log | |
| - name: Upload parity diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: r-parity-cache-diagnostics-${{ steps.package.outputs.r_version }} | |
| path: | | |
| parity-regeneration.log | |
| committed-cache-verification.log | |
| if-no-files-found: warn | |
| - name: Open or update R cache regeneration PR | |
| if: always() | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.OVVO_SYNC_TOKEN || github.token }} | |
| branch: automation/r-cache-nns-${{ steps.package.outputs.r_version }} | |
| delete-branch: true | |
| commit-message: Regenerate R parity cache for NNS ${{ steps.package.outputs.r_version }} | |
| title: Regenerate R parity cache for NNS ${{ steps.package.outputs.r_version }} | |
| body: | | |
| Automatic parity-cache refresh from the R source of truth. | |
| - R repository: `${{ steps.package.outputs.r_repo }}` | |
| - R commit: `${{ steps.package.outputs.r_commit }}` | |
| - NNS version: `${{ steps.package.outputs.r_version }}` | |
| - Source package: `${{ steps.package.outputs.source_tarball }}` | |
| - Windows binary verified: `${{ steps.package.outputs.windows_binary }}` | |
| - Live regeneration result: `${{ steps.regenerate.outcome }}` | |
| - Cache-only verification result: `${{ steps.verify.outcome }}` | |
| The exact R source package was installed and the committed parity cache was regenerated. `sync/nns_source.json` records this R commit as the behavioral-truth provenance. Any remaining Python/R parity mismatches are retained in the workflow diagnostics and should be repaired against this R-authored baseline. | |
| ${{ steps.cachediff.outputs.summary }} | |
| add-paths: | | |
| tests/_r.py | |
| tests/_r_cache/** | |
| sync/nns_source.json | |
| - name: Report parity status | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "Live regeneration outcome: ${{ steps.regenerate.outcome }}" | |
| echo "Cache-only verification outcome: ${{ steps.verify.outcome }}" | |
| echo "The regenerated R cache has been preserved in its pull request." |