Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
04a34ed
Squashed commit of the following:
jucor Nov 24, 2025
dddc4b1
Make each test run their own Conversation object
jucor Nov 24, 2025
18681a5
Actually reuse the fixtures...
jucor Nov 24, 2025
abee5b9
Enable parallel test execution by dataset using xdist_group markers
jucor Nov 24, 2025
0fa48dc
Factorize xdist_group pattern across test suite for parallel execution
jucor Nov 24, 2025
b6ed81e
Add --datasets option to run tests on specific datasets
jucor Nov 24, 2025
d001cf0
Fix --datasets filter for test_legacy_clojure_regression.py
jucor Nov 24, 2025
df072a1
Add normalized angle computation to PCA regression test
jucor Nov 24, 2025
7cfcad5
Remove madness "Align with clojure" and update golden records
jucor Nov 24, 2025
641a620
Save difference log regardless of errors
jucor Dec 1, 2025
d4b3286
Account for flip sign in numerical difference reporting
jucor Dec 1, 2025
9a2bc23
Add PCA benchmark script
jucor Dec 1, 2025
12e4a6f
Replace handwritten PCA by SKLearn's implementation
jucor Dec 1, 2025
9613227
Replace projection by Sklearn's fit_transform for better performance.
jucor Dec 1, 2025
20081b9
Remove unused start_vectors argument
jucor Dec 1, 2025
b2a1fe1
Remove function wrapped_pca
jucor Dec 1, 2025
755f1d5
Reformat help message
jucor Dec 3, 2025
94e075b
Add cluster centers to PCA-related paths for sign flip handling
jucor Jan 5, 2026
df15985
Allow for a few components with higher error tolerance
jucor Jan 6, 2026
7ae89b8
Fix per-component PCA sign flip detection
jucor Jan 6, 2026
c9c9a77
Replace element-wise tolerance with projection metrics for regression…
jucor Jan 6, 2026
c471c51
Update biodiversity and vw golden snapshots with sklearn PCA
jucor Jan 6, 2026
aacbd51
Fix PCA warnings in test_conversation.py
jucor Jan 6, 2026
7ea05c6
Replace PCA Clojure comparison test with pass
jucor Jan 6, 2026
1921d11
Consolidate Clojure comparison tests into test_legacy_clojure_regress…
jucor Jan 6, 2026
6087ad7
Address Copilot review feedback on PCA implementation
jucor Jan 6, 2026
5bc313c
Document parallel test execution with pytest-xdist
jucor Jan 6, 2026
60abfa3
FIx path of mock file for test run_math_pipeline_e2e
jucor Jan 7, 2026
9ed3639
Clarify test name
jucor Jan 7, 2026
d053797
Merge branch 'edge' into pca_work
jucor Feb 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
550 changes: 550 additions & 0 deletions delphi/docs/regression_testing.md

Large diffs are not rendered by default.

174 changes: 174 additions & 0 deletions delphi/polismath/benchmarks/bench_pca.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""
Benchmark script for PCA computation performance.

Usage:
cd delphi
../.venv/bin/python -m polismath.benchmarks.bench_pca <votes_csv_path> [--runs N]
../.venv/bin/python -m polismath.benchmarks.bench_pca <votes_csv_path> --profile

Example:
../.venv/bin/python -m polismath.benchmarks.bench_pca real_data/.local/r7wehfsmutrwndviddnii-bg2050/2025-11-25-1909-r7wehfsmutrwndviddnii-votes.csv --runs 3
../.venv/bin/python -m polismath.benchmarks.bench_pca real_data/.local/r7wehfsmutrwndviddnii-bg2050/2025-11-25-1909-r7wehfsmutrwndviddnii-votes.csv --profile
"""

import time
from pathlib import Path

import click

from polismath.benchmarks.benchmark_utils import (
load_votes_from_csv,
extract_dataset_name,
votes_csv_argument,
runs_option,
)
from polismath.conversation import Conversation
from polismath.pca_kmeans_rep.pca import pca_project_dataframe


profile_option = click.option(
'--profile', '-p',
is_flag=True,
help='Run with line profiler on PCA functions',
)


def setup_conversation(votes_csv: Path) -> tuple[Conversation, str, int, float]:
"""
Load votes and setup conversation (without computing PCA).

Args:
votes_csv: Path to votes CSV file

Returns:
Tuple of (conversation, dataset_name, n_votes, setup_time)
"""
dataset_name = extract_dataset_name(votes_csv)

print(f"Loading votes from '{votes_csv}'...")
votes_dict = load_votes_from_csv(votes_csv)
n_votes = len(votes_dict['votes'])
print(f"Loaded {n_votes:,} votes")
print()

print("Setting up conversation with votes...")
setup_start = time.perf_counter()
conv = Conversation(dataset_name)
conv = conv.update_votes(votes_dict, recompute=False)
setup_time = time.perf_counter() - setup_start

print(f"Setup completed in {setup_time:.2f}s")
print(f" Matrix shape: {conv.raw_rating_mat.shape}")
print()

return conv, dataset_name, n_votes, setup_time


def benchmark_pca(votes_csv: Path, runs: int = 3) -> dict:
"""
Benchmark PCA computation on a dataset.

Args:
votes_csv: Path to votes CSV file
runs: Number of runs to average

Returns:
Dictionary with benchmark results
"""
conv, dataset_name, n_votes, setup_time = setup_conversation(votes_csv)

# Benchmark PCA computation
print(f"Benchmarking PCA computation ({runs} runs)...")
times = []
for i in range(runs):
# Reset PCA state to force recomputation
conv.pca = None
conv.proj = None

start = time.perf_counter()
conv._compute_pca()
elapsed = time.perf_counter() - start
times.append(elapsed)

n_components = conv.pca['comps'].shape[0] if conv.pca else 0
n_projections = len(conv.proj) if conv.proj else 0
print(f" Run {i+1}: {elapsed:.3f}s ({n_components} components, {n_projections} projections)")

avg = sum(times) / len(times)
min_time = min(times)
max_time = max(times)

print()
print("=" * 50)
print(f"Dataset: {dataset_name}")
print(f"Votes: {n_votes:,}")
print(f"Matrix shape: {conv.raw_rating_mat.shape}")
print(f"Average PCA time: {avg:.3f}s")
print(f"Min/Max: {min_time:.3f}s / {max_time:.3f}s")

# Calculate throughput metrics
n_participants = conv.raw_rating_mat.shape[0]
n_comments = conv.raw_rating_mat.shape[1]

# PCA complexity is roughly O(min(n,p) * n * p) for n samples and p features
operations = min(n_participants, n_comments) * n_participants * n_comments
print(f"Throughput: {operations/avg:,.0f} ops/sec (min(n,p) x n x p)")
print(f"Participants/sec: {n_participants/avg:,.0f}")

return {
'dataset': dataset_name,
'n_votes': n_votes,
'shape': conv.raw_rating_mat.shape,
'times': times,
'avg': avg,
'min': min_time,
'max': max_time,
'setup_time': setup_time,
}


def profile_pca(votes_csv: Path) -> None:
"""
Run line profiler on PCA functions.

Args:
votes_csv: Path to votes CSV file
"""
from line_profiler import LineProfiler

conv, _, _, _ = setup_conversation(votes_csv)

# Get clean matrix for profiling
clean_matrix = conv._get_clean_matrix()

# Setup line profiler
profiler = LineProfiler()
profiler.add_function(pca_project_dataframe)

# Run profiled
print("Running pca_project_dataframe with line profiler...")
profiler.runcall(pca_project_dataframe, clean_matrix, 2)

# Print results
print()
print("=" * 70)
print("LINE PROFILE RESULTS")
print("=" * 70)
profiler.print_stats()


@click.command()
@votes_csv_argument
@runs_option
@profile_option
def main(votes_csv: Path, runs: int, profile: bool):
"""Benchmark PCA computation performance."""
if profile:
profile_pca(votes_csv)
else:
benchmark_pca(votes_csv, runs)


if __name__ == '__main__':
main()
Loading
Loading