Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions asr/evals/duhow_whisper_large_v3_turbo_cat.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"model": "duhow/whisper-large-v3-turbo-cat",
"params_b": null,
"memory_gb": null,
"hardware": "AMD RYZEN AI MAX+ PRO 395 w/ Radeon 8060S",
"evaluated_at": "2026-10-29T22:14:07+00:00",
"benchmarks": {
"fleurs_ca": {
"wer": 0.0983,
"cer": 0.0715,
"n": 400
},
"openslr69_ca": {
"wer": 0.048,
"cer": 0.017,
"n": 400
}
}
}
3 changes: 2 additions & 1 deletion asr/hf-eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class EvalResult:
"whisper-large-v3",
"whisper-large-v3-turbo",
"projecte-aina/whisper-large-v3-ca-3catparla",
"duhow/whisper-large-v3-turbo-cat",
]

GEMMA_MODELS = [
Expand Down Expand Up @@ -113,7 +114,7 @@ class WhisperWrapper:
def __init__(self, model_name: str, device: str):
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline

if "aina" not in model_name:
if "/" not in model_name:
model_id = f"openai/{model_name}"
else:
model_id = model_name
Expand Down
5 changes: 5 additions & 0 deletions asr/run_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@
"args": ["projecte-aina/whisper-large-v3-ca-3catparla"],
"output": "evals/whisper_large_v3_ca.json",
},
{
"label": "duhow/whisper-large-v3-turbo-cat",
"args": ["duhow/whisper-large-v3-turbo-cat"],
"output": "evals/duhow_whisper_large_v3_turbo_cat.json",
},
{
"label": "omniASR_CTC_300M",
"args": ["omniASR_CTC_300M"],
Expand Down
1 change: 1 addition & 0 deletions docs/asr.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ uv run python hf-eval.py --list-models
```bash
uv run python hf-eval.py whisper-large-v3 --device cuda --output evals/whisper_large_v3.json
uv run python hf-eval.py whisper-small --output evals/whisper_small.json
uv run python hf-eval.py duhow/whisper-large-v3-turbo-cat --device cuda --output evals/duhow_whisper_large_v3_turbo_cat.json
```

**Avaluar un model d'API:**
Expand Down
Loading