From 39d586ca78a7ed1af9f3ed5acb3646d76019d32b Mon Sep 17 00:00:00 2001 From: Jordi Mas Date: Fri, 30 Oct 2026 08:11:30 +0100 Subject: [PATCH 1/2] Add duhow Whisper large v3 turbo cat evaluation results --- .../duhow_whisper_large_v3_turbo_cat.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 asr/evals/duhow_whisper_large_v3_turbo_cat.json diff --git a/asr/evals/duhow_whisper_large_v3_turbo_cat.json b/asr/evals/duhow_whisper_large_v3_turbo_cat.json new file mode 100644 index 0000000..d0abe9b --- /dev/null +++ b/asr/evals/duhow_whisper_large_v3_turbo_cat.json @@ -0,0 +1,19 @@ +{ + "model": "duhow/whisper-large-v3-turbo-cat", + "params_b": null, + "memory_gb": null, + "hardware": "AMD RYZEN AI MAX+ PRO 395 w/ Radeon 8060S", + "evaluated_at": "2026-10-29T22:14:07+00:00", + "benchmarks": { + "fleurs_ca": { + "wer": 0.0983, + "cer": 0.0715, + "n": 400 + }, + "openslr69_ca": { + "wer": 0.048, + "cer": 0.017, + "n": 400 + } + } +} \ No newline at end of file From eca55da9f07337499a1f97c9a3a26124d41b3f40 Mon Sep 17 00:00:00 2001 From: Jordi Mas Date: Fri, 30 Oct 2026 08:13:03 +0100 Subject: [PATCH 2/2] Include DuHow Whisper turbo cat in future ASR evaluations --- asr/hf-eval.py | 3 ++- asr/run_evals.py | 5 +++++ docs/asr.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/asr/hf-eval.py b/asr/hf-eval.py index 9f7e6b4..b4d47d6 100644 --- a/asr/hf-eval.py +++ b/asr/hf-eval.py @@ -80,6 +80,7 @@ class EvalResult: "whisper-large-v3", "whisper-large-v3-turbo", "projecte-aina/whisper-large-v3-ca-3catparla", + "duhow/whisper-large-v3-turbo-cat", ] GEMMA_MODELS = [ @@ -113,7 +114,7 @@ class WhisperWrapper: def __init__(self, model_name: str, device: str): from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline - if "aina" not in model_name: + if "/" not in model_name: model_id = f"openai/{model_name}" else: model_id = model_name diff --git a/asr/run_evals.py b/asr/run_evals.py index 04f0aac..d265422 100644 --- a/asr/run_evals.py +++ b/asr/run_evals.py @@ -56,6 +56,11 @@ "args": ["projecte-aina/whisper-large-v3-ca-3catparla"], "output": "evals/whisper_large_v3_ca.json", }, + { + "label": "duhow/whisper-large-v3-turbo-cat", + "args": ["duhow/whisper-large-v3-turbo-cat"], + "output": "evals/duhow_whisper_large_v3_turbo_cat.json", + }, { "label": "omniASR_CTC_300M", "args": ["omniASR_CTC_300M"], diff --git a/docs/asr.md b/docs/asr.md index 2c80edf..d0b96b1 100644 --- a/docs/asr.md +++ b/docs/asr.md @@ -36,6 +36,7 @@ uv run python hf-eval.py --list-models ```bash uv run python hf-eval.py whisper-large-v3 --device cuda --output evals/whisper_large_v3.json uv run python hf-eval.py whisper-small --output evals/whisper_small.json +uv run python hf-eval.py duhow/whisper-large-v3-turbo-cat --device cuda --output evals/duhow_whisper_large_v3_turbo_cat.json ``` **Avaluar un model d'API:**