Skip to content

Commit a22f108

Browse files
committed
Update equivalence.rs
1 parent 799d04e commit a22f108

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

crates/bpe/benchmarks/equivalence.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ fn test_compare_dictionary() {
2626
let mut hugging_tokens = huggingface.get_vocab(false);
2727
// HACK: There are incorrect vocabularies in huggingface which have the added tokens stored together with the base tokens..
2828
// This is a workaround to remove them.
29-
for (added_token, _) in huggingface.get_added_vocabulary().get_vocab() {
29+
for added_token in huggingface.get_added_vocabulary().get_vocab().keys() {
3030
hugging_tokens.remove(added_token);
3131
}
3232
let mut hugging_tokens: Vec<_> = hugging_tokens.into_iter().collect();

0 commit comments

Comments
 (0)