We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 799d04e commit a22f108Copy full SHA for a22f108
crates/bpe/benchmarks/equivalence.rs
@@ -26,7 +26,7 @@ fn test_compare_dictionary() {
26
let mut hugging_tokens = huggingface.get_vocab(false);
27
// HACK: There are incorrect vocabularies in huggingface which have the added tokens stored together with the base tokens..
28
// This is a workaround to remove them.
29
- for (added_token, _) in huggingface.get_added_vocabulary().get_vocab() {
+ for added_token in huggingface.get_added_vocabulary().get_vocab().keys() {
30
hugging_tokens.remove(added_token);
31
}
32
let mut hugging_tokens: Vec<_> = hugging_tokens.into_iter().collect();
0 commit comments