diff --git a/dashscope/tokenizers/qwen_tokenizer.py b/dashscope/tokenizers/qwen_tokenizer.py index bfa1119..cbac668 100644 --- a/dashscope/tokenizers/qwen_tokenizer.py +++ b/dashscope/tokenizers/qwen_tokenizer.py @@ -58,7 +58,6 @@ def __init__(self, vocab_file, errors="replace", extra_vocab_file=None): token: index for index, token in SPECIAL_TOKENS } - # try load extra vocab from file if extra_vocab_file is not None: used_ids = set(self._mergeable_ranks.values()) | set( self._special_tokens.values(),