HoangHa commited on
Commit
30587ac
·
verified ·
1 Parent(s): 8f561e9

fix: tokenizer_class TokenizersBackend -> PreTrainedTokenizerFast

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -14,7 +14,7 @@
14
  "padding_side": "left",
15
  "sp_model_kwargs": {},
16
  "spaces_between_special_tokens": false,
17
- "tokenizer_class": "TokenizersBackend",
18
  "use_default_system_prompt": false,
19
  "use_fast": true
20
- }
 
14
  "padding_side": "left",
15
  "sp_model_kwargs": {},
16
  "spaces_between_special_tokens": false,
17
+ "tokenizer_class": "PreTrainedTokenizerFast",
18
  "use_default_system_prompt": false,
19
  "use_fast": true
20
+ }