diff --git a/dev_scripts/statement/tokenizer/special_tokens_map.json b/dev_scripts/statement/tokenizer/special_tokens_map.json index 5d7a5d0..47193ed 100644 --- a/dev_scripts/statement/tokenizer/special_tokens_map.json +++ b/dev_scripts/statement/tokenizer/special_tokens_map.json @@ -1085,6 +1085,13 @@ "normalized": false, "single_word": false }, + { + "content": "FORMAT_WITH_SPEC", + "lstrip": false, + "rstrip": true, + "normalized": false, + "single_word": false + }, { "content": "GEN_START", "lstrip": false,