zack3d/pylingual
1
0
Fork 0
mirror of https://github.com/syssec-utd/pylingual.git synced 2026-05-10 18:39:03 -07:00
Code Issues Packages Projects Releases Wiki Activity

add token for statement translation training

Browse Source
This commit is contained in:
Joel Flores
2025-07-24 17:50:37 -05:00
parent 06fffdf8c7
commit 69f28e59cb
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev_scripts/statement/tokenizer/special_tokens_map.json
+7
View File
@@ -1085,6 +1085,13 @@
"normalized": false, "normalized": false,
"single_word": false "single_word": false
}, },
{
"content": "FORMAT_WITH_SPEC",
"lstrip": false,
"rstrip": true,
"normalized": false,
"single_word": false
},
{ {
"content": "GEN_START", "content": "GEN_START",
"lstrip": false, "lstrip": false,