add token for statement translation training

2026-05-10 18:39:03 -07:00 · 2025-07-24 17:50:37 -05:00
parent 06fffdf8c7
commit 69f28e59cb
1 changed files with 7 additions and 0 deletions
@@ -1085,6 +1085,13 @@
      "normalized": false,
      "single_word": false
    },
+    {
+      "content": "FORMAT_WITH_SPEC",
+      "lstrip": false,
+      "rstrip": true,
+      "normalized": false,
+      "single_word": false
+    },
    {
      "content": "GEN_START",
      "lstrip": false,