From 69f28e59cb1dd20419c85feb5579e062e588fda4 Mon Sep 17 00:00:00 2001
From: Joel Flores <jjf190004@utdallas.edu>
Date: Thu, 24 Jul 2025 17:50:37 -0500
Subject: [PATCH] add token for statement translation training

---
 dev_scripts/statement/tokenizer/special_tokens_map.json | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/dev_scripts/statement/tokenizer/special_tokens_map.json b/dev_scripts/statement/tokenizer/special_tokens_map.json
index 5d7a5d0..47193ed 100644
--- a/dev_scripts/statement/tokenizer/special_tokens_map.json
+++ b/dev_scripts/statement/tokenizer/special_tokens_map.json
@@ -1085,6 +1085,13 @@
       "normalized": false,
       "single_word": false
     },
+    {
+      "content": "FORMAT_WITH_SPEC",
+      "lstrip": false,
+      "rstrip": true,
+      "normalized": false,
+      "single_word": false
+    },
     {
       "content": "GEN_START",
       "lstrip": false,