modify eos

special_tokens_map.json

@@ -1,5 +1,5 @@
 {
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer.json

@@ -5,13 +5,32 @@
   "added_tokens": [
     {
       "id": 50256,
-      "content": "<|endoftext|>",
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50257,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50258,
+      "content": "<unk>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
     }
+
   ],
   "normalizer": null,
   "pre_tokenizer": {
@@ -50296,7 +50315,9 @@
       "ĠCollider": 50253,
       "Ġinformants": 50254,
       "Ġgazed": 50255,
-      "<|endoftext|>": 50256
+      "</s>": 50256,
+      "<s>": 50257,
+      "<unk>": 50258
     },
     "merges": [
       "Ġ t",
@@ -100301,4 +100322,4 @@
       "Ġg azed"
     ]
   }
-}
+}

tokenizer_config.json

@@ -1,10 +1,10 @@
 {
   "add_prefix_space": false,
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
   "model_max_length": 1024,
   "name_or_path": "gpt2",
   "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<|endoftext|>"
+  "unk_token": "<unk>"
 }