Upload 4 files

added_tokens.json

@@ -99,9 +99,6 @@
   "<extra_id_98>": 32001,
   "<extra_id_99>": 32000,
   "<extra_id_9>": 32090,
-  "<s>": 32100,
-  "article:": 32101,
-  "category:": 32103,
-  "classifiction:": 32102,
-  "generation:": 32104
+  "article: ": 32100,
+  "category: ": 32101
 }

special_tokens_map.json

@@ -1,35 +1,20 @@
 {
   "additional_special_tokens": [
     {
-      "content": "article:",
+      "content": "article: ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     },
     {
-      "content": "classifiction:",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "category:",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "generation:",
+      "content": "category: ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
     }
   ],
-  "bos_token": "<s>",
   "eos_token": {
     "content": "</s>",
     "lstrip": false,

tokenizer_config.json

@@ -825,7 +825,7 @@
       "special": true
     },
     "32100": {
-      "content": "<s>",
+      "content": "article: ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -833,31 +833,7 @@
       "special": true
     },
     "32101": {
-      "content": "article:",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32102": {
-      "content": "classifiction:",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32103": {
-      "content": "category:",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32104": {
-      "content": "generation:",
+      "content": "category: ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -866,12 +842,9 @@
     }
   },
   "additional_special_tokens": [
-    "article:",
-    "classifiction:",
-    "category:",
-    "generation:"
+    "article: ",
+    "category: "
   ],
-  "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,