Update token config (#4)
Browse files- Updated configuration files w/ special tokens (04c7faae76950335500370f9f73a9958bcdd0251)
- special_tokens_map.json +5 -1
- tokenizer_config.json +5 -1
special_tokens_map.json
CHANGED
|
@@ -45,5 +45,9 @@
|
|
| 45 |
"⟨H≈1.6⟩",
|
| 46 |
"⟨H≈1.7⟩",
|
| 47 |
"⟨H≈1.8⟩"
|
| 48 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
}
|
|
|
|
| 45 |
"⟨H≈1.6⟩",
|
| 46 |
"⟨H≈1.7⟩",
|
| 47 |
"⟨H≈1.8⟩"
|
| 48 |
+
],
|
| 49 |
+
"bos_token": "<|begin_of_text|>",
|
| 50 |
+
"eos_token": "<|end_of_text|>",
|
| 51 |
+
"pad_token": "[PAD]",
|
| 52 |
+
"unk_token": "[UNK]"
|
| 53 |
}
|
tokenizer_config.json
CHANGED
|
@@ -440,8 +440,12 @@
|
|
| 440 |
"⟨H≈1.7⟩",
|
| 441 |
"⟨H≈1.8⟩"
|
| 442 |
],
|
|
|
|
| 443 |
"clean_up_tokenization_spaces": true,
|
|
|
|
| 444 |
"extra_special_tokens": {},
|
| 445 |
"model_max_length": 1000000000000000019884624838656,
|
| 446 |
-
"
|
|
|
|
|
|
|
| 447 |
}
|
|
|
|
| 440 |
"⟨H≈1.7⟩",
|
| 441 |
"⟨H≈1.8⟩"
|
| 442 |
],
|
| 443 |
+
"bos_token": "<|begin_of_text|>",
|
| 444 |
"clean_up_tokenization_spaces": true,
|
| 445 |
+
"eos_token": "<|end_of_text|>",
|
| 446 |
"extra_special_tokens": {},
|
| 447 |
"model_max_length": 1000000000000000019884624838656,
|
| 448 |
+
"pad_token": "[PAD]",
|
| 449 |
+
"tokenizer_class": "PreTrainedTokenizer",
|
| 450 |
+
"unk_token": "[UNK]"
|
| 451 |
}
|