Pclanglais commited on
Commit
a673605
·
verified ·
1 Parent(s): 7cdbfe5

Update token config (#4)

Browse files

- Updated configuration files w/ special tokens (04c7faae76950335500370f9f73a9958bcdd0251)

Files changed (2) hide show
  1. special_tokens_map.json +5 -1
  2. tokenizer_config.json +5 -1
special_tokens_map.json CHANGED
@@ -45,5 +45,9 @@
45
  "⟨H≈1.6⟩",
46
  "⟨H≈1.7⟩",
47
  "⟨H≈1.8⟩"
48
- ]
 
 
 
 
49
  }
 
45
  "⟨H≈1.6⟩",
46
  "⟨H≈1.7⟩",
47
  "⟨H≈1.8⟩"
48
+ ],
49
+ "bos_token": "<|begin_of_text|>",
50
+ "eos_token": "<|end_of_text|>",
51
+ "pad_token": "[PAD]",
52
+ "unk_token": "[UNK]"
53
  }
tokenizer_config.json CHANGED
@@ -440,8 +440,12 @@
440
  "⟨H≈1.7⟩",
441
  "⟨H≈1.8⟩"
442
  ],
 
443
  "clean_up_tokenization_spaces": true,
 
444
  "extra_special_tokens": {},
445
  "model_max_length": 1000000000000000019884624838656,
446
- "tokenizer_class": "PreTrainedTokenizer"
 
 
447
  }
 
440
  "⟨H≈1.7⟩",
441
  "⟨H≈1.8⟩"
442
  ],
443
+ "bos_token": "<|begin_of_text|>",
444
  "clean_up_tokenization_spaces": true,
445
+ "eos_token": "<|end_of_text|>",
446
  "extra_special_tokens": {},
447
  "model_max_length": 1000000000000000019884624838656,
448
+ "pad_token": "[PAD]",
449
+ "tokenizer_class": "PreTrainedTokenizer",
450
+ "unk_token": "[UNK]"
451
  }