Upload 12 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,531 @@
+---
+license: mit
+language:
+- en
+- ru
+metrics:
+- accuracy
+- f1
+- recall
+library_name: transformers
+pipeline_tag: sentence-similarity
+tags:
+- mteb
+- retrieval
+- retriever
+- pruned
+- e5
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+model-index:
+- name: e5-large-en-ru
+  results:
+  - task:
+      type: Classification
+    dataset:
+      type: mteb/amazon_counterfactual
+      name: MTEB AmazonCounterfactualClassification (en)
+      config: en
+      split: test
+      revision: e8379541af4e31359cca9fbcf4b00f2671dba205
+    metrics:
+    - type: accuracy
+      value: 79.5671641791045
+    - type: ap
+      value: 44.011060753169424
+    - type: f1
+      value: 73.76504135120175
+  - task:
+      type: Reranking
+    dataset:
+      type: mteb/askubuntudupquestions-reranking
+      name: MTEB AskUbuntuDupQuestions
+      config: default
+      split: test
+      revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
+    metrics:
+    - type: map
+      value: 57.69669466706412
+    - type: mrr
+      value: 70.61370531592138
+  - task:
+      type: STS
+    dataset:
+      type: mteb/biosses-sts
+      name: MTEB BIOSSES
+      config: default
+      split: test
+      revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
+    metrics:
+    - type: cos_sim_pearson
+      value: 86.36465960226795
+    - type: cos_sim_spearman
+      value: 84.57602350761223
+    - type: euclidean_pearson
+      value: 84.31391364490506
+    - type: euclidean_spearman
+      value: 84.57602350761223
+    - type: manhattan_pearson
+      value: 84.15796224236456
+    - type: manhattan_spearman
+      value: 84.3645729064343
+  - task:
+      type: Reranking
+    dataset:
+      type: mteb/mind_small
+      name: MTEB MindSmallReranking
+      config: default
+      split: test
+      revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69
+    metrics:
+    - type: map
+      value: 31.105698873583098
+    - type: mrr
+      value: 32.163780846856206
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sickr-sts
+      name: MTEB SICK-R
+      config: default
+      split: test
+      revision: a6ea5a8cab320b040a23452cc28066d9beae2cee
+    metrics:
+    - type: cos_sim_pearson
+      value: 83.75973907678062
+    - type: cos_sim_spearman
+      value: 80.54994608351296
+    - type: euclidean_pearson
+      value: 80.58496551316748
+    - type: euclidean_spearman
+      value: 80.54993996457814
+    - type: manhattan_pearson
+      value: 80.49280884070782
+    - type: manhattan_spearman
+      value: 80.41230093993471
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts12-sts
+      name: MTEB STS12
+      config: default
+      split: test
+      revision: a0d554a64d88156834ff5ae9920b964011b16384
+    metrics:
+    - type: cos_sim_pearson
+      value: 87.345503928209
+    - type: cos_sim_spearman
+      value: 80.4634619001261
+    - type: euclidean_pearson
+      value: 84.2666575030677
+    - type: euclidean_spearman
+      value: 80.46347579495351
+    - type: manhattan_pearson
+      value: 84.14370038922885
+    - type: manhattan_spearman
+      value: 80.36565043629274
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts13-sts
+      name: MTEB STS13
+      config: default
+      split: test
+      revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
+    metrics:
+    - type: cos_sim_pearson
+      value: 75.14644787456163
+    - type: cos_sim_spearman
+      value: 75.88443166051762
+    - type: euclidean_pearson
+      value: 76.19117255044588
+    - type: euclidean_spearman
+      value: 75.88443166051762
+    - type: manhattan_pearson
+      value: 76.00450128624708
+    - type: manhattan_spearman
+      value: 75.69943934692938
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts14-sts
+      name: MTEB STS14
+      config: default
+      split: test
+      revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
+    metrics:
+    - type: cos_sim_pearson
+      value: 77.60763524019471
+    - type: cos_sim_spearman
+      value: 77.2591077818027
+    - type: euclidean_pearson
+      value: 77.14021401348042
+    - type: euclidean_spearman
+      value: 77.25911027186999
+    - type: manhattan_pearson
+      value: 76.87139081109731
+    - type: manhattan_spearman
+      value: 76.98379627773018
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts15-sts
+      name: MTEB STS15
+      config: default
+      split: test
+      revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
+    metrics:
+    - type: cos_sim_pearson
+      value: 88.18321035966198
+    - type: cos_sim_spearman
+      value: 89.0469892725742
+    - type: euclidean_pearson
+      value: 88.05085809092137
+    - type: euclidean_spearman
+      value: 89.04698194601134
+    - type: manhattan_pearson
+      value: 88.03620967628684
+    - type: manhattan_spearman
+      value: 89.02859425307943
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts16-sts
+      name: MTEB STS16
+      config: default
+      split: test
+      revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
+    metrics:
+    - type: cos_sim_pearson
+      value: 82.39166503459249
+    - type: cos_sim_spearman
+      value: 83.71826060604693
+    - type: euclidean_pearson
+      value: 82.70145770530107
+    - type: euclidean_spearman
+      value: 83.71826045549452
+    - type: manhattan_pearson
+      value: 82.56870669205291
+    - type: manhattan_spearman
+      value: 83.55353737670136
+  - task:
+      type: STS
+    dataset:
+      type: mteb/sts17-crosslingual-sts
+      name: MTEB STS17 (en-en)
+      config: en-en
+      split: test
+      revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
+    metrics:
+    - type: cos_sim_pearson
+      value: 89.58290721169323
+    - type: cos_sim_spearman
+      value: 89.25956993522081
+    - type: euclidean_pearson
+      value: 89.4716703635447
+    - type: euclidean_spearman
+      value: 89.25956993522081
+    - type: manhattan_pearson
+      value: 89.4475864648432
+    - type: manhattan_spearman
+      value: 89.14694174575615
+  - task:
+      type: Reranking
+    dataset:
+      type: mteb/scidocs-reranking
+      name: MTEB SciDocsRR
+      config: default
+      split: test
+      revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
+    metrics:
+    - type: map
+      value: 81.4879065181404
+    - type: mrr
+      value: 94.81295937178291
+  - task:
+      type: PairClassification
+    dataset:
+      type: mteb/sprintduplicatequestions-pairclassification
+      name: MTEB SprintDuplicateQuestions
+      config: default
+      split: test
+      revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
+    metrics:
+    - type: cos_sim_accuracy
+      value: 99.73960396039604
+    - type: cos_sim_ap
+      value: 92.70840767967965
+    - type: cos_sim_f1
+      value: 86.90890990542557
+    - type: cos_sim_precision
+      value: 86.5213082259663
+    - type: cos_sim_recall
+      value: 87.3
+    - type: dot_accuracy
+      value: 99.73960396039604
+    - type: dot_ap
+      value: 92.70828452993575
+    - type: dot_f1
+      value: 86.90890990542557
+    - type: dot_precision
+      value: 86.5213082259663
+    - type: dot_recall
+      value: 87.3
+    - type: euclidean_accuracy
+      value: 99.73960396039604
+    - type: euclidean_ap
+      value: 92.7084093403562
+    - type: euclidean_f1
+      value: 86.90890990542557
+    - type: euclidean_precision
+      value: 86.5213082259663
+    - type: euclidean_recall
+      value: 87.3
+    - type: manhattan_accuracy
+      value: 99.74059405940594
+    - type: manhattan_ap
+      value: 92.7406819850299
+    - type: manhattan_f1
+      value: 87.01234567901234
+    - type: manhattan_precision
+      value: 85.95121951219512
+    - type: manhattan_recall
+      value: 88.1
+    - type: max_accuracy
+      value: 99.74059405940594
+    - type: max_ap
+      value: 92.7406819850299
+    - type: max_f1
+      value: 87.01234567901234
+  - task:
+      type: Reranking
+    dataset:
+      type: mteb/stackoverflowdupquestions-reranking
+      name: MTEB StackOverflowDupQuestions
+      config: default
+      split: test
+      revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
+    metrics:
+    - type: map
+      value: 48.566931484512196
+    - type: mrr
+      value: 49.23111100500807
+  - task:
+      type: PairClassification
+    dataset:
+      type: mteb/twittersemeval2015-pairclassification
+      name: MTEB TwitterSemEval2015
+      config: default
+      split: test
+      revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
+    metrics:
+    - type: cos_sim_accuracy
+      value: 86.27287357692079
+    - type: cos_sim_ap
+      value: 74.20855854505362
+    - type: cos_sim_f1
+      value: 69.09903201787044
+    - type: cos_sim_precision
+      value: 65.22961574507966
+    - type: cos_sim_recall
+      value: 73.45646437994723
+    - type: dot_accuracy
+      value: 86.27287357692079
+    - type: dot_ap
+      value: 74.20853189774614
+    - type: dot_f1
+      value: 69.09903201787044
+    - type: dot_precision
+      value: 65.22961574507966
+    - type: dot_recall
+      value: 73.45646437994723
+    - type: euclidean_accuracy
+      value: 86.27287357692079
+    - type: euclidean_ap
+      value: 74.20857455896677
+    - type: euclidean_f1
+      value: 69.09903201787044
+    - type: euclidean_precision
+      value: 65.22961574507966
+    - type: euclidean_recall
+      value: 73.45646437994723
+    - type: manhattan_accuracy
+      value: 86.2192287059665
+    - type: manhattan_ap
+      value: 74.0513280969461
+    - type: manhattan_f1
+      value: 69.13344473621389
+    - type: manhattan_precision
+      value: 63.12118570183086
+    - type: manhattan_recall
+      value: 76.41160949868075
+    - type: max_accuracy
+      value: 86.27287357692079
+    - type: max_ap
+      value: 74.20857455896677
+    - type: max_f1
+      value: 69.13344473621389
+  - task:
+      type: PairClassification
+    dataset:
+      type: mteb/twitterurlcorpus-pairclassification
+      name: MTEB TwitterURLCorpus
+      config: default
+      split: test
+      revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
+    metrics:
+    - type: cos_sim_accuracy
+      value: 89.16055419722902
+    - type: cos_sim_ap
+      value: 86.03614264194854
+    - type: cos_sim_f1
+      value: 78.89855695205357
+    - type: cos_sim_precision
+      value: 73.74656938215409
+    - type: cos_sim_recall
+      value: 84.82445334154605
+    - type: dot_accuracy
+      value: 89.16055419722902
+    - type: dot_ap
+      value: 86.03614225282097
+    - type: dot_f1
+      value: 78.89855695205357
+    - type: dot_precision
+      value: 73.74656938215409
+    - type: dot_recall
+      value: 84.82445334154605
+    - type: euclidean_accuracy
+      value: 89.16055419722902
+    - type: euclidean_ap
+      value: 86.0361548355667
+    - type: euclidean_f1
+      value: 78.89855695205357
+    - type: euclidean_precision
+      value: 73.74656938215409
+    - type: euclidean_recall
+      value: 84.82445334154605
+    - type: manhattan_accuracy
+      value: 89.11786393448985
+    - type: manhattan_ap
+      value: 86.00799361972808
+    - type: manhattan_f1
+      value: 78.84721152788472
+    - type: manhattan_precision
+      value: 75.26776338816941
+    - type: manhattan_recall
+      value: 82.78410840776101
+    - type: max_accuracy
+      value: 89.16055419722902
+    - type: max_ap
+      value: 86.0361548355667
+    - type: max_f1
+      value: 78.89855695205357
+---
+
+# E5-large-en-ru
+
+## Model info
+
+This is vocabulary pruned version of [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large).
+
+Uses only russian and english tokens.
+
+### Size
+
+|  | intfloat/multilingual-e5-large | d0rj/e5-large-en-ru |
+| --- | --- | --- |
+| Model size (MB) | 2135.82 | 1394.8 |
+| Params (count) | 559,890,946 | 365,638,14 |
+| Word embeddings dim | 256,002,048 | 61,749,248 |
+
+### Performance
+
+Equal performance on SberQuAD dev benchmark.
+
+| Metric on SberQuAD (4122 questions) | intfloat/multilingual-e5-large | d0rj/e5-large-en-ru |
+| --- | --- | --- |
+| recall@3 | 0.787239204269772 | **0.7882096069868996** |
+| map@3 | 0.7230713245997101 | **0.723192624939351** |
+| mrr@3 | 0.7241630276564784 | **0.7243651948892132** |
+| recall@5 | 0.8277535177098496 | **0.8284813197476953** |
+| map@5 | 0.7301603186155587 | **0.7302573588872716** |
+| mrr@5 | 0.7334667637069385 | **0.7335718906679607** |
+| recall@10 | **0.8716642406598738** | 0.871421639980592 |
+| map@10 | **0.7314774917730316** | 0.7313000338687417 |
+| mrr@10 | **0.7392223685527911** | 0.7391814537556898 |
+
+## Usage
+
+- Use **dot product** distance for retrieval.
+
+- Use "query: " and "passage: " correspondingly for asymmetric tasks such as passage retrieval in open QA, ad-hoc information retrieval.
+
+- Use "query: " prefix for symmetric tasks such as semantic similarity, bitext mining, paraphrase retrieval.
+
+- Use "query: " prefix if you want to use embeddings as features, such as linear probing classification, clustering.
+
+### transformers
+
+#### Direct usage
+
+```python
+import torch.nn.functional as F
+from torch import Tensor
+from transformers import XLMRobertaTokenizer, XLMRobertaModel
+
+
+def average_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor:
+    last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
+    return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+
+
+input_texts = [
+  'query: How does a corporate website differ from a business card website?',
+  'query: Где был создан первый троллейбус?',
+  'passage: The first trolleybus was created in Germany by engineer Werner von Siemens, probably influenced by the idea of his brother, Dr. Wilhelm Siemens, who lived in England, expressed on May 18, 1881 at the twenty-second meeting of the Royal Scientific Society. The electrical circuit was carried out by an eight-wheeled cart (Kontaktwagen) rolling along two parallel contact wires. The wires were located quite close to each other, and in strong winds they often overlapped, which led to short circuits. An experimental trolleybus line with a length of 540 m (591 yards), opened by Siemens & Halske in the Berlin suburb of Halensee, operated from April 29 to June 13, 1882.',
+  'passage: Корпоративный сайт — содержит полную информацию о компании-владельце, услугах/продукции, событиях в жизни компании. Отличается от сайта-визитки и представительского сайта полнотой представленной информации, зачаст��ю содержит различные функциональные инструменты для работы с контентом (поиск и фильтры, календари событий, фотогалереи, корпоративные блоги, форумы). Может быть интегрирован с внутренними информационными системами компании-владельца (КИС, CRM, бухгалтерскими системами). Может содержать закрытые разделы для тех или иных групп пользователей — сотрудников, дилеров, контрагентов и пр.',
+]
+
+tokenizer = XLMRobertaTokenizer.from_pretrained('d0rj/e5-large-en-ru', use_cache=False)
+model = XLMRobertaModel.from_pretrained('d0rj/e5-large-en-ru', use_cache=False)
+
+batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt')
+
+outputs = model(**batch_dict)
+embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
+
+embeddings = F.normalize(embeddings, p=2, dim=1)
+scores = (embeddings[:2] @ embeddings[2:].T) * 100
+print(scores.tolist())
+# [[68.59542846679688, 81.75910949707031], [80.36100769042969, 64.77748107910156]]
+```
+
+#### Pipeline
+
+```python
+from transformers import pipeline
+
+
+pipe = pipeline('feature-extraction', model='d0rj/e5-large-en-ru')
+embeddings = pipe(input_texts, return_tensors=True)
+embeddings[0].size()
+# torch.Size([1, 17, 1024])
+```
+
+### sentence-transformers
+
+```python
+from sentence_transformers import SentenceTransformer
+
+
+sentences = [
+    'query: Что такое круглые тензоры?',
+    'passage: Abstract: we introduce a novel method for compressing round tensors based on their inherent radial symmetry. We start by generalising PCA and eigen decomposition on round tensors...',
+]
+
+model = SentenceTransformer('d0rj/e5-large-en-ru')
+embeddings = model.encode(sentences, convert_to_tensor=True)
+embeddings.size()
+# torch.Size([2, 1024])
+```

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "d0rj/e5-large-en-ru",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 60302
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.30.1",
+    "pytorch": "1.12.1"
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a7ecf8d203a2b7931d6ed58a4235464429fd8c41c1937056d257850ed1a950
+size 1462599688

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a900d8829b407aaadc83b6315504ba1acdfde420b5e2288c706a0215c6b11ddb
+size 1462678449

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 514,
+  "do_lower_case": false
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34e9d938fbab77d2bb4acd48953c66bb7ad2d0b675b4c0911e6ed25caf20acd6
+size 1270564

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}