Duplicate from kakaocorp/kanana-1.5-15.7b-a3b-instruct
Browse filesCo-authored-by: kevin.nlp <[email protected]>
- .gitattributes +36 -0
- LICENSE +73 -0
- README.md +238 -0
- assets/logo/kanana-logo.png +0 -0
- config.json +36 -0
- generation_config.json +10 -0
- kanana_tool_calls/__pycache__/functionary_kanana_tool_parser.cpython-310.pyc +0 -0
- kanana_tool_calls/__pycache__/functionary_kanana_tool_parser.cpython-312.pyc +0 -0
- kanana_tool_calls/functionary_kanana_tool_parser.py +378 -0
- kanana_tool_calls/lmalign_v1.jinja +320 -0
- model-00001-of-00007.safetensors +3 -0
- model-00002-of-00007.safetensors +3 -0
- model-00003-of-00007.safetensors +3 -0
- model-00004-of-00007.safetensors +3 -0
- model-00005-of-00007.safetensors +3 -0
- model-00006-of-00007.safetensors +3 -0
- model-00007-of-00007.safetensors +3 -0
- model.safetensors.index.json +0 -0
- special_tokens_map.json +23 -0
- tokenizer.json +3 -0
- tokenizer_config.json +2092 -0
.gitattributes
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
KANANA LICENSE AGREEMENT
|
| 2 |
+
|
| 3 |
+
Kanana Release Date: July 17, 2025
|
| 4 |
+
|
| 5 |
+
This KANANA LICENSE AGREEMENT (this “Agreement”) is made by and between you and Kakao Corp. (“KAKAO”) that governs your use of Kanana Materials that KAKAO provides to you.
|
| 6 |
+
By using, copying, modifying, distributing, performing, or displaying all or part of Kanana Materials, or otherwise accepting the terms and conditions of this Agreement, you agree to be bound by this Agreement. You hereby represent and warrant that (i) you are legally authorized to enter into this Agreement, and (ii) if you are entering into this Agreement on behalf of a legal entity, you have the authority to legally and validly bind such entity.
|
| 7 |
+
|
| 8 |
+
1. Definition
|
| 9 |
+
1.1 “Agreement” means the terms and conditions for use, copying, distribution and modification of Kanana Materials as set forth herein.
|
| 10 |
+
1.2 “KAKAO” means Kakao Corp.
|
| 11 |
+
1.3 “You” means an individual or legal entity that enters into this Agreement with KAKAO and exercises its rights hereunder or uses Kanana Materials for any purpose. If you enter into this Agreement on behalf of a legal entity, “you” shall include such entity.
|
| 12 |
+
1.4 “Kanana” means the basic large-scale language model, software, and algorithms distributed by KAKAO under this Agreement, including parameters (such as Model Weights and optimizer status), machine learning model codes, inference/learning/fine-tuning codes, and other related elements.
|
| 13 |
+
1.5 “Documentation” means the specifications, manuals, and other documentation accompanying Kanana distributed by KAKAO.
|
| 14 |
+
1.6 “Kanana Materials” means, collectively, Kanana and Documentation, including any portions or components thereof.
|
| 15 |
+
1.7 “Outputs” means information content generated by operating or otherwise using Kanana Materials.
|
| 16 |
+
1.8 “Derivative Works” means (i) any modifications to Kanana, (ii) any work of authorship based on Kanana, or (iii) any other designed machine learning models that either directly use the patterns of Model Weights, parameters, operations, and/or outputs or incorporate a substantial part of Kanana’s performance or functional characteristics through methods including, but not limited to, transfer learning, fine-tuning, or knowledge distillation. This includes distillation methods using Kanana’s intermediate data representations or a method based on the synthetic data outputs generated by Kanana; provided, however, that Outputs shall not be deemed to be Derivative Works.
|
| 17 |
+
1.9 “Model Weights” means a set of numerical parameter values generated during Kanana’s learning process, representing the result of substantial investment and effort by KAKAO.
|
| 18 |
+
|
| 19 |
+
2. Grant of License and Use Policy
|
| 20 |
+
2.1 Grant of License. Subject to the terms and conditions of this Agreement, you are granted a non-exclusive, worldwide, non-transferrable, royalty-free limited license under KAKAO’s intellectual property or other rights owned by KAKAO that enables you to access, download, install, copy, use, reproduce, distribute, create Derivative Works of, and make modifications to Kanana Materials.
|
| 21 |
+
2.2 Policy on Prohibited Use. Your use of Kanana Materials and Derivative Works must comply with applicable laws and regulations and adhere to KAKAO’s Guidelines For Responsible AI (https://www.kakaocorp.com/page/responsible/detail/guidelinesForResponsibleAI), which is hereby incorporated into this Agreement.
|
| 22 |
+
2.3 This Agreement applies solely to Kanana-*** and shall not apply to any other models distributed by KAKAO under separate licenses. Licenses applicable to such other models shall not apply to Kanana-***.
|
| 23 |
+
2.4 The license terms applicable to a specific version of Kanana applies exclusively to that version and shall not extend to any other versions. Each version shall be deemed as an independent and separate work of authorship.
|
| 24 |
+
2.5 You may use each version of Kanana only in accordance with the license terms expressly specified for that version, and you shall not claim that the license terms applicable to one version apply to any other version.
|
| 25 |
+
2.6 You shall not combine different versions of Kanana versions that are subject to different license terms in order to circumvent any applicable license terms.
|
| 26 |
+
|
| 27 |
+
3. Redistribution
|
| 28 |
+
3.1 You may copy, distribute or disclose Kanana, Derivative Works, or any products or services that contain Kanana or Derivative Works; provided, however, that you shall:
|
| 29 |
+
(i) incorporate the compliance obligation set forth in the Policy on Prohibited Use provision of Section 2.2 in any agreement for use and distribution and notify subsequent users that such use restrictions apply;
|
| 30 |
+
(ii) provide any recipients of Kanana Materials or Derivative Works a copy of this Agreement;
|
| 31 |
+
(iii) expressly indicate in any files you have modified that it has been modified by you;
|
| 32 |
+
(iv) include a “Notice” text file that includes the following notice:
|
| 33 |
+
“Kanana is licensed in accordance with the Kanana License Agreement. Copyright © KAKAO Corp. All Rights Reserved.”; and
|
| 34 |
+
(v) clearly display the phrase “Powered by Kanana” on related websites, user interfaces, blog posts, introduction pages, or product documentation in a manner that is easily recognizable to users. In addition, if you use Kanana Materials or their outputs to create, train, improve, or enhance other AI models and distribute them, you must include ‘Kanana’ as a prefix to the name of such AI models.
|
| 35 |
+
3.2 You may add your own copyright statement to your modifications of Kanana Materials and may provide additional or different license terms and conditions; provided, however, that such additional or different license terms and conditions shall not violate or conflict with any provisions of this Agreement.
|
| 36 |
+
|
| 37 |
+
4. Additional Commercial Terms
|
| 38 |
+
4.1 If you wish to engage in any of the following activities using Kanana Materials or any Derivative Works, you must obtain a separate commercial license expressly granted by KAKAO:
|
| 39 |
+
(i) Offering or (re)selling to third parties access to Kanana Materials or any Derivative Works through API, cloud platforms, or other remote access services;
|
| 40 |
+
(ii) Offering or (re)selling to third parties Kanana Materials or any Derivative Works in whole or in part, as part of a system integration (SI) or on-premise deployment solution; or
|
| 41 |
+
(iii) Offering or (re)selling to third parties Kanana Materials or any Derivative Works embedded in an on-device domains.
|
| 42 |
+
4.2 If, as of Kanana Release Date, the number of monthly active users of the products or services provided by you and/or your affiliates, is greater than 10 million in the preceding calendar month, you must obtain a separate commercial license expressly granted by KAKAO.
|
| 43 |
+
4.3 For clarity, unless your activities or conditions fall within those specified in Sections 4.1 and 4.2 above, you may use Kanana Materials or any Derivative Works for the development and operation of your own services without obtaining a commercial license from KAKAO.
|
| 44 |
+
4.4 The grant of any commercial license under Sections 4.1 and 4.2 shall be at KAKAO’s sole discretion
|
| 45 |
+
|
| 46 |
+
5. Outputs
|
| 47 |
+
KAKAO will not claim any rights to Outputs you generate using Kanana Materials. You shall be solely responsible for Outputs and the use thereof.
|
| 48 |
+
|
| 49 |
+
6. Disclaimer of Warranty
|
| 50 |
+
Unless required by law, Kanana Materials are provided on an “AS IS” basis, and KAKAO disclaims all warranties of any kind, both express and implied, including, without limitation, any warranties of title, non-infringement, merchantability, or fitness for a particular purpose.
|
| 51 |
+
|
| 52 |
+
7. Limitation on Liability
|
| 53 |
+
Unless required by law, in no event shall KAKAO be liable to you for damages, including any direct, indirect, special, consequential, incidental, and punitive damages of any character arising out of the use or inability to use Kanana Materials, Derivative Works, or Outputs, even if KAKAO has been advised of the possibility of such damages.
|
| 54 |
+
|
| 55 |
+
8. Indemnification
|
| 56 |
+
You shall indemnify and hold KAKAO harmless from and against any and all claims that may be filed by a third party as a result of your infringement of any third party’s rights or violation of any applicable law, to the extent caused by your use or distribution of Kanana Materials, Derivative Works, or Outputs; provided, however, that the foregoing shall not apply to claims resulting from KAKAO’s willful or gross negligence.
|
| 57 |
+
|
| 58 |
+
9. Intellectual Property
|
| 59 |
+
9.1 This Agreement does not grant you any rights to use KAKAO’s trademarks, service marks, or product names. However, on a limited basis and solely for the purpose of complying with Section 3.1(v), KAKAO authorizes you to use the Kanana trademark, provided that KAKAO may require you to discontinue such use at any time if you impair the value of the Kanana trademark.
|
| 60 |
+
9.2 KAKAO retains ownership of Kanana Materials and Derivative Works created by KAKAO, but you will retain ownership of any Derivative Works and modifications made by you.
|
| 61 |
+
9.3 If you bring any legal action or proceeding against KAKAO or a third party alleging that the Kanana Materials, Derivative Works, or Outputs infringe your intellectual property rights, your rights under this Agreement shall automatically terminate as of the date such action is filed.
|
| 62 |
+
9.4 You acknowledge that Model Weights are a valuable asset of KAKAO. You shall not extract, copy, distribute, modify Model Weights or use them to train new models, except as expressly permitted under this Agreement.
|
| 63 |
+
9.5 The protections under this Agreement apply to all components of Kanana Materials (irrespective of whether it is recognized as a work of authorship), including, but not limited to, Model Weights, parameters, algorithms, or structures. You may exercise your rights in these components only to the extent expressly permitted under this Agreement.
|
| 64 |
+
|
| 65 |
+
10. Term and Termination
|
| 66 |
+
The term of this Agreement will commence upon your acceptance of this Agreement or access to Kanana Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. KAKAO may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of Kanana Materials and Derivative Works. Sections 5, 6, 7, 8, 10 and 11 shall survive the termination of this Agreement.
|
| 67 |
+
|
| 68 |
+
11. Governing Law and Arbitration
|
| 69 |
+
11.1 This Agreement will be governed and construed under the laws of the Republic of Korea, without regard to its conflicts of laws principles.
|
| 70 |
+
11.2 Any disputes arising out of or in connection with this Agreement shall be finally settled by arbitration in accordance with the International Arbitration Rules of the Korean Commercial Arbitration Board. The number of arbitrators shall be one. The seat, or legal place, of arbitral proceedings shall be Seoul, Republic of Korea. The language to be used in the arbitral proceedings shall be English. Either party may seek interim or provisional relief from a court of competent jurisdiction, which shall not be considered a waiver of any provision in this Section. The arbitral tribunal also has the authority to issue orders for interim or provisional relief.
|
| 71 |
+
|
| 72 |
+
12. No Waiver
|
| 73 |
+
KAKAO’s failure or delay in exercising any of its rights under this Agreement shall not constitute a waiver of such rights.
|
README.md
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
+
- ko
|
| 5 |
+
library_name: transformers
|
| 6 |
+
license: other
|
| 7 |
+
license_name: "kanana"
|
| 8 |
+
license_link: LICENSE
|
| 9 |
+
pipeline_tag: text-generation
|
| 10 |
+
model_id: kakaocorp/kanana-1.5-15.7b-a3b-instruct
|
| 11 |
+
repo: kakaocorp/kanana-1.5-15.7b-a3b-instruct
|
| 12 |
+
developers: Kanana LLM
|
| 13 |
+
training_regime: bf16 mixed precision
|
| 14 |
+
---
|
| 15 |
+
|
| 16 |
+
<p align="center">
|
| 17 |
+
<br>
|
| 18 |
+
<picture>
|
| 19 |
+
<img src="./assets/logo/kanana-logo.png" width="60%" style="margin: 40px auto;">
|
| 20 |
+
</picture>
|
| 21 |
+
</br>
|
| 22 |
+
|
| 23 |
+
<p align="center">
|
| 24 |
+
🤗 <a href="https://kko.kakao.com/kananallm">1.5 HF Models</a>   |
|
| 25 |
+
  📕 <a href="https://tech.kakao.com/posts/716">Kanana-1.5-15.7B-A3B Blog</a>  
|
| 26 |
+
|
| 27 |
+
<br>
|
| 28 |
+
|
| 29 |
+
## News 🔥
|
| 30 |
+
|
| 31 |
+
- ✨`2025/07/24`: Published a [blog post](https://tech.kakao.com/posts/716) about `Kanana-1.5-15.7B-A3B` models and released 🤗[HF model weights](https://kko.kakao.com/kananallm).
|
| 32 |
+
- 📕`2025/05/23`: Published a [blog post](https://tech.kakao.com/posts/707) about `Kanana 1.5` models and released 🤗[HF model weights](https://kko.kakao.com/kananallm).
|
| 33 |
+
- 📜`2025/02/27`: Released [Technical Report](https://arxiv.org/abs/2502.18934) and 🤗[HF model weights](https://huggingface.co/collections/kakaocorp/kanana-nano-21b-67a326cda1c449c8d4172259).
|
| 34 |
+
- 📕`2025/01/10`: Published a [blog post](https://tech.kakao.com/posts/682) about the development of `Kanana Nano` model.
|
| 35 |
+
- 📕`2024/11/14`: Published blog posts ([pre-training](https://tech.kakao.com/posts/661), [post-training](https://tech.kakao.com/posts/662)) about the development of `Kanana` models.
|
| 36 |
+
- ▶️`2024/11/06`: Published a [presentation video](https://youtu.be/HTBl142x9GI?si=o_we6t9suYK8DfX3) about the development of the `Kanana` models.
|
| 37 |
+
|
| 38 |
+
<br>
|
| 39 |
+
|
| 40 |
+
## Table of Contents
|
| 41 |
+
|
| 42 |
+
- [Kanana-1.5-15.7B-A3B](#kanana-15-157b-a3b)
|
| 43 |
+
- [Performance](#performance)
|
| 44 |
+
- [Base Model Evaluation](#base-model-evaluation)
|
| 45 |
+
- [Instruct Model Evaluation](#instruct-model-evaluation)
|
| 46 |
+
- [Contributors](#contributors)
|
| 47 |
+
- [Citation](#citation)
|
| 48 |
+
- [Contact](#contact)
|
| 49 |
+
|
| 50 |
+
<br>
|
| 51 |
+
|
| 52 |
+
# Kanana-1.5-15.7B-A3B
|
| 53 |
+
|
| 54 |
+
Introducing `Kanana-1.5-15.7B-A3B`, the first Mixture-of-Experts (MoE) model in our Kanana family, engineered for exceptional efficiency and powerful performance. `Kanana-1.5-15.7B-A3B`, which has sparse architecture, delivers capabilities comparable to the `Kanana-1.5-8B` dense model while utilizing only 37% of the FLOPS per token, making it a highly inference-efficient and cost-effective solution for real-world applications. Furthermore, `Kanana-1.5-15.7B-A3B` is powered by our newly enhanced post-training strategy, which includes on-policy distillation followed by reinforcement learning.
|
| 55 |
+
|
| 56 |
+
> [!Note]
|
| 57 |
+
> Neither the pre-training nor the post-training data includes Kakao user data.
|
| 58 |
+
|
| 59 |
+
## Performance
|
| 60 |
+
|
| 61 |
+
### Base Model Evaluation
|
| 62 |
+
<table>
|
| 63 |
+
<tr>
|
| 64 |
+
<th>Models</th>
|
| 65 |
+
<th>MMLU</th>
|
| 66 |
+
<th>KMMLU</th>
|
| 67 |
+
<th>HAERAE</th>
|
| 68 |
+
<th>HumanEval</th>
|
| 69 |
+
<th>MBPP</th>
|
| 70 |
+
<th>GSM8K</th>
|
| 71 |
+
</tr>
|
| 72 |
+
<tr>
|
| 73 |
+
<td><strong>Kanana-1.5-15.7B-A3B</strong></td>
|
| 74 |
+
<td align="center">64.79</td>
|
| 75 |
+
<td align="center">51.77</td>
|
| 76 |
+
<td align="center">83.23</td>
|
| 77 |
+
<td align="center">59.76</td>
|
| 78 |
+
<td align="center">60.10</td>
|
| 79 |
+
<td align="center">61.18</td>
|
| 80 |
+
</tr>
|
| 81 |
+
<tr>
|
| 82 |
+
<td>Kanana-1.5-8B</td>
|
| 83 |
+
<td align="center">64.24</td>
|
| 84 |
+
<td align="center">48.94</td>
|
| 85 |
+
<td align="center">82.77</td>
|
| 86 |
+
<td align="center">61.59</td>
|
| 87 |
+
<td align="center">57.80</td>
|
| 88 |
+
<td align="center">63.53</td>
|
| 89 |
+
</tr>
|
| 90 |
+
<tr>
|
| 91 |
+
<td>Kanana-1.5-3B*</td>
|
| 92 |
+
<td align="center">59.23</td>
|
| 93 |
+
<td align="center">47.30</td>
|
| 94 |
+
<td align="center">78.00</td>
|
| 95 |
+
<td align="center">46.34</td>
|
| 96 |
+
<td align="center">46.80</td>
|
| 97 |
+
<td align="center">61.79</td>
|
| 98 |
+
</tr>
|
| 99 |
+
</table>
|
| 100 |
+
|
| 101 |
+
<br>
|
| 102 |
+
|
| 103 |
+
### Instruct Model Evaluation
|
| 104 |
+
<table>
|
| 105 |
+
<tr>
|
| 106 |
+
<th>Models</th>
|
| 107 |
+
<th>MT-Bench</th>
|
| 108 |
+
<th>KoMT-Bench</th>
|
| 109 |
+
<th>IFEval</th>
|
| 110 |
+
<th>HumanEval+</th>
|
| 111 |
+
<th>MBPP+</th>
|
| 112 |
+
<th>GSM8K (0-shot)</th>
|
| 113 |
+
<th>MATH</th>
|
| 114 |
+
<th>MMLU (0-shot, CoT)</th>
|
| 115 |
+
<th>KMMLU (0-shot, CoT)</th>
|
| 116 |
+
</tr>
|
| 117 |
+
<tr>
|
| 118 |
+
<td><strong>Kanana-1.5-15.7B-A3B</strong></td>
|
| 119 |
+
<td align="center">7.67</td>
|
| 120 |
+
<td align="center">7.24</td>
|
| 121 |
+
<td align="center">73.35</td>
|
| 122 |
+
<td align="center">79.27</td>
|
| 123 |
+
<td align="center">70.37</td>
|
| 124 |
+
<td align="center">83.02</td>
|
| 125 |
+
<td align="center">66.42</td>
|
| 126 |
+
<td align="center">68.55</td>
|
| 127 |
+
<td align="center">48.92</td>
|
| 128 |
+
</tr>
|
| 129 |
+
<tr>
|
| 130 |
+
<td>Kanana-1.5-8B</td>
|
| 131 |
+
<td align="center">7.76</td>
|
| 132 |
+
<td align="center">7.63</td>
|
| 133 |
+
<td align="center">80.11</td>
|
| 134 |
+
<td align="center">76.83</td>
|
| 135 |
+
<td align="center">67.99</td>
|
| 136 |
+
<td align="center">87.64</td>
|
| 137 |
+
<td align="center">67.54</td>
|
| 138 |
+
<td align="center">68.82</td>
|
| 139 |
+
<td align="center">48.28</td>
|
| 140 |
+
</tr>
|
| 141 |
+
<tr>
|
| 142 |
+
<td>Kanana-1.5-3B*</td>
|
| 143 |
+
<td align="center">7.01</td>
|
| 144 |
+
<td align="center">6.52</td>
|
| 145 |
+
<td align="center">70.08</td>
|
| 146 |
+
<td align="center">70.73</td>
|
| 147 |
+
<td align="center">64.29</td>
|
| 148 |
+
<td align="center">80.36</td>
|
| 149 |
+
<td align="center">56.70</td>
|
| 150 |
+
<td align="center">59.69</td>
|
| 151 |
+
<td align="center">37.60</td>
|
| 152 |
+
</tr>
|
| 153 |
+
</table>
|
| 154 |
+
|
| 155 |
+
> [!Note]
|
| 156 |
+
> \* This model is not an open-sourced, just for comparison with Kanana-1.5-15.7B-A3B
|
| 157 |
+
|
| 158 |
+
<br>
|
| 159 |
+
|
| 160 |
+
### Evaluation Protocol
|
| 161 |
+
- Base Model Benchmarks
|
| 162 |
+
- MMLU, KMMLU, HAE-RAE: 5-shot, log-likelihood
|
| 163 |
+
- HumanEval: 0-shot, pass@1
|
| 164 |
+
- MBPP: 3-shot, pass@1
|
| 165 |
+
- GSM8K: 5-shot, exact-match (strict-match)
|
| 166 |
+
|
| 167 |
+
- Instruct Model Benchmarks
|
| 168 |
+
- MT-Bench, KoMT-Bench: 0-shot, gpt-4o-2024-08-06 as judge model
|
| 169 |
+
- IFEval: 0-shot, mean of strict-prompt-level and strict-instruction-level
|
| 170 |
+
- HumanEval+, MBPP+: 0-shot, pass@1
|
| 171 |
+
- GSM8K, MATH: 0-shot, rule-based verification
|
| 172 |
+
|
| 173 |
+
<br>
|
| 174 |
+
|
| 175 |
+
## Quickstart
|
| 176 |
+
|
| 177 |
+
### vLLM
|
| 178 |
+
- `vllm>=0.8.5` or the latest version is required to run `Kanana` model.
|
| 179 |
+
|
| 180 |
+
#### Example Usage for `Kanana-1.5-15.7B-A3B-Instruct`
|
| 181 |
+
```bash
|
| 182 |
+
vllm serve $path_to_model \
|
| 183 |
+
--served_model_name kanana-1.5-15.7b-a3b-instruct \
|
| 184 |
+
--max-model-len 32768 \
|
| 185 |
+
--gpu-memory-utilization 0.9 \
|
| 186 |
+
--port 8000 \
|
| 187 |
+
--dtype auto \
|
| 188 |
+
--disable_cascade_attn \
|
| 189 |
+
--tool-parser-plugin kanana_tool_calls/functionary_kanana_tool_parser.py \
|
| 190 |
+
--tool-call-parser functionary_v3_llama_31 \
|
| 191 |
+
--enable-auto-tool-choice \
|
| 192 |
+
--chat-template kanana_tool_calls/lmalign_v1.jinja
|
| 193 |
+
|
| 194 |
+
curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{
|
| 195 |
+
"model": "kanana-1.5-15.7b-a3b-instruct",
|
| 196 |
+
"messages": [
|
| 197 |
+
{"role": "system", "content": "You are a helpful AI assistant developed by Kakao."},
|
| 198 |
+
{"role": "user", "content": "Explain LLM to a 5-year-old child in two simple sentences."}
|
| 199 |
+
],
|
| 200 |
+
"top_k": 1,
|
| 201 |
+
"max_tokens": 72
|
| 202 |
+
}'
|
| 203 |
+
|
| 204 |
+
# Output:
|
| 205 |
+
'''
|
| 206 |
+
...
|
| 207 |
+
"choices":[{"index":0,"message":{"role":"assistant","content":"Sure! Imagine you have a super smart friend who can read and understand lots of books, talk to you in many different languages, and even help you with your homework. That is kind of like a Large Language Model (LLM) - it is a special computer friend that can do many cool things with words!" ...
|
| 208 |
+
...
|
| 209 |
+
'''
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
<br>
|
| 213 |
+
|
| 214 |
+
## Contributors
|
| 215 |
+
- Language Model Training
|
| 216 |
+
- Yunju Bak, Doohae Jung, Boseop Kim, Nayeon Kim, Hojin Lee, Jaesun Park, Minho Ryu, Jiyeon Ham, Seungjae Jung, Hyunho Kim, Hyunwoong Ko, Changmin Lee, Taegyeong Eo
|
| 217 |
+
|
| 218 |
+
<br>
|
| 219 |
+
|
| 220 |
+
## Citation
|
| 221 |
+
|
| 222 |
+
```
|
| 223 |
+
@misc{kananallmteam2025kananacomputeefficientbilinguallanguage,
|
| 224 |
+
title={Kanana: Compute-efficient Bilingual Language Models},
|
| 225 |
+
author={Kanana LLM Team and Yunju Bak and Hojin Lee and Minho Ryu and Jiyeon Ham and Seungjae Jung and Daniel Wontae Nam and Taegyeong Eo and Donghun Lee and Doohae Jung and Boseop Kim and Nayeon Kim and Jaesun Park and Hyunho Kim and Hyunwoong Ko and Changmin Lee and Kyoung-Woon On and Seulye Baeg and Junrae Cho and Sunghee Jung and Jieun Kang and EungGyun Kim and Eunhwa Kim and Byeongil Ko and Daniel Lee and Minchul Lee and Miok Lee and Shinbok Lee and Gaeun Seo},
|
| 226 |
+
year={2025},
|
| 227 |
+
eprint={2502.18934},
|
| 228 |
+
archivePrefix={arXiv},
|
| 229 |
+
primaryClass={cs.CL},
|
| 230 |
+
url={https://arxiv.org/abs/2502.18934},
|
| 231 |
+
}
|
| 232 |
+
```
|
| 233 |
+
|
| 234 |
+
<br>
|
| 235 |
+
|
| 236 |
+
## Contact
|
| 237 |
+
- Kanana LLM Team Technical Support: [email protected]
|
| 238 |
+
- Business & Partnership Contact: [email protected]
|
assets/logo/kanana-logo.png
ADDED
|
config.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"MixtralForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_dropout": 0.0,
|
| 6 |
+
"bos_token_id": 128000,
|
| 7 |
+
"eos_token_id": [
|
| 8 |
+
128009,
|
| 9 |
+
128256
|
| 10 |
+
],
|
| 11 |
+
"head_dim": 128,
|
| 12 |
+
"hidden_act": "silu",
|
| 13 |
+
"hidden_size": 2048,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 1152,
|
| 16 |
+
"max_position_embeddings": 32768,
|
| 17 |
+
"model_type": "mixtral",
|
| 18 |
+
"num_attention_heads": 32,
|
| 19 |
+
"num_experts_per_tok": 8,
|
| 20 |
+
"num_hidden_layers": 32,
|
| 21 |
+
"num_key_value_heads": 8,
|
| 22 |
+
"num_local_experts": 64,
|
| 23 |
+
"output_router_logits": false,
|
| 24 |
+
"pad_token_id": 128001,
|
| 25 |
+
"rms_norm_eps": 1e-05,
|
| 26 |
+
"rope_theta": 16000000.0,
|
| 27 |
+
"router_aux_loss_coef": 0.005,
|
| 28 |
+
"router_jitter_noise": 0.0,
|
| 29 |
+
"scoring_func": "softmax",
|
| 30 |
+
"sliding_window": null,
|
| 31 |
+
"tie_word_embeddings": false,
|
| 32 |
+
"torch_dtype": "bfloat16",
|
| 33 |
+
"transformers_version": "4.51.1",
|
| 34 |
+
"use_cache": true,
|
| 35 |
+
"vocab_size": 128259
|
| 36 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 128000,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
128009,
|
| 6 |
+
128256
|
| 7 |
+
],
|
| 8 |
+
"pad_token_id": 128001,
|
| 9 |
+
"transformers_version": "4.51.1"
|
| 10 |
+
}
|
kanana_tool_calls/__pycache__/functionary_kanana_tool_parser.cpython-310.pyc
ADDED
|
Binary file (8.11 kB). View file
|
|
|
kanana_tool_calls/__pycache__/functionary_kanana_tool_parser.cpython-312.pyc
ADDED
|
Binary file (13.8 kB). View file
|
|
|
kanana_tool_calls/functionary_kanana_tool_parser.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from json import JSONDecodeError, JSONDecoder
|
| 3 |
+
import copy
|
| 4 |
+
import uuid
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Dict, List, Optional, Union, Sequence
|
| 7 |
+
from abc import ABC, abstractmethod
|
| 8 |
+
|
| 9 |
+
import partial_json_parser
|
| 10 |
+
from partial_json_parser.core.options import Allow
|
| 11 |
+
|
| 12 |
+
from transformers import PreTrainedTokenizerBase
|
| 13 |
+
from vllm.entrypoints.openai.protocol import (
|
| 14 |
+
ChatCompletionRequest,
|
| 15 |
+
ExtractedToolCallInformation,
|
| 16 |
+
DeltaMessage,
|
| 17 |
+
DeltaFunctionCall,
|
| 18 |
+
DeltaToolCall
|
| 19 |
+
)
|
| 20 |
+
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ToolParser, ToolParserManager
|
| 21 |
+
from vllm.entrypoints.openai.tool_parsers.utils import find_common_prefix
|
| 22 |
+
from vllm.logger import init_logger
|
| 23 |
+
from vllm.utils import random_uuid
|
| 24 |
+
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
from transformers import PreTrainedTokenizer
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
logger = init_logger(__name__)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class BaseTemplate(ABC):
|
| 36 |
+
@abstractmethod
|
| 37 |
+
def response_to_messages(self, generated_text):
|
| 38 |
+
raise NotImplementedError
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class FunctionaryV3Llama31Template(BaseTemplate):
|
| 42 |
+
def __init__(
|
| 43 |
+
self,
|
| 44 |
+
tokenizer: PreTrainedTokenizer
|
| 45 |
+
):
|
| 46 |
+
self.tokenizer = tokenizer
|
| 47 |
+
|
| 48 |
+
self.system_tokens = "<|start_header_id|>system<|end_header_id|>\n\n"
|
| 49 |
+
self.user_tokens = "<|start_header_id|>user<|end_header_id|>\n\n"
|
| 50 |
+
self.assistant_tokens = "<|start_header_id|>assistant<|end_header_id|>\n\n"
|
| 51 |
+
self.tool_tokens = "<|start_header_id|>ipython<|end_header_id|>\n\n"
|
| 52 |
+
|
| 53 |
+
def parse_function_call_from_text(self, function_call_text: str) -> Optional[Dict]:
|
| 54 |
+
index = function_call_text.find(">")
|
| 55 |
+
if index >= 0:
|
| 56 |
+
func_name = function_call_text[:index].strip()
|
| 57 |
+
arguments = function_call_text[index + 1:].strip()
|
| 58 |
+
return {"name": func_name, "arguments": arguments}
|
| 59 |
+
return None
|
| 60 |
+
|
| 61 |
+
def response_to_messages(self, generated_text):
|
| 62 |
+
# first remove stop tokens if there exists
|
| 63 |
+
for stop in [self.tokenizer.eos_token, "<|eom_id|>", "<|end_of_text|>"]:
|
| 64 |
+
if generated_text.endswith(stop):
|
| 65 |
+
generated_text = generated_text[: -len(stop)]
|
| 66 |
+
|
| 67 |
+
tool_calls = []
|
| 68 |
+
text_response = ""
|
| 69 |
+
|
| 70 |
+
func_prefix = "<function="
|
| 71 |
+
end_func = "</function>"
|
| 72 |
+
python_tag = "<|python_tag|>"
|
| 73 |
+
|
| 74 |
+
while len(generated_text) > 0:
|
| 75 |
+
if generated_text.startswith(python_tag): # check if use code interpreter
|
| 76 |
+
code = generated_text[len(python_tag) :]
|
| 77 |
+
function_call = {
|
| 78 |
+
"name": "python",
|
| 79 |
+
"arguments": code,
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
tool_calls.append(
|
| 83 |
+
{
|
| 84 |
+
"type": "function",
|
| 85 |
+
"id": str(uuid.uuid4()),
|
| 86 |
+
"function": function_call,
|
| 87 |
+
}
|
| 88 |
+
)
|
| 89 |
+
generated_text = ""
|
| 90 |
+
elif generated_text.startswith(func_prefix): # Check if function_call
|
| 91 |
+
end_index = generated_text.find(end_func)
|
| 92 |
+
if end_index >= 0:
|
| 93 |
+
function_call_text = generated_text[len(func_prefix) : end_index]
|
| 94 |
+
function_call = self.parse_function_call_from_text(function_call_text)
|
| 95 |
+
|
| 96 |
+
tool_calls.append(
|
| 97 |
+
{
|
| 98 |
+
"type": "function",
|
| 99 |
+
"id": str(uuid.uuid4()),
|
| 100 |
+
"function": function_call,
|
| 101 |
+
}
|
| 102 |
+
)
|
| 103 |
+
generated_text = generated_text[end_index + len(end_func) :]
|
| 104 |
+
else:
|
| 105 |
+
# TODO cannot find close function call
|
| 106 |
+
text_response += generated_text
|
| 107 |
+
break
|
| 108 |
+
else: # If text-response
|
| 109 |
+
text_response += generated_text[0]
|
| 110 |
+
generated_text = generated_text[1:]
|
| 111 |
+
|
| 112 |
+
if not text_response:
|
| 113 |
+
text_response = None
|
| 114 |
+
elif len(text_response.strip()) == 0:
|
| 115 |
+
text_response = None
|
| 116 |
+
|
| 117 |
+
if not tool_calls:
|
| 118 |
+
tool_calls = None
|
| 119 |
+
|
| 120 |
+
if tool_calls:
|
| 121 |
+
return {"role": "assistant", "content": text_response, "tool_calls": tool_calls}
|
| 122 |
+
else:
|
| 123 |
+
return {"role": "assistant", "content": text_response}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# partial_json_parser doesn't support extra data and
|
| 128 |
+
# JSONDecorder.raw_decode doesn't support partial JSON
|
| 129 |
+
def partial_json_loads(input_str, flags):
|
| 130 |
+
try:
|
| 131 |
+
return partial_json_parser.loads(input_str, flags), len(input_str)
|
| 132 |
+
except JSONDecodeError as e:
|
| 133 |
+
if "Extra data" in e.msg:
|
| 134 |
+
dec = JSONDecoder()
|
| 135 |
+
return dec.raw_decode(input_str)
|
| 136 |
+
else:
|
| 137 |
+
raise
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def is_complete_json(input_str):
|
| 141 |
+
try:
|
| 142 |
+
json.loads(input_str)
|
| 143 |
+
return True
|
| 144 |
+
except JSONDecodeError:
|
| 145 |
+
return False
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
@ToolParserManager.register_module(["functionary_v3_llama_31"])
|
| 149 |
+
class FunctionaryV3Llama31ToolParser(ToolParser):
|
| 150 |
+
def __init__(self, tokenizer: Union[PreTrainedTokenizerBase, AnyTokenizer]):
|
| 151 |
+
super().__init__(tokenizer)
|
| 152 |
+
self.template = FunctionaryV3Llama31Template(tokenizer=tokenizer)
|
| 153 |
+
|
| 154 |
+
self._func_prefix = "<function="
|
| 155 |
+
self._func_suffix = "</function>"
|
| 156 |
+
self._python_tag = "<|python_tag|>"
|
| 157 |
+
self.current_tool_name_sent: bool = False
|
| 158 |
+
self.current_tool_id: int = -1
|
| 159 |
+
self.streamed_args_for_tool: List[str] = [
|
| 160 |
+
] # map what has been streamed for each tool so far to a list
|
| 161 |
+
self.eos_token = tokenizer.eos_token
|
| 162 |
+
self._python_tag_id = tokenizer.encode(self._python_tag,
|
| 163 |
+
add_special_tokens=False)[0]
|
| 164 |
+
|
| 165 |
+
def extract_tool_calls(
|
| 166 |
+
self, model_output: str,
|
| 167 |
+
request: ChatCompletionRequest) -> ExtractedToolCallInformation:
|
| 168 |
+
result = self.template.response_to_messages(model_output)
|
| 169 |
+
if "tool_calls" in result:
|
| 170 |
+
return ExtractedToolCallInformation(tools_called=True,
|
| 171 |
+
tool_calls=result["tool_calls"],
|
| 172 |
+
content=None)
|
| 173 |
+
else:
|
| 174 |
+
return ExtractedToolCallInformation(tools_called=False,
|
| 175 |
+
tool_calls=[],
|
| 176 |
+
content=result["content"])
|
| 177 |
+
|
| 178 |
+
# our template: <function=function_name>{"arg":"var"}<function>
|
| 179 |
+
def extract_tool_calls_streaming(
|
| 180 |
+
self,
|
| 181 |
+
previous_text: str,
|
| 182 |
+
current_text: str,
|
| 183 |
+
delta_text: str,
|
| 184 |
+
previous_token_ids: Sequence[int],
|
| 185 |
+
current_token_ids: Sequence[int],
|
| 186 |
+
delta_token_ids: Sequence[int],
|
| 187 |
+
request: ChatCompletionRequest,
|
| 188 |
+
) -> Union[DeltaMessage, None]:
|
| 189 |
+
# if current_text does not start with function tag (or python tag),
|
| 190 |
+
# stream right away as delta.content
|
| 191 |
+
if not (current_text.startswith(self._python_tag)
|
| 192 |
+
or current_text.startswith(self._func_prefix)
|
| 193 |
+
or self._func_prefix.startswith(current_text)):
|
| 194 |
+
return DeltaMessage(content=delta_text)
|
| 195 |
+
|
| 196 |
+
# if current_text ends with stop token,
|
| 197 |
+
# remove it from the text
|
| 198 |
+
# CHECK: sometimes text is generated beyond <|eom_id|>
|
| 199 |
+
for stop in [self.eos_token, "<|eom_id|>", "<|end_of_text|>"]:
|
| 200 |
+
if current_text.rstrip().endswith(stop):
|
| 201 |
+
current_text = current_text.rstrip()[: -len(stop)]
|
| 202 |
+
|
| 203 |
+
# if current_tool_name is not sent yet,
|
| 204 |
+
# don't allow partial sending of strings
|
| 205 |
+
# (supposedly, openai also only sends the entire tool name at once)
|
| 206 |
+
# not really relevant to kanana
|
| 207 |
+
flags = Allow.ALL if self.current_tool_name_sent \
|
| 208 |
+
else Allow.ALL & ~Allow.STR
|
| 209 |
+
|
| 210 |
+
try:
|
| 211 |
+
tool_call_arr = []
|
| 212 |
+
is_complete = []
|
| 213 |
+
obj = {}
|
| 214 |
+
try:
|
| 215 |
+
# set start_idx
|
| 216 |
+
start_idx = 0
|
| 217 |
+
while start_idx < len(current_text):
|
| 218 |
+
name = None
|
| 219 |
+
# set function_name and move start_idx to the end of function_name
|
| 220 |
+
if current_text[start_idx:].startswith(self._python_tag):
|
| 221 |
+
name = "python"
|
| 222 |
+
start_idx += len(self._python_tag)
|
| 223 |
+
elif current_text[start_idx:].startswith(self._func_prefix):
|
| 224 |
+
idx = current_text[start_idx:].find(">")
|
| 225 |
+
# found ">" which means function name is ready
|
| 226 |
+
if idx != -1:
|
| 227 |
+
function_name = current_text[start_idx+len(self._func_prefix):start_idx+idx]
|
| 228 |
+
name = function_name
|
| 229 |
+
start_idx += len(self._func_prefix) + len(function_name) + len(">")
|
| 230 |
+
# for cases like: </function>abcd<function=
|
| 231 |
+
# ignore tokens in between </function> and <function=
|
| 232 |
+
# and move start_idx to the start of the new function
|
| 233 |
+
else:
|
| 234 |
+
idx = current_text[start_idx:].find(self._func_prefix)
|
| 235 |
+
if idx != -1:
|
| 236 |
+
start_idx += idx
|
| 237 |
+
continue
|
| 238 |
+
|
| 239 |
+
# partially load the function arguments
|
| 240 |
+
(obj, end_idx) = partial_json_loads(current_text[start_idx:], flags)
|
| 241 |
+
if "arguments" not in obj:
|
| 242 |
+
obj["arguments"] = json.loads(json.dumps(obj, ensure_ascii=False))
|
| 243 |
+
if name is not None:
|
| 244 |
+
obj["name"] = name
|
| 245 |
+
|
| 246 |
+
is_complete.append(
|
| 247 |
+
is_complete_json(current_text[start_idx:start_idx +
|
| 248 |
+
end_idx]))
|
| 249 |
+
|
| 250 |
+
start_idx += end_idx
|
| 251 |
+
|
| 252 |
+
# TODO: is this necessary for kanana? can we remove this?
|
| 253 |
+
# depending on the prompt Llama can use
|
| 254 |
+
# either arguments or parameters
|
| 255 |
+
if "parameters" in obj:
|
| 256 |
+
assert "arguments" not in obj, \
|
| 257 |
+
"model generated both parameters and arguments"
|
| 258 |
+
obj["arguments"] = obj["parameters"]
|
| 259 |
+
tool_call_arr.append(obj)
|
| 260 |
+
|
| 261 |
+
# if you can find </fuction>, which declares the end of a function,
|
| 262 |
+
# move start_idx to the end so it can skip </fuction>
|
| 263 |
+
function_end_idx = current_text[start_idx:].find(self._func_suffix)
|
| 264 |
+
if function_end_idx != -1:
|
| 265 |
+
start_idx += len(self._func_suffix)
|
| 266 |
+
# if </function> hasn't been generated fully yet,
|
| 267 |
+
# but the function is complete, we break out from while loop
|
| 268 |
+
# and stream it right away
|
| 269 |
+
elif is_complete[-1]:
|
| 270 |
+
break
|
| 271 |
+
except partial_json_parser.core.exceptions.MalformedJSON:
|
| 272 |
+
logger.debug('not enough tokens to parse into JSON yet')
|
| 273 |
+
return None
|
| 274 |
+
|
| 275 |
+
# current_tool_call is the one that is being streamed
|
| 276 |
+
current_tool_call: dict = tool_call_arr[self.current_tool_id] \
|
| 277 |
+
if len(tool_call_arr) > 0 else {}
|
| 278 |
+
|
| 279 |
+
# case0: if there is no tool call in the array, return None
|
| 280 |
+
if len(tool_call_arr) == 0:
|
| 281 |
+
return None
|
| 282 |
+
|
| 283 |
+
# case1: we are starting a new tool in the array
|
| 284 |
+
# -> tool_call_arr has > 0 length AND has more elements than cursor
|
| 285 |
+
elif (len(tool_call_arr) > 0
|
| 286 |
+
and len(tool_call_arr) > self.current_tool_id + 1):
|
| 287 |
+
# if we're moving on to a new call, first make sure we
|
| 288 |
+
# haven't missed anything in the previous one that was
|
| 289 |
+
# auto-generated due to JSON completions, but wasn't
|
| 290 |
+
# streamed to the client yet.
|
| 291 |
+
# print("starting a new tool in the array, print remaining")
|
| 292 |
+
if self.current_tool_id >= 0:
|
| 293 |
+
cur_arguments = current_tool_call.get("arguments")
|
| 294 |
+
if cur_arguments:
|
| 295 |
+
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
|
| 296 |
+
sent = len(
|
| 297 |
+
self.streamed_args_for_tool[self.current_tool_id]) # streamed args for current tool call
|
| 298 |
+
argument_diff = cur_args_json[sent:] # args not yet streamed for current tool call
|
| 299 |
+
|
| 300 |
+
logger.debug("got arguments diff: %s", argument_diff)
|
| 301 |
+
delta = DeltaMessage(tool_calls=[
|
| 302 |
+
DeltaToolCall(index=self.current_tool_id,
|
| 303 |
+
function=DeltaFunctionCall(
|
| 304 |
+
arguments=argument_diff).
|
| 305 |
+
model_dump(exclude_none=True))
|
| 306 |
+
])
|
| 307 |
+
self.streamed_args_for_tool[
|
| 308 |
+
self.current_tool_id] += argument_diff # update streamed args for current tool call
|
| 309 |
+
else:
|
| 310 |
+
delta = None
|
| 311 |
+
else:
|
| 312 |
+
delta = None
|
| 313 |
+
# re-set stuff pertaining to progress in the current tool
|
| 314 |
+
self.current_tool_id = len(tool_call_arr) - 1 # update current tool call # 1을 더하는게 아니라 이렇게 하네 ..
|
| 315 |
+
self.current_tool_name_sent = False
|
| 316 |
+
self.streamed_args_for_tool.append("")
|
| 317 |
+
logger.debug("starting on new tool %d", self.current_tool_id)
|
| 318 |
+
return delta
|
| 319 |
+
|
| 320 |
+
# case2: if the current tool name hasn't been sent, send if available
|
| 321 |
+
elif not self.current_tool_name_sent:
|
| 322 |
+
function_name = current_tool_call.get("name")
|
| 323 |
+
if function_name:
|
| 324 |
+
delta = DeltaMessage(tool_calls=[
|
| 325 |
+
DeltaToolCall(index=self.current_tool_id,
|
| 326 |
+
type="function",
|
| 327 |
+
id=f"chatcmpl-tool-{random_uuid()}",
|
| 328 |
+
function=DeltaFunctionCall(
|
| 329 |
+
name=function_name).model_dump(
|
| 330 |
+
exclude_none=True))
|
| 331 |
+
])
|
| 332 |
+
self.current_tool_name_sent = True
|
| 333 |
+
else:
|
| 334 |
+
delta = None
|
| 335 |
+
|
| 336 |
+
# case3: now we know we're on the same tool call
|
| 337 |
+
# and can stream arguments
|
| 338 |
+
else:
|
| 339 |
+
cur_arguments = current_tool_call.get("arguments")
|
| 340 |
+
delta = None
|
| 341 |
+
# '{}' 인경우도 출력 해야함.
|
| 342 |
+
sent = len(
|
| 343 |
+
self.streamed_args_for_tool[self.current_tool_id]) # streamed args for current tool call
|
| 344 |
+
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
|
| 345 |
+
prev_arguments = self.prev_tool_call_arr[
|
| 346 |
+
self.current_tool_id].get("arguments")
|
| 347 |
+
argument_diff = None
|
| 348 |
+
# if current tool call is complete
|
| 349 |
+
if is_complete[self.current_tool_id]:
|
| 350 |
+
argument_diff = cur_args_json[sent:]
|
| 351 |
+
# if current tool call is not complete and prev_arguments exists
|
| 352 |
+
elif prev_arguments:
|
| 353 |
+
prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
|
| 354 |
+
if cur_args_json != prev_args_json:
|
| 355 |
+
# common prefix betwn prev and current args
|
| 356 |
+
prefix = find_common_prefix(
|
| 357 |
+
prev_args_json, cur_args_json)
|
| 358 |
+
argument_diff = prefix[sent:]
|
| 359 |
+
if argument_diff is not None and len(argument_diff) > 0:
|
| 360 |
+
delta = DeltaMessage(tool_calls=[
|
| 361 |
+
DeltaToolCall(index=self.current_tool_id,
|
| 362 |
+
function=DeltaFunctionCall(
|
| 363 |
+
arguments=argument_diff).
|
| 364 |
+
model_dump(exclude_none=True))
|
| 365 |
+
])
|
| 366 |
+
|
| 367 |
+
self.streamed_args_for_tool[
|
| 368 |
+
self.current_tool_id] += argument_diff
|
| 369 |
+
# update previous tool call array
|
| 370 |
+
self.prev_tool_call_arr = tool_call_arr
|
| 371 |
+
return delta
|
| 372 |
+
|
| 373 |
+
except Exception as e:
|
| 374 |
+
logger.exception("Error trying to handle streaming tool call.")
|
| 375 |
+
logger.debug(
|
| 376 |
+
"Skipping chunk as a result of tool streaming extraction "
|
| 377 |
+
"error")
|
| 378 |
+
return None
|
kanana_tool_calls/lmalign_v1.jinja
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{# version=v3-llama3.1 #}{%- macro append_new_param_info(param_declaration, comment_info, examples_info, depth) -%}
|
| 2 |
+
{%- set offset = "" -%}
|
| 3 |
+
{%- if depth >= 1 -%}
|
| 4 |
+
{%- set offset = " " * depth -%}
|
| 5 |
+
{%- endif -%}
|
| 6 |
+
{%- if comment_info != "<|NONE|>" -%}
|
| 7 |
+
{{ "\n" + offset + comment_info }}
|
| 8 |
+
{%- if examples_info | length > 0 -%}
|
| 9 |
+
{# Append each example info #}
|
| 10 |
+
{%- for example in examples_info -%}
|
| 11 |
+
{{ "\n" + offset + "// " + example|string|replace("'", '"') }}
|
| 12 |
+
{%- endfor -%}
|
| 13 |
+
{%- endif -%}
|
| 14 |
+
{%- endif -%}
|
| 15 |
+
{{ "\n" + offset + param_declaration }}
|
| 16 |
+
{%- endmacro -%}
|
| 17 |
+
|
| 18 |
+
{%- macro convert_data_type(param_type) -%}
|
| 19 |
+
{%- if param_type == "integer" or param_type == "float" -%}
|
| 20 |
+
{{ "number" }}
|
| 21 |
+
{%- else -%}
|
| 22 |
+
{{ param_type }}
|
| 23 |
+
{%- endif -%}
|
| 24 |
+
{%- endmacro -%}
|
| 25 |
+
|
| 26 |
+
{%- macro get_param_type(param) -%}
|
| 27 |
+
{%- set param_type = "any" -%}
|
| 28 |
+
|
| 29 |
+
{%- if "type" in param -%}
|
| 30 |
+
{%- set raw_param_type = param["type"] -%}
|
| 31 |
+
{%- if raw_param_type is iterable and raw_param_type is not string -%}
|
| 32 |
+
{%- set param_type = raw_param_type | join(" | ") -%}
|
| 33 |
+
{%- else -%}
|
| 34 |
+
{%- set param_type = raw_param_type -%}
|
| 35 |
+
{%- endif -%}
|
| 36 |
+
{{ convert_data_type(param_type) }}
|
| 37 |
+
{%- elif "oneOf" in param -%}
|
| 38 |
+
{%- set one_of_types = param["oneOf"]|selectattr("type", "defined")|list -%}
|
| 39 |
+
{%- set one_of_types = one_of_types|map(attribute="type")|unique|list -%}
|
| 40 |
+
{{ convert_data_type(one_of_types | join(" | ")) }}
|
| 41 |
+
{%- endif -%}
|
| 42 |
+
{%- endmacro -%}
|
| 43 |
+
|
| 44 |
+
{%- macro get_format_param(param) -%}
|
| 45 |
+
{%- if "format" in param -%}
|
| 46 |
+
{{ param["format"] }}
|
| 47 |
+
{%- elif "oneOf" in param -%}
|
| 48 |
+
{%- set formats = [] -%}
|
| 49 |
+
{%- for item in param["oneOf"] -%}
|
| 50 |
+
{%- if "format" in item -%}
|
| 51 |
+
{%- if item["format"] == param["oneOf"][-1]["format"] -%}
|
| 52 |
+
{{ item["format"] }}
|
| 53 |
+
{%- else -%}
|
| 54 |
+
{{ item["format"] + " or "}}
|
| 55 |
+
{%- endif -%}
|
| 56 |
+
{%- endif -%}
|
| 57 |
+
{%- endfor -%}
|
| 58 |
+
{%- else -%}
|
| 59 |
+
{{ "<|NONE|>" }}
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
{%- endmacro -%}
|
| 62 |
+
|
| 63 |
+
{%- macro get_param_info(param) -%}
|
| 64 |
+
{%- set param_type = param.get("type", "any") -%}
|
| 65 |
+
{%- set format_param = get_format_param(param) -%}
|
| 66 |
+
|
| 67 |
+
{%- if "description" in param or "default" in param or format_param != "<|NONE|>" or param["maximum"] or param["minimum"] or param["maxLength"] or param["minLength"] -%}
|
| 68 |
+
{{ "//" }}
|
| 69 |
+
{%- if "description" in param -%}
|
| 70 |
+
{%- set desc = param["description"] -%}
|
| 71 |
+
{%- if not desc.endswith(".") -%}
|
| 72 |
+
{%- set desc = desc + "." -%}
|
| 73 |
+
{%- endif -%}
|
| 74 |
+
{{ " " + desc }}
|
| 75 |
+
{%- endif -%}
|
| 76 |
+
|
| 77 |
+
{%- if "default" in param -%}
|
| 78 |
+
{%- set default_value = param["default"] -%}
|
| 79 |
+
{%- if param_type == "string" -%}
|
| 80 |
+
{%- set default_value = '"' ~ default_value ~ '"' -%}
|
| 81 |
+
{%- endif -%}
|
| 82 |
+
{{ " Default=" ~ default_value ~ "." }}
|
| 83 |
+
{%- endif -%}
|
| 84 |
+
|
| 85 |
+
{%- set format_param = get_format_param(param) -%}
|
| 86 |
+
{%- if format_param != "<|NONE|>" -%}
|
| 87 |
+
{{ " Format=" ~ format_param }}
|
| 88 |
+
{%- endif -%}
|
| 89 |
+
|
| 90 |
+
{%- for field, field_name in [("maximum", "Maximum"), ("minimum", "Minimum"), ("maxLength", "Maximum length"), ("minLength", "Minimum length")] -%}
|
| 91 |
+
{%- if field in param -%}
|
| 92 |
+
{{ " " + field_name ~ "=" ~ param[field] }}
|
| 93 |
+
{%- endif -%}
|
| 94 |
+
{%- endfor -%}
|
| 95 |
+
{%- else -%}
|
| 96 |
+
{{ "<|NONE|>"}}
|
| 97 |
+
{%- endif -%}
|
| 98 |
+
{%- endmacro -%}
|
| 99 |
+
|
| 100 |
+
{%- macro get_enum_option_str(enum_options) -%}
|
| 101 |
+
{%- for v in enum_options -%}
|
| 102 |
+
{%- if v is string -%}
|
| 103 |
+
{{ '"' + v + '"' }}
|
| 104 |
+
{%- else -%}
|
| 105 |
+
{{ v }}
|
| 106 |
+
{%- endif -%}
|
| 107 |
+
{%- if enum_options|length > 0 and v != enum_options[-1] -%}
|
| 108 |
+
{{ " | " }}
|
| 109 |
+
{%- endif -%}
|
| 110 |
+
{%- endfor -%}
|
| 111 |
+
{%- endmacro -%}
|
| 112 |
+
|
| 113 |
+
{%- macro get_array_typescript(param_name, param_dic, depth) -%}
|
| 114 |
+
{%- set offset = '' -%}
|
| 115 |
+
{%- if depth >= 1 -%}
|
| 116 |
+
{%- set offset = " " * depth -%}
|
| 117 |
+
{%- endif -%}
|
| 118 |
+
{%- set items_info = param_dic.get('items', {}) -%}
|
| 119 |
+
|
| 120 |
+
{%- if items_info|length == 0 -%}
|
| 121 |
+
{%- if param_name -%}
|
| 122 |
+
{{ "\n" + offset + param_name + ": []" }}
|
| 123 |
+
{%- else -%}
|
| 124 |
+
{{ "\n" + offset + "[]" }}
|
| 125 |
+
{%- endif -%}
|
| 126 |
+
{%- else -%}
|
| 127 |
+
{%- set array_type = get_param_type(items_info) -%}
|
| 128 |
+
{%- if array_type == 'object' -%}
|
| 129 |
+
{%- if param_name -%}
|
| 130 |
+
{{ "\n" + offset + param_name + ": {" }}
|
| 131 |
+
{%- else -%}
|
| 132 |
+
{{ "\n" + offset + "{" }}
|
| 133 |
+
{%- endif -%}
|
| 134 |
+
{{ get_parameter_typescript(items_info.get('properties', {}), items_info.get('required', []), depth + 1) -}}
|
| 135 |
+
{{- "\n" + offset + "}[]" }}
|
| 136 |
+
{%- elif array_type == 'array' -%}
|
| 137 |
+
{%- set item_info = get_array_typescript(None, items_info, depth + 1) -%}
|
| 138 |
+
{%- if not param_name -%}
|
| 139 |
+
{{ "\n" + item_info + "[]" }}
|
| 140 |
+
{%- else -%}
|
| 141 |
+
{{ "\n" + offset + param_name + ": " + item_info|trim + "[]" }}
|
| 142 |
+
{%- endif -%}
|
| 143 |
+
{%- else -%}
|
| 144 |
+
{%- if 'enum' in items_info -%}
|
| 145 |
+
{%- set item_type = get_enum_option_str(items_info['enum']) -%}
|
| 146 |
+
{%- if param_name is none -%}
|
| 147 |
+
{{ "(" + item_type + ")[]"}}
|
| 148 |
+
{%- else -%}
|
| 149 |
+
{{ "\n" + offset + param_name + ": (" + item_type + ")[]" }}
|
| 150 |
+
{%- endif -%}
|
| 151 |
+
{%- else -%}
|
| 152 |
+
{%- if param_name is none -%}
|
| 153 |
+
{{ "\n" + array_type + "[]" }}
|
| 154 |
+
{%- else -%}
|
| 155 |
+
{{ "\n" + offset + param_name + ": " + array_type + "[]," }}
|
| 156 |
+
{%- endif -%}
|
| 157 |
+
{%- endif -%}
|
| 158 |
+
{%- endif -%}
|
| 159 |
+
{%- endif -%}
|
| 160 |
+
{%- endmacro -%}
|
| 161 |
+
|
| 162 |
+
{%- macro get_parameter_typescript(properties, required_params, depth=0) -%}
|
| 163 |
+
{%- set res = "" -%}
|
| 164 |
+
{%- for param_name, param in properties.items() -%}
|
| 165 |
+
{%- if param is mapping -%}
|
| 166 |
+
{%- set comment_info = get_param_info(param) -%}
|
| 167 |
+
{# Param Examples #}
|
| 168 |
+
{%- set examples_info = [] -%}
|
| 169 |
+
{%- if "examples" in param -%}
|
| 170 |
+
{%- set examples_info = ["Example " + param_name + ":"] -%}
|
| 171 |
+
{%- set examples_info = examples_info + param["examples"] -%}
|
| 172 |
+
{%- endif -%}
|
| 173 |
+
|
| 174 |
+
{# Param Name declaration #}
|
| 175 |
+
{%- set param_declaration = param_name -%}
|
| 176 |
+
{%- if required_params is iterable and param_name not in required_params -%}
|
| 177 |
+
{%- set param_declaration = param_declaration + "?" -%}
|
| 178 |
+
{%- endif -%}
|
| 179 |
+
|
| 180 |
+
{%- set param_type = get_param_type(param) -%}
|
| 181 |
+
|
| 182 |
+
{# Handle indentation based on depth #}
|
| 183 |
+
{%- set offset = "" -%}
|
| 184 |
+
{%- if depth >= 1 -%}
|
| 185 |
+
{%- set offset = " " * depth -%}
|
| 186 |
+
{%- endif -%}
|
| 187 |
+
|
| 188 |
+
{%- if param_type == "object" -%}
|
| 189 |
+
{%- if comment_info != "<|NONE|>" -%}
|
| 190 |
+
{{ "\n" + offset + comment_info }}
|
| 191 |
+
{%- endif -%}
|
| 192 |
+
{%- if examples_info|length > 0 -%}
|
| 193 |
+
{%- for example in examples_info -%}
|
| 194 |
+
{{ "\n" + offset + "// " + example|string|replace("'", '"') }}
|
| 195 |
+
{%- endfor -%}
|
| 196 |
+
{%- endif -%}
|
| 197 |
+
{%- set param_declaration = param_declaration + ": {" -%}
|
| 198 |
+
{{ "\n" + offset + param_declaration -}}
|
| 199 |
+
{{- get_parameter_typescript(param.get("properties", {}), param.get("required", []), depth + 1) -}}
|
| 200 |
+
{{- "\n" + offset + "}," }}
|
| 201 |
+
{%- elif param_type == "array" -%}
|
| 202 |
+
{%- set item_info = param.get("items", {}) -%}
|
| 203 |
+
{%- if "type" not in item_info -%}
|
| 204 |
+
{%- set param_declaration = param_declaration + ": []," -%}
|
| 205 |
+
{{ append_new_param_info(param_declaration, comment_info, examples_info, depth) }}
|
| 206 |
+
{%- else -%}
|
| 207 |
+
{%- if comment_info != "<|NONE|>" -%}
|
| 208 |
+
{{ "\n" + offset + comment_info }}
|
| 209 |
+
{%- endif -%}
|
| 210 |
+
{%- if examples_info|length > 0 -%}
|
| 211 |
+
{%- for example in examples_info -%}
|
| 212 |
+
{{ "\n" + offset + "// " + example|string|replace("'", '"') }}
|
| 213 |
+
{%- endfor -%}
|
| 214 |
+
{%- endif -%}
|
| 215 |
+
{%- set array_declaration = get_array_typescript(param_declaration, param, depth) -%}
|
| 216 |
+
{%- if not array_declaration.endswith(",") -%}
|
| 217 |
+
{%- set array_declaration = array_declaration + "," -%}
|
| 218 |
+
{%- endif -%}
|
| 219 |
+
{{ array_declaration}}
|
| 220 |
+
{%- endif -%}
|
| 221 |
+
{%- else -%}
|
| 222 |
+
{%- if "enum" in param -%}
|
| 223 |
+
{%- set param_type = get_enum_option_str(param["enum"]) -%}
|
| 224 |
+
{%- endif -%}
|
| 225 |
+
{%- if "nullable" in param and param["nullable"] -%}
|
| 226 |
+
{%- set param_type = param_type + " | null" -%}
|
| 227 |
+
{%- endif -%}
|
| 228 |
+
{%- set param_declaration = param_declaration + ": " + param_type + "," -%}
|
| 229 |
+
{{ append_new_param_info(param_declaration, comment_info, examples_info, depth) }}
|
| 230 |
+
{%- endif -%}
|
| 231 |
+
{%- endif -%}
|
| 232 |
+
{%- endfor -%}
|
| 233 |
+
{%- endmacro -%}
|
| 234 |
+
|
| 235 |
+
{%- macro generate_schema_from_functions(functions, namespace='functions') -%}
|
| 236 |
+
{{ "// Supported function definitions that should be called when necessary.\n" -}}
|
| 237 |
+
{{- "namespace " + namespace + " {\n\n" -}}
|
| 238 |
+
|
| 239 |
+
{%- for function in functions -%}
|
| 240 |
+
{%- if function.get("function") -%}
|
| 241 |
+
{%- set function = function.get("function") -%}
|
| 242 |
+
{%- endif -%}
|
| 243 |
+
|
| 244 |
+
{%- set function_name = function.get("name") -%}
|
| 245 |
+
{%- if function_name -%}
|
| 246 |
+
{%- set description = function.get('description', '') -%}
|
| 247 |
+
{%- set parameters = function.get('parameters', {}) -%}
|
| 248 |
+
{{- "// " + description + "\n" -}}
|
| 249 |
+
{{- "type " + function_name -}}
|
| 250 |
+
{%- if parameters and parameters.get("properties") -%}
|
| 251 |
+
{{- " = (_: {" -}}
|
| 252 |
+
{%- set required_params = parameters.get("required", []) -%}
|
| 253 |
+
{{ get_parameter_typescript(parameters.get("properties"), required_params, 0) -}}
|
| 254 |
+
{{- "\n}) => any;\n\n" }}
|
| 255 |
+
{%- else -%}
|
| 256 |
+
{{ " = () => any;\n\n" }}
|
| 257 |
+
{%- endif -%}
|
| 258 |
+
{%- endif -%}
|
| 259 |
+
{%- endfor -%}
|
| 260 |
+
{{ "} // namespace " + namespace }}
|
| 261 |
+
{%- endmacro -%}
|
| 262 |
+
{%- if not tools is defined -%}
|
| 263 |
+
{%- set tools = none -%}
|
| 264 |
+
{%- endif -%}
|
| 265 |
+
|
| 266 |
+
{%- set has_code_interpreter = tools | selectattr("type", "equalto", "code_interpreter") | list | length > 0 -%}
|
| 267 |
+
{%- if has_code_interpreter -%}
|
| 268 |
+
{%- set tools = tools | rejectattr("type", "equalto", "code_interpreter") | list -%}
|
| 269 |
+
{%- endif -%}
|
| 270 |
+
|
| 271 |
+
{#- System message + builtin tools #}
|
| 272 |
+
{{- bos_token + "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
| 273 |
+
{%- if has_code_interpreter %}
|
| 274 |
+
{{- "Environment: ipython\n\n" }}
|
| 275 |
+
{%- else -%}
|
| 276 |
+
{{ "\n"}}
|
| 277 |
+
{%- endif %}
|
| 278 |
+
{%- if tools %}
|
| 279 |
+
{{- "\nYou have access to the following functions:\n\n" }}
|
| 280 |
+
{%- for t in tools %}
|
| 281 |
+
{%- if "type" in t -%}
|
| 282 |
+
{{ "Use the function '" + t["function"]["name"] + "' to '" + t["function"]["description"] + "'\n" + t["function"] | tojson() }}
|
| 283 |
+
{%- else -%}
|
| 284 |
+
{{ "Use the function '" + t["name"] + "' to '" + t["description"] + "'\n" + t | tojson }}
|
| 285 |
+
{%- endif -%}
|
| 286 |
+
{{- "\n\n" }}
|
| 287 |
+
{%- endfor %}
|
| 288 |
+
{{- '\nThink very carefully before calling functions.\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => `<function`\nparameters => a JSON dict with the function argument name as key and function argument value as value.\nend_tag => `</function>`\n\nHere is an example,\n<function=example_function_name>{"example_name": "example_value"}</function>\n\nReminder:\n- If looking for real time information use relevant functions before falling back to brave_search\n- Function calls MUST follow the specified format, start with <function= and end with </function>\n- Required parameters MUST be specified\n- Only call one function at a time\n- Put the entire function call reply on one line\n\n' -}}
|
| 289 |
+
{%- endif %}
|
| 290 |
+
{{- "<|eot_id|>" -}}
|
| 291 |
+
|
| 292 |
+
{%- for message in messages -%}
|
| 293 |
+
{%- if message['role'] == 'user' or message['role'] == 'system' -%}
|
| 294 |
+
{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
|
| 295 |
+
{%- elif message['role'] == 'tool' -%}
|
| 296 |
+
{{ '<|start_header_id|>ipython<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
|
| 297 |
+
{%- else -%}
|
| 298 |
+
{%- if (message['content'] and message['content']|length > 0) or ('tool_calls' in message and message['tool_calls'] and message['tool_calls']|length > 0) -%}
|
| 299 |
+
{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'}}
|
| 300 |
+
{%- endif -%}
|
| 301 |
+
{%- if message['content'] and message['content']|length > 0 -%}
|
| 302 |
+
{{ message['content'] }}
|
| 303 |
+
{%- endif -%}
|
| 304 |
+
{%- if 'tool_calls' in message and message['tool_calls'] and message['tool_calls']|length > 0 -%}
|
| 305 |
+
{%- for tool_call in message['tool_calls'] -%}
|
| 306 |
+
{%- if tool_call["function"]["name"] == "python" -%}
|
| 307 |
+
{{ '<|python_tag|>' + tool_call['function']['arguments']|tojson }}
|
| 308 |
+
{%- else -%}
|
| 309 |
+
{{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments']|tojson + '</function>' }}
|
| 310 |
+
{%- endif -%}
|
| 311 |
+
{%- endfor -%}
|
| 312 |
+
{{ '<|eom_id|>' }}
|
| 313 |
+
{%- elif message['content'] and message['content']|length > 0 -%}
|
| 314 |
+
{{ '<|eot_id|>' }}
|
| 315 |
+
{%- endif -%}
|
| 316 |
+
{%- endif -%}
|
| 317 |
+
{%- endfor -%}
|
| 318 |
+
{%- if add_generation_prompt -%}
|
| 319 |
+
{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
| 320 |
+
{%- endif -%}
|
model-00001-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4dad6208f73ba0cc3888e79b00cca1e1cda31918e7eaa54d4817434fb65e9752
|
| 3 |
+
size 4996655056
|
model-00002-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fe55eb0566228b34a8048e27ac87be1c3e40707332a9aed5dc20f706b18ed2b8
|
| 3 |
+
size 4996387760
|
model-00003-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e5702b956fc53c7794a5ea02c2970ce8f04c74a1df6d30680f0aa6aba0a8f8c9
|
| 3 |
+
size 4997955960
|
model-00004-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9028ffca029922250bef1ae796c5dd92ced54c06b1b0928250f0f5884afad0a4
|
| 3 |
+
size 4996092176
|
model-00005-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9169539b4dba1bb628cba22b414664da537aeb497188a7858b9deb6fcf7183ba
|
| 3 |
+
size 4996096824
|
model-00006-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0fe57c4eafa292e8cc26d7b9ed1b7c8d5ba659e824fd5b99fb19096313169432
|
| 3 |
+
size 4996409992
|
model-00007-of-00007.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5f241156348251c9ed009dad06600725fe2adf85adceb5420bff385fbe9bfb5
|
| 3 |
+
size 1413793632
|
model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|begin_of_text|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|eot_id|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<|end_of_text|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d44e2a3cfdfa7530be35f0d72c39b37ff438d4a1e69cc285b3ee461987d0bfa7
|
| 3 |
+
size 17210623
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,2092 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"128000": {
|
| 4 |
+
"content": "<|begin_of_text|>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"128001": {
|
| 12 |
+
"content": "<|end_of_text|>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"128002": {
|
| 20 |
+
"content": "<|reserved_special_token_0|>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"128003": {
|
| 28 |
+
"content": "<|reserved_special_token_1|>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
},
|
| 35 |
+
"128004": {
|
| 36 |
+
"content": "<|reserved_special_token_2|>",
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"normalized": false,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": true
|
| 42 |
+
},
|
| 43 |
+
"128005": {
|
| 44 |
+
"content": "<|reserved_special_token_3|>",
|
| 45 |
+
"lstrip": false,
|
| 46 |
+
"normalized": false,
|
| 47 |
+
"rstrip": false,
|
| 48 |
+
"single_word": false,
|
| 49 |
+
"special": true
|
| 50 |
+
},
|
| 51 |
+
"128006": {
|
| 52 |
+
"content": "<|start_header_id|>",
|
| 53 |
+
"lstrip": false,
|
| 54 |
+
"normalized": false,
|
| 55 |
+
"rstrip": false,
|
| 56 |
+
"single_word": false,
|
| 57 |
+
"special": true
|
| 58 |
+
},
|
| 59 |
+
"128007": {
|
| 60 |
+
"content": "<|end_header_id|>",
|
| 61 |
+
"lstrip": false,
|
| 62 |
+
"normalized": false,
|
| 63 |
+
"rstrip": false,
|
| 64 |
+
"single_word": false,
|
| 65 |
+
"special": true
|
| 66 |
+
},
|
| 67 |
+
"128008": {
|
| 68 |
+
"content": "<|reserved_special_token_4|>",
|
| 69 |
+
"lstrip": false,
|
| 70 |
+
"normalized": false,
|
| 71 |
+
"rstrip": false,
|
| 72 |
+
"single_word": false,
|
| 73 |
+
"special": true
|
| 74 |
+
},
|
| 75 |
+
"128009": {
|
| 76 |
+
"content": "<|eot_id|>",
|
| 77 |
+
"lstrip": false,
|
| 78 |
+
"normalized": false,
|
| 79 |
+
"rstrip": false,
|
| 80 |
+
"single_word": false,
|
| 81 |
+
"special": true
|
| 82 |
+
},
|
| 83 |
+
"128010": {
|
| 84 |
+
"content": "<|reserved_special_token_5|>",
|
| 85 |
+
"lstrip": false,
|
| 86 |
+
"normalized": false,
|
| 87 |
+
"rstrip": false,
|
| 88 |
+
"single_word": false,
|
| 89 |
+
"special": true
|
| 90 |
+
},
|
| 91 |
+
"128011": {
|
| 92 |
+
"content": "<|reserved_special_token_6|>",
|
| 93 |
+
"lstrip": false,
|
| 94 |
+
"normalized": false,
|
| 95 |
+
"rstrip": false,
|
| 96 |
+
"single_word": false,
|
| 97 |
+
"special": true
|
| 98 |
+
},
|
| 99 |
+
"128012": {
|
| 100 |
+
"content": "<|reserved_special_token_7|>",
|
| 101 |
+
"lstrip": false,
|
| 102 |
+
"normalized": false,
|
| 103 |
+
"rstrip": false,
|
| 104 |
+
"single_word": false,
|
| 105 |
+
"special": true
|
| 106 |
+
},
|
| 107 |
+
"128013": {
|
| 108 |
+
"content": "<|reserved_special_token_8|>",
|
| 109 |
+
"lstrip": false,
|
| 110 |
+
"normalized": false,
|
| 111 |
+
"rstrip": false,
|
| 112 |
+
"single_word": false,
|
| 113 |
+
"special": true
|
| 114 |
+
},
|
| 115 |
+
"128014": {
|
| 116 |
+
"content": "<|reserved_special_token_9|>",
|
| 117 |
+
"lstrip": false,
|
| 118 |
+
"normalized": false,
|
| 119 |
+
"rstrip": false,
|
| 120 |
+
"single_word": false,
|
| 121 |
+
"special": true
|
| 122 |
+
},
|
| 123 |
+
"128015": {
|
| 124 |
+
"content": "<|reserved_special_token_10|>",
|
| 125 |
+
"lstrip": false,
|
| 126 |
+
"normalized": false,
|
| 127 |
+
"rstrip": false,
|
| 128 |
+
"single_word": false,
|
| 129 |
+
"special": true
|
| 130 |
+
},
|
| 131 |
+
"128016": {
|
| 132 |
+
"content": "<|reserved_special_token_11|>",
|
| 133 |
+
"lstrip": false,
|
| 134 |
+
"normalized": false,
|
| 135 |
+
"rstrip": false,
|
| 136 |
+
"single_word": false,
|
| 137 |
+
"special": true
|
| 138 |
+
},
|
| 139 |
+
"128017": {
|
| 140 |
+
"content": "<|reserved_special_token_12|>",
|
| 141 |
+
"lstrip": false,
|
| 142 |
+
"normalized": false,
|
| 143 |
+
"rstrip": false,
|
| 144 |
+
"single_word": false,
|
| 145 |
+
"special": true
|
| 146 |
+
},
|
| 147 |
+
"128018": {
|
| 148 |
+
"content": "<|reserved_special_token_13|>",
|
| 149 |
+
"lstrip": false,
|
| 150 |
+
"normalized": false,
|
| 151 |
+
"rstrip": false,
|
| 152 |
+
"single_word": false,
|
| 153 |
+
"special": true
|
| 154 |
+
},
|
| 155 |
+
"128019": {
|
| 156 |
+
"content": "<|reserved_special_token_14|>",
|
| 157 |
+
"lstrip": false,
|
| 158 |
+
"normalized": false,
|
| 159 |
+
"rstrip": false,
|
| 160 |
+
"single_word": false,
|
| 161 |
+
"special": true
|
| 162 |
+
},
|
| 163 |
+
"128020": {
|
| 164 |
+
"content": "<|reserved_special_token_15|>",
|
| 165 |
+
"lstrip": false,
|
| 166 |
+
"normalized": false,
|
| 167 |
+
"rstrip": false,
|
| 168 |
+
"single_word": false,
|
| 169 |
+
"special": true
|
| 170 |
+
},
|
| 171 |
+
"128021": {
|
| 172 |
+
"content": "<|reserved_special_token_16|>",
|
| 173 |
+
"lstrip": false,
|
| 174 |
+
"normalized": false,
|
| 175 |
+
"rstrip": false,
|
| 176 |
+
"single_word": false,
|
| 177 |
+
"special": true
|
| 178 |
+
},
|
| 179 |
+
"128022": {
|
| 180 |
+
"content": "<|reserved_special_token_17|>",
|
| 181 |
+
"lstrip": false,
|
| 182 |
+
"normalized": false,
|
| 183 |
+
"rstrip": false,
|
| 184 |
+
"single_word": false,
|
| 185 |
+
"special": true
|
| 186 |
+
},
|
| 187 |
+
"128023": {
|
| 188 |
+
"content": "<|reserved_special_token_18|>",
|
| 189 |
+
"lstrip": false,
|
| 190 |
+
"normalized": false,
|
| 191 |
+
"rstrip": false,
|
| 192 |
+
"single_word": false,
|
| 193 |
+
"special": true
|
| 194 |
+
},
|
| 195 |
+
"128024": {
|
| 196 |
+
"content": "<|reserved_special_token_19|>",
|
| 197 |
+
"lstrip": false,
|
| 198 |
+
"normalized": false,
|
| 199 |
+
"rstrip": false,
|
| 200 |
+
"single_word": false,
|
| 201 |
+
"special": true
|
| 202 |
+
},
|
| 203 |
+
"128025": {
|
| 204 |
+
"content": "<|reserved_special_token_20|>",
|
| 205 |
+
"lstrip": false,
|
| 206 |
+
"normalized": false,
|
| 207 |
+
"rstrip": false,
|
| 208 |
+
"single_word": false,
|
| 209 |
+
"special": true
|
| 210 |
+
},
|
| 211 |
+
"128026": {
|
| 212 |
+
"content": "<|reserved_special_token_21|>",
|
| 213 |
+
"lstrip": false,
|
| 214 |
+
"normalized": false,
|
| 215 |
+
"rstrip": false,
|
| 216 |
+
"single_word": false,
|
| 217 |
+
"special": true
|
| 218 |
+
},
|
| 219 |
+
"128027": {
|
| 220 |
+
"content": "<|reserved_special_token_22|>",
|
| 221 |
+
"lstrip": false,
|
| 222 |
+
"normalized": false,
|
| 223 |
+
"rstrip": false,
|
| 224 |
+
"single_word": false,
|
| 225 |
+
"special": true
|
| 226 |
+
},
|
| 227 |
+
"128028": {
|
| 228 |
+
"content": "<|reserved_special_token_23|>",
|
| 229 |
+
"lstrip": false,
|
| 230 |
+
"normalized": false,
|
| 231 |
+
"rstrip": false,
|
| 232 |
+
"single_word": false,
|
| 233 |
+
"special": true
|
| 234 |
+
},
|
| 235 |
+
"128029": {
|
| 236 |
+
"content": "<|reserved_special_token_24|>",
|
| 237 |
+
"lstrip": false,
|
| 238 |
+
"normalized": false,
|
| 239 |
+
"rstrip": false,
|
| 240 |
+
"single_word": false,
|
| 241 |
+
"special": true
|
| 242 |
+
},
|
| 243 |
+
"128030": {
|
| 244 |
+
"content": "<|reserved_special_token_25|>",
|
| 245 |
+
"lstrip": false,
|
| 246 |
+
"normalized": false,
|
| 247 |
+
"rstrip": false,
|
| 248 |
+
"single_word": false,
|
| 249 |
+
"special": true
|
| 250 |
+
},
|
| 251 |
+
"128031": {
|
| 252 |
+
"content": "<|reserved_special_token_26|>",
|
| 253 |
+
"lstrip": false,
|
| 254 |
+
"normalized": false,
|
| 255 |
+
"rstrip": false,
|
| 256 |
+
"single_word": false,
|
| 257 |
+
"special": true
|
| 258 |
+
},
|
| 259 |
+
"128032": {
|
| 260 |
+
"content": "<|reserved_special_token_27|>",
|
| 261 |
+
"lstrip": false,
|
| 262 |
+
"normalized": false,
|
| 263 |
+
"rstrip": false,
|
| 264 |
+
"single_word": false,
|
| 265 |
+
"special": true
|
| 266 |
+
},
|
| 267 |
+
"128033": {
|
| 268 |
+
"content": "<|reserved_special_token_28|>",
|
| 269 |
+
"lstrip": false,
|
| 270 |
+
"normalized": false,
|
| 271 |
+
"rstrip": false,
|
| 272 |
+
"single_word": false,
|
| 273 |
+
"special": true
|
| 274 |
+
},
|
| 275 |
+
"128034": {
|
| 276 |
+
"content": "<|reserved_special_token_29|>",
|
| 277 |
+
"lstrip": false,
|
| 278 |
+
"normalized": false,
|
| 279 |
+
"rstrip": false,
|
| 280 |
+
"single_word": false,
|
| 281 |
+
"special": true
|
| 282 |
+
},
|
| 283 |
+
"128035": {
|
| 284 |
+
"content": "<|reserved_special_token_30|>",
|
| 285 |
+
"lstrip": false,
|
| 286 |
+
"normalized": false,
|
| 287 |
+
"rstrip": false,
|
| 288 |
+
"single_word": false,
|
| 289 |
+
"special": true
|
| 290 |
+
},
|
| 291 |
+
"128036": {
|
| 292 |
+
"content": "<|reserved_special_token_31|>",
|
| 293 |
+
"lstrip": false,
|
| 294 |
+
"normalized": false,
|
| 295 |
+
"rstrip": false,
|
| 296 |
+
"single_word": false,
|
| 297 |
+
"special": true
|
| 298 |
+
},
|
| 299 |
+
"128037": {
|
| 300 |
+
"content": "<|reserved_special_token_32|>",
|
| 301 |
+
"lstrip": false,
|
| 302 |
+
"normalized": false,
|
| 303 |
+
"rstrip": false,
|
| 304 |
+
"single_word": false,
|
| 305 |
+
"special": true
|
| 306 |
+
},
|
| 307 |
+
"128038": {
|
| 308 |
+
"content": "<|reserved_special_token_33|>",
|
| 309 |
+
"lstrip": false,
|
| 310 |
+
"normalized": false,
|
| 311 |
+
"rstrip": false,
|
| 312 |
+
"single_word": false,
|
| 313 |
+
"special": true
|
| 314 |
+
},
|
| 315 |
+
"128039": {
|
| 316 |
+
"content": "<|reserved_special_token_34|>",
|
| 317 |
+
"lstrip": false,
|
| 318 |
+
"normalized": false,
|
| 319 |
+
"rstrip": false,
|
| 320 |
+
"single_word": false,
|
| 321 |
+
"special": true
|
| 322 |
+
},
|
| 323 |
+
"128040": {
|
| 324 |
+
"content": "<|reserved_special_token_35|>",
|
| 325 |
+
"lstrip": false,
|
| 326 |
+
"normalized": false,
|
| 327 |
+
"rstrip": false,
|
| 328 |
+
"single_word": false,
|
| 329 |
+
"special": true
|
| 330 |
+
},
|
| 331 |
+
"128041": {
|
| 332 |
+
"content": "<|reserved_special_token_36|>",
|
| 333 |
+
"lstrip": false,
|
| 334 |
+
"normalized": false,
|
| 335 |
+
"rstrip": false,
|
| 336 |
+
"single_word": false,
|
| 337 |
+
"special": true
|
| 338 |
+
},
|
| 339 |
+
"128042": {
|
| 340 |
+
"content": "<|reserved_special_token_37|>",
|
| 341 |
+
"lstrip": false,
|
| 342 |
+
"normalized": false,
|
| 343 |
+
"rstrip": false,
|
| 344 |
+
"single_word": false,
|
| 345 |
+
"special": true
|
| 346 |
+
},
|
| 347 |
+
"128043": {
|
| 348 |
+
"content": "<|reserved_special_token_38|>",
|
| 349 |
+
"lstrip": false,
|
| 350 |
+
"normalized": false,
|
| 351 |
+
"rstrip": false,
|
| 352 |
+
"single_word": false,
|
| 353 |
+
"special": true
|
| 354 |
+
},
|
| 355 |
+
"128044": {
|
| 356 |
+
"content": "<|reserved_special_token_39|>",
|
| 357 |
+
"lstrip": false,
|
| 358 |
+
"normalized": false,
|
| 359 |
+
"rstrip": false,
|
| 360 |
+
"single_word": false,
|
| 361 |
+
"special": true
|
| 362 |
+
},
|
| 363 |
+
"128045": {
|
| 364 |
+
"content": "<|reserved_special_token_40|>",
|
| 365 |
+
"lstrip": false,
|
| 366 |
+
"normalized": false,
|
| 367 |
+
"rstrip": false,
|
| 368 |
+
"single_word": false,
|
| 369 |
+
"special": true
|
| 370 |
+
},
|
| 371 |
+
"128046": {
|
| 372 |
+
"content": "<|reserved_special_token_41|>",
|
| 373 |
+
"lstrip": false,
|
| 374 |
+
"normalized": false,
|
| 375 |
+
"rstrip": false,
|
| 376 |
+
"single_word": false,
|
| 377 |
+
"special": true
|
| 378 |
+
},
|
| 379 |
+
"128047": {
|
| 380 |
+
"content": "<|reserved_special_token_42|>",
|
| 381 |
+
"lstrip": false,
|
| 382 |
+
"normalized": false,
|
| 383 |
+
"rstrip": false,
|
| 384 |
+
"single_word": false,
|
| 385 |
+
"special": true
|
| 386 |
+
},
|
| 387 |
+
"128048": {
|
| 388 |
+
"content": "<|reserved_special_token_43|>",
|
| 389 |
+
"lstrip": false,
|
| 390 |
+
"normalized": false,
|
| 391 |
+
"rstrip": false,
|
| 392 |
+
"single_word": false,
|
| 393 |
+
"special": true
|
| 394 |
+
},
|
| 395 |
+
"128049": {
|
| 396 |
+
"content": "<|reserved_special_token_44|>",
|
| 397 |
+
"lstrip": false,
|
| 398 |
+
"normalized": false,
|
| 399 |
+
"rstrip": false,
|
| 400 |
+
"single_word": false,
|
| 401 |
+
"special": true
|
| 402 |
+
},
|
| 403 |
+
"128050": {
|
| 404 |
+
"content": "<|reserved_special_token_45|>",
|
| 405 |
+
"lstrip": false,
|
| 406 |
+
"normalized": false,
|
| 407 |
+
"rstrip": false,
|
| 408 |
+
"single_word": false,
|
| 409 |
+
"special": true
|
| 410 |
+
},
|
| 411 |
+
"128051": {
|
| 412 |
+
"content": "<|reserved_special_token_46|>",
|
| 413 |
+
"lstrip": false,
|
| 414 |
+
"normalized": false,
|
| 415 |
+
"rstrip": false,
|
| 416 |
+
"single_word": false,
|
| 417 |
+
"special": true
|
| 418 |
+
},
|
| 419 |
+
"128052": {
|
| 420 |
+
"content": "<|reserved_special_token_47|>",
|
| 421 |
+
"lstrip": false,
|
| 422 |
+
"normalized": false,
|
| 423 |
+
"rstrip": false,
|
| 424 |
+
"single_word": false,
|
| 425 |
+
"special": true
|
| 426 |
+
},
|
| 427 |
+
"128053": {
|
| 428 |
+
"content": "<|reserved_special_token_48|>",
|
| 429 |
+
"lstrip": false,
|
| 430 |
+
"normalized": false,
|
| 431 |
+
"rstrip": false,
|
| 432 |
+
"single_word": false,
|
| 433 |
+
"special": true
|
| 434 |
+
},
|
| 435 |
+
"128054": {
|
| 436 |
+
"content": "<|reserved_special_token_49|>",
|
| 437 |
+
"lstrip": false,
|
| 438 |
+
"normalized": false,
|
| 439 |
+
"rstrip": false,
|
| 440 |
+
"single_word": false,
|
| 441 |
+
"special": true
|
| 442 |
+
},
|
| 443 |
+
"128055": {
|
| 444 |
+
"content": "<|reserved_special_token_50|>",
|
| 445 |
+
"lstrip": false,
|
| 446 |
+
"normalized": false,
|
| 447 |
+
"rstrip": false,
|
| 448 |
+
"single_word": false,
|
| 449 |
+
"special": true
|
| 450 |
+
},
|
| 451 |
+
"128056": {
|
| 452 |
+
"content": "<|reserved_special_token_51|>",
|
| 453 |
+
"lstrip": false,
|
| 454 |
+
"normalized": false,
|
| 455 |
+
"rstrip": false,
|
| 456 |
+
"single_word": false,
|
| 457 |
+
"special": true
|
| 458 |
+
},
|
| 459 |
+
"128057": {
|
| 460 |
+
"content": "<|reserved_special_token_52|>",
|
| 461 |
+
"lstrip": false,
|
| 462 |
+
"normalized": false,
|
| 463 |
+
"rstrip": false,
|
| 464 |
+
"single_word": false,
|
| 465 |
+
"special": true
|
| 466 |
+
},
|
| 467 |
+
"128058": {
|
| 468 |
+
"content": "<|reserved_special_token_53|>",
|
| 469 |
+
"lstrip": false,
|
| 470 |
+
"normalized": false,
|
| 471 |
+
"rstrip": false,
|
| 472 |
+
"single_word": false,
|
| 473 |
+
"special": true
|
| 474 |
+
},
|
| 475 |
+
"128059": {
|
| 476 |
+
"content": "<|reserved_special_token_54|>",
|
| 477 |
+
"lstrip": false,
|
| 478 |
+
"normalized": false,
|
| 479 |
+
"rstrip": false,
|
| 480 |
+
"single_word": false,
|
| 481 |
+
"special": true
|
| 482 |
+
},
|
| 483 |
+
"128060": {
|
| 484 |
+
"content": "<|reserved_special_token_55|>",
|
| 485 |
+
"lstrip": false,
|
| 486 |
+
"normalized": false,
|
| 487 |
+
"rstrip": false,
|
| 488 |
+
"single_word": false,
|
| 489 |
+
"special": true
|
| 490 |
+
},
|
| 491 |
+
"128061": {
|
| 492 |
+
"content": "<|reserved_special_token_56|>",
|
| 493 |
+
"lstrip": false,
|
| 494 |
+
"normalized": false,
|
| 495 |
+
"rstrip": false,
|
| 496 |
+
"single_word": false,
|
| 497 |
+
"special": true
|
| 498 |
+
},
|
| 499 |
+
"128062": {
|
| 500 |
+
"content": "<|reserved_special_token_57|>",
|
| 501 |
+
"lstrip": false,
|
| 502 |
+
"normalized": false,
|
| 503 |
+
"rstrip": false,
|
| 504 |
+
"single_word": false,
|
| 505 |
+
"special": true
|
| 506 |
+
},
|
| 507 |
+
"128063": {
|
| 508 |
+
"content": "<|reserved_special_token_58|>",
|
| 509 |
+
"lstrip": false,
|
| 510 |
+
"normalized": false,
|
| 511 |
+
"rstrip": false,
|
| 512 |
+
"single_word": false,
|
| 513 |
+
"special": true
|
| 514 |
+
},
|
| 515 |
+
"128064": {
|
| 516 |
+
"content": "<|reserved_special_token_59|>",
|
| 517 |
+
"lstrip": false,
|
| 518 |
+
"normalized": false,
|
| 519 |
+
"rstrip": false,
|
| 520 |
+
"single_word": false,
|
| 521 |
+
"special": true
|
| 522 |
+
},
|
| 523 |
+
"128065": {
|
| 524 |
+
"content": "<|reserved_special_token_60|>",
|
| 525 |
+
"lstrip": false,
|
| 526 |
+
"normalized": false,
|
| 527 |
+
"rstrip": false,
|
| 528 |
+
"single_word": false,
|
| 529 |
+
"special": true
|
| 530 |
+
},
|
| 531 |
+
"128066": {
|
| 532 |
+
"content": "<|reserved_special_token_61|>",
|
| 533 |
+
"lstrip": false,
|
| 534 |
+
"normalized": false,
|
| 535 |
+
"rstrip": false,
|
| 536 |
+
"single_word": false,
|
| 537 |
+
"special": true
|
| 538 |
+
},
|
| 539 |
+
"128067": {
|
| 540 |
+
"content": "<|reserved_special_token_62|>",
|
| 541 |
+
"lstrip": false,
|
| 542 |
+
"normalized": false,
|
| 543 |
+
"rstrip": false,
|
| 544 |
+
"single_word": false,
|
| 545 |
+
"special": true
|
| 546 |
+
},
|
| 547 |
+
"128068": {
|
| 548 |
+
"content": "<|reserved_special_token_63|>",
|
| 549 |
+
"lstrip": false,
|
| 550 |
+
"normalized": false,
|
| 551 |
+
"rstrip": false,
|
| 552 |
+
"single_word": false,
|
| 553 |
+
"special": true
|
| 554 |
+
},
|
| 555 |
+
"128069": {
|
| 556 |
+
"content": "<|reserved_special_token_64|>",
|
| 557 |
+
"lstrip": false,
|
| 558 |
+
"normalized": false,
|
| 559 |
+
"rstrip": false,
|
| 560 |
+
"single_word": false,
|
| 561 |
+
"special": true
|
| 562 |
+
},
|
| 563 |
+
"128070": {
|
| 564 |
+
"content": "<|reserved_special_token_65|>",
|
| 565 |
+
"lstrip": false,
|
| 566 |
+
"normalized": false,
|
| 567 |
+
"rstrip": false,
|
| 568 |
+
"single_word": false,
|
| 569 |
+
"special": true
|
| 570 |
+
},
|
| 571 |
+
"128071": {
|
| 572 |
+
"content": "<|reserved_special_token_66|>",
|
| 573 |
+
"lstrip": false,
|
| 574 |
+
"normalized": false,
|
| 575 |
+
"rstrip": false,
|
| 576 |
+
"single_word": false,
|
| 577 |
+
"special": true
|
| 578 |
+
},
|
| 579 |
+
"128072": {
|
| 580 |
+
"content": "<|reserved_special_token_67|>",
|
| 581 |
+
"lstrip": false,
|
| 582 |
+
"normalized": false,
|
| 583 |
+
"rstrip": false,
|
| 584 |
+
"single_word": false,
|
| 585 |
+
"special": true
|
| 586 |
+
},
|
| 587 |
+
"128073": {
|
| 588 |
+
"content": "<|reserved_special_token_68|>",
|
| 589 |
+
"lstrip": false,
|
| 590 |
+
"normalized": false,
|
| 591 |
+
"rstrip": false,
|
| 592 |
+
"single_word": false,
|
| 593 |
+
"special": true
|
| 594 |
+
},
|
| 595 |
+
"128074": {
|
| 596 |
+
"content": "<|reserved_special_token_69|>",
|
| 597 |
+
"lstrip": false,
|
| 598 |
+
"normalized": false,
|
| 599 |
+
"rstrip": false,
|
| 600 |
+
"single_word": false,
|
| 601 |
+
"special": true
|
| 602 |
+
},
|
| 603 |
+
"128075": {
|
| 604 |
+
"content": "<|reserved_special_token_70|>",
|
| 605 |
+
"lstrip": false,
|
| 606 |
+
"normalized": false,
|
| 607 |
+
"rstrip": false,
|
| 608 |
+
"single_word": false,
|
| 609 |
+
"special": true
|
| 610 |
+
},
|
| 611 |
+
"128076": {
|
| 612 |
+
"content": "<|reserved_special_token_71|>",
|
| 613 |
+
"lstrip": false,
|
| 614 |
+
"normalized": false,
|
| 615 |
+
"rstrip": false,
|
| 616 |
+
"single_word": false,
|
| 617 |
+
"special": true
|
| 618 |
+
},
|
| 619 |
+
"128077": {
|
| 620 |
+
"content": "<|reserved_special_token_72|>",
|
| 621 |
+
"lstrip": false,
|
| 622 |
+
"normalized": false,
|
| 623 |
+
"rstrip": false,
|
| 624 |
+
"single_word": false,
|
| 625 |
+
"special": true
|
| 626 |
+
},
|
| 627 |
+
"128078": {
|
| 628 |
+
"content": "<|reserved_special_token_73|>",
|
| 629 |
+
"lstrip": false,
|
| 630 |
+
"normalized": false,
|
| 631 |
+
"rstrip": false,
|
| 632 |
+
"single_word": false,
|
| 633 |
+
"special": true
|
| 634 |
+
},
|
| 635 |
+
"128079": {
|
| 636 |
+
"content": "<|reserved_special_token_74|>",
|
| 637 |
+
"lstrip": false,
|
| 638 |
+
"normalized": false,
|
| 639 |
+
"rstrip": false,
|
| 640 |
+
"single_word": false,
|
| 641 |
+
"special": true
|
| 642 |
+
},
|
| 643 |
+
"128080": {
|
| 644 |
+
"content": "<|reserved_special_token_75|>",
|
| 645 |
+
"lstrip": false,
|
| 646 |
+
"normalized": false,
|
| 647 |
+
"rstrip": false,
|
| 648 |
+
"single_word": false,
|
| 649 |
+
"special": true
|
| 650 |
+
},
|
| 651 |
+
"128081": {
|
| 652 |
+
"content": "<|reserved_special_token_76|>",
|
| 653 |
+
"lstrip": false,
|
| 654 |
+
"normalized": false,
|
| 655 |
+
"rstrip": false,
|
| 656 |
+
"single_word": false,
|
| 657 |
+
"special": true
|
| 658 |
+
},
|
| 659 |
+
"128082": {
|
| 660 |
+
"content": "<|reserved_special_token_77|>",
|
| 661 |
+
"lstrip": false,
|
| 662 |
+
"normalized": false,
|
| 663 |
+
"rstrip": false,
|
| 664 |
+
"single_word": false,
|
| 665 |
+
"special": true
|
| 666 |
+
},
|
| 667 |
+
"128083": {
|
| 668 |
+
"content": "<|reserved_special_token_78|>",
|
| 669 |
+
"lstrip": false,
|
| 670 |
+
"normalized": false,
|
| 671 |
+
"rstrip": false,
|
| 672 |
+
"single_word": false,
|
| 673 |
+
"special": true
|
| 674 |
+
},
|
| 675 |
+
"128084": {
|
| 676 |
+
"content": "<|reserved_special_token_79|>",
|
| 677 |
+
"lstrip": false,
|
| 678 |
+
"normalized": false,
|
| 679 |
+
"rstrip": false,
|
| 680 |
+
"single_word": false,
|
| 681 |
+
"special": true
|
| 682 |
+
},
|
| 683 |
+
"128085": {
|
| 684 |
+
"content": "<|reserved_special_token_80|>",
|
| 685 |
+
"lstrip": false,
|
| 686 |
+
"normalized": false,
|
| 687 |
+
"rstrip": false,
|
| 688 |
+
"single_word": false,
|
| 689 |
+
"special": true
|
| 690 |
+
},
|
| 691 |
+
"128086": {
|
| 692 |
+
"content": "<|reserved_special_token_81|>",
|
| 693 |
+
"lstrip": false,
|
| 694 |
+
"normalized": false,
|
| 695 |
+
"rstrip": false,
|
| 696 |
+
"single_word": false,
|
| 697 |
+
"special": true
|
| 698 |
+
},
|
| 699 |
+
"128087": {
|
| 700 |
+
"content": "<|reserved_special_token_82|>",
|
| 701 |
+
"lstrip": false,
|
| 702 |
+
"normalized": false,
|
| 703 |
+
"rstrip": false,
|
| 704 |
+
"single_word": false,
|
| 705 |
+
"special": true
|
| 706 |
+
},
|
| 707 |
+
"128088": {
|
| 708 |
+
"content": "<|reserved_special_token_83|>",
|
| 709 |
+
"lstrip": false,
|
| 710 |
+
"normalized": false,
|
| 711 |
+
"rstrip": false,
|
| 712 |
+
"single_word": false,
|
| 713 |
+
"special": true
|
| 714 |
+
},
|
| 715 |
+
"128089": {
|
| 716 |
+
"content": "<|reserved_special_token_84|>",
|
| 717 |
+
"lstrip": false,
|
| 718 |
+
"normalized": false,
|
| 719 |
+
"rstrip": false,
|
| 720 |
+
"single_word": false,
|
| 721 |
+
"special": true
|
| 722 |
+
},
|
| 723 |
+
"128090": {
|
| 724 |
+
"content": "<|reserved_special_token_85|>",
|
| 725 |
+
"lstrip": false,
|
| 726 |
+
"normalized": false,
|
| 727 |
+
"rstrip": false,
|
| 728 |
+
"single_word": false,
|
| 729 |
+
"special": true
|
| 730 |
+
},
|
| 731 |
+
"128091": {
|
| 732 |
+
"content": "<|reserved_special_token_86|>",
|
| 733 |
+
"lstrip": false,
|
| 734 |
+
"normalized": false,
|
| 735 |
+
"rstrip": false,
|
| 736 |
+
"single_word": false,
|
| 737 |
+
"special": true
|
| 738 |
+
},
|
| 739 |
+
"128092": {
|
| 740 |
+
"content": "<|reserved_special_token_87|>",
|
| 741 |
+
"lstrip": false,
|
| 742 |
+
"normalized": false,
|
| 743 |
+
"rstrip": false,
|
| 744 |
+
"single_word": false,
|
| 745 |
+
"special": true
|
| 746 |
+
},
|
| 747 |
+
"128093": {
|
| 748 |
+
"content": "<|reserved_special_token_88|>",
|
| 749 |
+
"lstrip": false,
|
| 750 |
+
"normalized": false,
|
| 751 |
+
"rstrip": false,
|
| 752 |
+
"single_word": false,
|
| 753 |
+
"special": true
|
| 754 |
+
},
|
| 755 |
+
"128094": {
|
| 756 |
+
"content": "<|reserved_special_token_89|>",
|
| 757 |
+
"lstrip": false,
|
| 758 |
+
"normalized": false,
|
| 759 |
+
"rstrip": false,
|
| 760 |
+
"single_word": false,
|
| 761 |
+
"special": true
|
| 762 |
+
},
|
| 763 |
+
"128095": {
|
| 764 |
+
"content": "<|reserved_special_token_90|>",
|
| 765 |
+
"lstrip": false,
|
| 766 |
+
"normalized": false,
|
| 767 |
+
"rstrip": false,
|
| 768 |
+
"single_word": false,
|
| 769 |
+
"special": true
|
| 770 |
+
},
|
| 771 |
+
"128096": {
|
| 772 |
+
"content": "<|reserved_special_token_91|>",
|
| 773 |
+
"lstrip": false,
|
| 774 |
+
"normalized": false,
|
| 775 |
+
"rstrip": false,
|
| 776 |
+
"single_word": false,
|
| 777 |
+
"special": true
|
| 778 |
+
},
|
| 779 |
+
"128097": {
|
| 780 |
+
"content": "<|reserved_special_token_92|>",
|
| 781 |
+
"lstrip": false,
|
| 782 |
+
"normalized": false,
|
| 783 |
+
"rstrip": false,
|
| 784 |
+
"single_word": false,
|
| 785 |
+
"special": true
|
| 786 |
+
},
|
| 787 |
+
"128098": {
|
| 788 |
+
"content": "<|reserved_special_token_93|>",
|
| 789 |
+
"lstrip": false,
|
| 790 |
+
"normalized": false,
|
| 791 |
+
"rstrip": false,
|
| 792 |
+
"single_word": false,
|
| 793 |
+
"special": true
|
| 794 |
+
},
|
| 795 |
+
"128099": {
|
| 796 |
+
"content": "<|reserved_special_token_94|>",
|
| 797 |
+
"lstrip": false,
|
| 798 |
+
"normalized": false,
|
| 799 |
+
"rstrip": false,
|
| 800 |
+
"single_word": false,
|
| 801 |
+
"special": true
|
| 802 |
+
},
|
| 803 |
+
"128100": {
|
| 804 |
+
"content": "<|reserved_special_token_95|>",
|
| 805 |
+
"lstrip": false,
|
| 806 |
+
"normalized": false,
|
| 807 |
+
"rstrip": false,
|
| 808 |
+
"single_word": false,
|
| 809 |
+
"special": true
|
| 810 |
+
},
|
| 811 |
+
"128101": {
|
| 812 |
+
"content": "<|reserved_special_token_96|>",
|
| 813 |
+
"lstrip": false,
|
| 814 |
+
"normalized": false,
|
| 815 |
+
"rstrip": false,
|
| 816 |
+
"single_word": false,
|
| 817 |
+
"special": true
|
| 818 |
+
},
|
| 819 |
+
"128102": {
|
| 820 |
+
"content": "<|reserved_special_token_97|>",
|
| 821 |
+
"lstrip": false,
|
| 822 |
+
"normalized": false,
|
| 823 |
+
"rstrip": false,
|
| 824 |
+
"single_word": false,
|
| 825 |
+
"special": true
|
| 826 |
+
},
|
| 827 |
+
"128103": {
|
| 828 |
+
"content": "<|reserved_special_token_98|>",
|
| 829 |
+
"lstrip": false,
|
| 830 |
+
"normalized": false,
|
| 831 |
+
"rstrip": false,
|
| 832 |
+
"single_word": false,
|
| 833 |
+
"special": true
|
| 834 |
+
},
|
| 835 |
+
"128104": {
|
| 836 |
+
"content": "<|reserved_special_token_99|>",
|
| 837 |
+
"lstrip": false,
|
| 838 |
+
"normalized": false,
|
| 839 |
+
"rstrip": false,
|
| 840 |
+
"single_word": false,
|
| 841 |
+
"special": true
|
| 842 |
+
},
|
| 843 |
+
"128105": {
|
| 844 |
+
"content": "<|reserved_special_token_100|>",
|
| 845 |
+
"lstrip": false,
|
| 846 |
+
"normalized": false,
|
| 847 |
+
"rstrip": false,
|
| 848 |
+
"single_word": false,
|
| 849 |
+
"special": true
|
| 850 |
+
},
|
| 851 |
+
"128106": {
|
| 852 |
+
"content": "<|reserved_special_token_101|>",
|
| 853 |
+
"lstrip": false,
|
| 854 |
+
"normalized": false,
|
| 855 |
+
"rstrip": false,
|
| 856 |
+
"single_word": false,
|
| 857 |
+
"special": true
|
| 858 |
+
},
|
| 859 |
+
"128107": {
|
| 860 |
+
"content": "<|reserved_special_token_102|>",
|
| 861 |
+
"lstrip": false,
|
| 862 |
+
"normalized": false,
|
| 863 |
+
"rstrip": false,
|
| 864 |
+
"single_word": false,
|
| 865 |
+
"special": true
|
| 866 |
+
},
|
| 867 |
+
"128108": {
|
| 868 |
+
"content": "<|reserved_special_token_103|>",
|
| 869 |
+
"lstrip": false,
|
| 870 |
+
"normalized": false,
|
| 871 |
+
"rstrip": false,
|
| 872 |
+
"single_word": false,
|
| 873 |
+
"special": true
|
| 874 |
+
},
|
| 875 |
+
"128109": {
|
| 876 |
+
"content": "<|reserved_special_token_104|>",
|
| 877 |
+
"lstrip": false,
|
| 878 |
+
"normalized": false,
|
| 879 |
+
"rstrip": false,
|
| 880 |
+
"single_word": false,
|
| 881 |
+
"special": true
|
| 882 |
+
},
|
| 883 |
+
"128110": {
|
| 884 |
+
"content": "<|reserved_special_token_105|>",
|
| 885 |
+
"lstrip": false,
|
| 886 |
+
"normalized": false,
|
| 887 |
+
"rstrip": false,
|
| 888 |
+
"single_word": false,
|
| 889 |
+
"special": true
|
| 890 |
+
},
|
| 891 |
+
"128111": {
|
| 892 |
+
"content": "<|reserved_special_token_106|>",
|
| 893 |
+
"lstrip": false,
|
| 894 |
+
"normalized": false,
|
| 895 |
+
"rstrip": false,
|
| 896 |
+
"single_word": false,
|
| 897 |
+
"special": true
|
| 898 |
+
},
|
| 899 |
+
"128112": {
|
| 900 |
+
"content": "<|reserved_special_token_107|>",
|
| 901 |
+
"lstrip": false,
|
| 902 |
+
"normalized": false,
|
| 903 |
+
"rstrip": false,
|
| 904 |
+
"single_word": false,
|
| 905 |
+
"special": true
|
| 906 |
+
},
|
| 907 |
+
"128113": {
|
| 908 |
+
"content": "<|reserved_special_token_108|>",
|
| 909 |
+
"lstrip": false,
|
| 910 |
+
"normalized": false,
|
| 911 |
+
"rstrip": false,
|
| 912 |
+
"single_word": false,
|
| 913 |
+
"special": true
|
| 914 |
+
},
|
| 915 |
+
"128114": {
|
| 916 |
+
"content": "<|reserved_special_token_109|>",
|
| 917 |
+
"lstrip": false,
|
| 918 |
+
"normalized": false,
|
| 919 |
+
"rstrip": false,
|
| 920 |
+
"single_word": false,
|
| 921 |
+
"special": true
|
| 922 |
+
},
|
| 923 |
+
"128115": {
|
| 924 |
+
"content": "<|reserved_special_token_110|>",
|
| 925 |
+
"lstrip": false,
|
| 926 |
+
"normalized": false,
|
| 927 |
+
"rstrip": false,
|
| 928 |
+
"single_word": false,
|
| 929 |
+
"special": true
|
| 930 |
+
},
|
| 931 |
+
"128116": {
|
| 932 |
+
"content": "<|reserved_special_token_111|>",
|
| 933 |
+
"lstrip": false,
|
| 934 |
+
"normalized": false,
|
| 935 |
+
"rstrip": false,
|
| 936 |
+
"single_word": false,
|
| 937 |
+
"special": true
|
| 938 |
+
},
|
| 939 |
+
"128117": {
|
| 940 |
+
"content": "<|reserved_special_token_112|>",
|
| 941 |
+
"lstrip": false,
|
| 942 |
+
"normalized": false,
|
| 943 |
+
"rstrip": false,
|
| 944 |
+
"single_word": false,
|
| 945 |
+
"special": true
|
| 946 |
+
},
|
| 947 |
+
"128118": {
|
| 948 |
+
"content": "<|reserved_special_token_113|>",
|
| 949 |
+
"lstrip": false,
|
| 950 |
+
"normalized": false,
|
| 951 |
+
"rstrip": false,
|
| 952 |
+
"single_word": false,
|
| 953 |
+
"special": true
|
| 954 |
+
},
|
| 955 |
+
"128119": {
|
| 956 |
+
"content": "<|reserved_special_token_114|>",
|
| 957 |
+
"lstrip": false,
|
| 958 |
+
"normalized": false,
|
| 959 |
+
"rstrip": false,
|
| 960 |
+
"single_word": false,
|
| 961 |
+
"special": true
|
| 962 |
+
},
|
| 963 |
+
"128120": {
|
| 964 |
+
"content": "<|reserved_special_token_115|>",
|
| 965 |
+
"lstrip": false,
|
| 966 |
+
"normalized": false,
|
| 967 |
+
"rstrip": false,
|
| 968 |
+
"single_word": false,
|
| 969 |
+
"special": true
|
| 970 |
+
},
|
| 971 |
+
"128121": {
|
| 972 |
+
"content": "<|reserved_special_token_116|>",
|
| 973 |
+
"lstrip": false,
|
| 974 |
+
"normalized": false,
|
| 975 |
+
"rstrip": false,
|
| 976 |
+
"single_word": false,
|
| 977 |
+
"special": true
|
| 978 |
+
},
|
| 979 |
+
"128122": {
|
| 980 |
+
"content": "<|reserved_special_token_117|>",
|
| 981 |
+
"lstrip": false,
|
| 982 |
+
"normalized": false,
|
| 983 |
+
"rstrip": false,
|
| 984 |
+
"single_word": false,
|
| 985 |
+
"special": true
|
| 986 |
+
},
|
| 987 |
+
"128123": {
|
| 988 |
+
"content": "<|reserved_special_token_118|>",
|
| 989 |
+
"lstrip": false,
|
| 990 |
+
"normalized": false,
|
| 991 |
+
"rstrip": false,
|
| 992 |
+
"single_word": false,
|
| 993 |
+
"special": true
|
| 994 |
+
},
|
| 995 |
+
"128124": {
|
| 996 |
+
"content": "<|reserved_special_token_119|>",
|
| 997 |
+
"lstrip": false,
|
| 998 |
+
"normalized": false,
|
| 999 |
+
"rstrip": false,
|
| 1000 |
+
"single_word": false,
|
| 1001 |
+
"special": true
|
| 1002 |
+
},
|
| 1003 |
+
"128125": {
|
| 1004 |
+
"content": "<|reserved_special_token_120|>",
|
| 1005 |
+
"lstrip": false,
|
| 1006 |
+
"normalized": false,
|
| 1007 |
+
"rstrip": false,
|
| 1008 |
+
"single_word": false,
|
| 1009 |
+
"special": true
|
| 1010 |
+
},
|
| 1011 |
+
"128126": {
|
| 1012 |
+
"content": "<|reserved_special_token_121|>",
|
| 1013 |
+
"lstrip": false,
|
| 1014 |
+
"normalized": false,
|
| 1015 |
+
"rstrip": false,
|
| 1016 |
+
"single_word": false,
|
| 1017 |
+
"special": true
|
| 1018 |
+
},
|
| 1019 |
+
"128127": {
|
| 1020 |
+
"content": "<|reserved_special_token_122|>",
|
| 1021 |
+
"lstrip": false,
|
| 1022 |
+
"normalized": false,
|
| 1023 |
+
"rstrip": false,
|
| 1024 |
+
"single_word": false,
|
| 1025 |
+
"special": true
|
| 1026 |
+
},
|
| 1027 |
+
"128128": {
|
| 1028 |
+
"content": "<|reserved_special_token_123|>",
|
| 1029 |
+
"lstrip": false,
|
| 1030 |
+
"normalized": false,
|
| 1031 |
+
"rstrip": false,
|
| 1032 |
+
"single_word": false,
|
| 1033 |
+
"special": true
|
| 1034 |
+
},
|
| 1035 |
+
"128129": {
|
| 1036 |
+
"content": "<|reserved_special_token_124|>",
|
| 1037 |
+
"lstrip": false,
|
| 1038 |
+
"normalized": false,
|
| 1039 |
+
"rstrip": false,
|
| 1040 |
+
"single_word": false,
|
| 1041 |
+
"special": true
|
| 1042 |
+
},
|
| 1043 |
+
"128130": {
|
| 1044 |
+
"content": "<|reserved_special_token_125|>",
|
| 1045 |
+
"lstrip": false,
|
| 1046 |
+
"normalized": false,
|
| 1047 |
+
"rstrip": false,
|
| 1048 |
+
"single_word": false,
|
| 1049 |
+
"special": true
|
| 1050 |
+
},
|
| 1051 |
+
"128131": {
|
| 1052 |
+
"content": "<|reserved_special_token_126|>",
|
| 1053 |
+
"lstrip": false,
|
| 1054 |
+
"normalized": false,
|
| 1055 |
+
"rstrip": false,
|
| 1056 |
+
"single_word": false,
|
| 1057 |
+
"special": true
|
| 1058 |
+
},
|
| 1059 |
+
"128132": {
|
| 1060 |
+
"content": "<|reserved_special_token_127|>",
|
| 1061 |
+
"lstrip": false,
|
| 1062 |
+
"normalized": false,
|
| 1063 |
+
"rstrip": false,
|
| 1064 |
+
"single_word": false,
|
| 1065 |
+
"special": true
|
| 1066 |
+
},
|
| 1067 |
+
"128133": {
|
| 1068 |
+
"content": "<|reserved_special_token_128|>",
|
| 1069 |
+
"lstrip": false,
|
| 1070 |
+
"normalized": false,
|
| 1071 |
+
"rstrip": false,
|
| 1072 |
+
"single_word": false,
|
| 1073 |
+
"special": true
|
| 1074 |
+
},
|
| 1075 |
+
"128134": {
|
| 1076 |
+
"content": "<|reserved_special_token_129|>",
|
| 1077 |
+
"lstrip": false,
|
| 1078 |
+
"normalized": false,
|
| 1079 |
+
"rstrip": false,
|
| 1080 |
+
"single_word": false,
|
| 1081 |
+
"special": true
|
| 1082 |
+
},
|
| 1083 |
+
"128135": {
|
| 1084 |
+
"content": "<|reserved_special_token_130|>",
|
| 1085 |
+
"lstrip": false,
|
| 1086 |
+
"normalized": false,
|
| 1087 |
+
"rstrip": false,
|
| 1088 |
+
"single_word": false,
|
| 1089 |
+
"special": true
|
| 1090 |
+
},
|
| 1091 |
+
"128136": {
|
| 1092 |
+
"content": "<|reserved_special_token_131|>",
|
| 1093 |
+
"lstrip": false,
|
| 1094 |
+
"normalized": false,
|
| 1095 |
+
"rstrip": false,
|
| 1096 |
+
"single_word": false,
|
| 1097 |
+
"special": true
|
| 1098 |
+
},
|
| 1099 |
+
"128137": {
|
| 1100 |
+
"content": "<|reserved_special_token_132|>",
|
| 1101 |
+
"lstrip": false,
|
| 1102 |
+
"normalized": false,
|
| 1103 |
+
"rstrip": false,
|
| 1104 |
+
"single_word": false,
|
| 1105 |
+
"special": true
|
| 1106 |
+
},
|
| 1107 |
+
"128138": {
|
| 1108 |
+
"content": "<|reserved_special_token_133|>",
|
| 1109 |
+
"lstrip": false,
|
| 1110 |
+
"normalized": false,
|
| 1111 |
+
"rstrip": false,
|
| 1112 |
+
"single_word": false,
|
| 1113 |
+
"special": true
|
| 1114 |
+
},
|
| 1115 |
+
"128139": {
|
| 1116 |
+
"content": "<|reserved_special_token_134|>",
|
| 1117 |
+
"lstrip": false,
|
| 1118 |
+
"normalized": false,
|
| 1119 |
+
"rstrip": false,
|
| 1120 |
+
"single_word": false,
|
| 1121 |
+
"special": true
|
| 1122 |
+
},
|
| 1123 |
+
"128140": {
|
| 1124 |
+
"content": "<|reserved_special_token_135|>",
|
| 1125 |
+
"lstrip": false,
|
| 1126 |
+
"normalized": false,
|
| 1127 |
+
"rstrip": false,
|
| 1128 |
+
"single_word": false,
|
| 1129 |
+
"special": true
|
| 1130 |
+
},
|
| 1131 |
+
"128141": {
|
| 1132 |
+
"content": "<|reserved_special_token_136|>",
|
| 1133 |
+
"lstrip": false,
|
| 1134 |
+
"normalized": false,
|
| 1135 |
+
"rstrip": false,
|
| 1136 |
+
"single_word": false,
|
| 1137 |
+
"special": true
|
| 1138 |
+
},
|
| 1139 |
+
"128142": {
|
| 1140 |
+
"content": "<|reserved_special_token_137|>",
|
| 1141 |
+
"lstrip": false,
|
| 1142 |
+
"normalized": false,
|
| 1143 |
+
"rstrip": false,
|
| 1144 |
+
"single_word": false,
|
| 1145 |
+
"special": true
|
| 1146 |
+
},
|
| 1147 |
+
"128143": {
|
| 1148 |
+
"content": "<|reserved_special_token_138|>",
|
| 1149 |
+
"lstrip": false,
|
| 1150 |
+
"normalized": false,
|
| 1151 |
+
"rstrip": false,
|
| 1152 |
+
"single_word": false,
|
| 1153 |
+
"special": true
|
| 1154 |
+
},
|
| 1155 |
+
"128144": {
|
| 1156 |
+
"content": "<|reserved_special_token_139|>",
|
| 1157 |
+
"lstrip": false,
|
| 1158 |
+
"normalized": false,
|
| 1159 |
+
"rstrip": false,
|
| 1160 |
+
"single_word": false,
|
| 1161 |
+
"special": true
|
| 1162 |
+
},
|
| 1163 |
+
"128145": {
|
| 1164 |
+
"content": "<|reserved_special_token_140|>",
|
| 1165 |
+
"lstrip": false,
|
| 1166 |
+
"normalized": false,
|
| 1167 |
+
"rstrip": false,
|
| 1168 |
+
"single_word": false,
|
| 1169 |
+
"special": true
|
| 1170 |
+
},
|
| 1171 |
+
"128146": {
|
| 1172 |
+
"content": "<|reserved_special_token_141|>",
|
| 1173 |
+
"lstrip": false,
|
| 1174 |
+
"normalized": false,
|
| 1175 |
+
"rstrip": false,
|
| 1176 |
+
"single_word": false,
|
| 1177 |
+
"special": true
|
| 1178 |
+
},
|
| 1179 |
+
"128147": {
|
| 1180 |
+
"content": "<|reserved_special_token_142|>",
|
| 1181 |
+
"lstrip": false,
|
| 1182 |
+
"normalized": false,
|
| 1183 |
+
"rstrip": false,
|
| 1184 |
+
"single_word": false,
|
| 1185 |
+
"special": true
|
| 1186 |
+
},
|
| 1187 |
+
"128148": {
|
| 1188 |
+
"content": "<|reserved_special_token_143|>",
|
| 1189 |
+
"lstrip": false,
|
| 1190 |
+
"normalized": false,
|
| 1191 |
+
"rstrip": false,
|
| 1192 |
+
"single_word": false,
|
| 1193 |
+
"special": true
|
| 1194 |
+
},
|
| 1195 |
+
"128149": {
|
| 1196 |
+
"content": "<|reserved_special_token_144|>",
|
| 1197 |
+
"lstrip": false,
|
| 1198 |
+
"normalized": false,
|
| 1199 |
+
"rstrip": false,
|
| 1200 |
+
"single_word": false,
|
| 1201 |
+
"special": true
|
| 1202 |
+
},
|
| 1203 |
+
"128150": {
|
| 1204 |
+
"content": "<|reserved_special_token_145|>",
|
| 1205 |
+
"lstrip": false,
|
| 1206 |
+
"normalized": false,
|
| 1207 |
+
"rstrip": false,
|
| 1208 |
+
"single_word": false,
|
| 1209 |
+
"special": true
|
| 1210 |
+
},
|
| 1211 |
+
"128151": {
|
| 1212 |
+
"content": "<|reserved_special_token_146|>",
|
| 1213 |
+
"lstrip": false,
|
| 1214 |
+
"normalized": false,
|
| 1215 |
+
"rstrip": false,
|
| 1216 |
+
"single_word": false,
|
| 1217 |
+
"special": true
|
| 1218 |
+
},
|
| 1219 |
+
"128152": {
|
| 1220 |
+
"content": "<|reserved_special_token_147|>",
|
| 1221 |
+
"lstrip": false,
|
| 1222 |
+
"normalized": false,
|
| 1223 |
+
"rstrip": false,
|
| 1224 |
+
"single_word": false,
|
| 1225 |
+
"special": true
|
| 1226 |
+
},
|
| 1227 |
+
"128153": {
|
| 1228 |
+
"content": "<|reserved_special_token_148|>",
|
| 1229 |
+
"lstrip": false,
|
| 1230 |
+
"normalized": false,
|
| 1231 |
+
"rstrip": false,
|
| 1232 |
+
"single_word": false,
|
| 1233 |
+
"special": true
|
| 1234 |
+
},
|
| 1235 |
+
"128154": {
|
| 1236 |
+
"content": "<|reserved_special_token_149|>",
|
| 1237 |
+
"lstrip": false,
|
| 1238 |
+
"normalized": false,
|
| 1239 |
+
"rstrip": false,
|
| 1240 |
+
"single_word": false,
|
| 1241 |
+
"special": true
|
| 1242 |
+
},
|
| 1243 |
+
"128155": {
|
| 1244 |
+
"content": "<|reserved_special_token_150|>",
|
| 1245 |
+
"lstrip": false,
|
| 1246 |
+
"normalized": false,
|
| 1247 |
+
"rstrip": false,
|
| 1248 |
+
"single_word": false,
|
| 1249 |
+
"special": true
|
| 1250 |
+
},
|
| 1251 |
+
"128156": {
|
| 1252 |
+
"content": "<|reserved_special_token_151|>",
|
| 1253 |
+
"lstrip": false,
|
| 1254 |
+
"normalized": false,
|
| 1255 |
+
"rstrip": false,
|
| 1256 |
+
"single_word": false,
|
| 1257 |
+
"special": true
|
| 1258 |
+
},
|
| 1259 |
+
"128157": {
|
| 1260 |
+
"content": "<|reserved_special_token_152|>",
|
| 1261 |
+
"lstrip": false,
|
| 1262 |
+
"normalized": false,
|
| 1263 |
+
"rstrip": false,
|
| 1264 |
+
"single_word": false,
|
| 1265 |
+
"special": true
|
| 1266 |
+
},
|
| 1267 |
+
"128158": {
|
| 1268 |
+
"content": "<|reserved_special_token_153|>",
|
| 1269 |
+
"lstrip": false,
|
| 1270 |
+
"normalized": false,
|
| 1271 |
+
"rstrip": false,
|
| 1272 |
+
"single_word": false,
|
| 1273 |
+
"special": true
|
| 1274 |
+
},
|
| 1275 |
+
"128159": {
|
| 1276 |
+
"content": "<|reserved_special_token_154|>",
|
| 1277 |
+
"lstrip": false,
|
| 1278 |
+
"normalized": false,
|
| 1279 |
+
"rstrip": false,
|
| 1280 |
+
"single_word": false,
|
| 1281 |
+
"special": true
|
| 1282 |
+
},
|
| 1283 |
+
"128160": {
|
| 1284 |
+
"content": "<|reserved_special_token_155|>",
|
| 1285 |
+
"lstrip": false,
|
| 1286 |
+
"normalized": false,
|
| 1287 |
+
"rstrip": false,
|
| 1288 |
+
"single_word": false,
|
| 1289 |
+
"special": true
|
| 1290 |
+
},
|
| 1291 |
+
"128161": {
|
| 1292 |
+
"content": "<|reserved_special_token_156|>",
|
| 1293 |
+
"lstrip": false,
|
| 1294 |
+
"normalized": false,
|
| 1295 |
+
"rstrip": false,
|
| 1296 |
+
"single_word": false,
|
| 1297 |
+
"special": true
|
| 1298 |
+
},
|
| 1299 |
+
"128162": {
|
| 1300 |
+
"content": "<|reserved_special_token_157|>",
|
| 1301 |
+
"lstrip": false,
|
| 1302 |
+
"normalized": false,
|
| 1303 |
+
"rstrip": false,
|
| 1304 |
+
"single_word": false,
|
| 1305 |
+
"special": true
|
| 1306 |
+
},
|
| 1307 |
+
"128163": {
|
| 1308 |
+
"content": "<|reserved_special_token_158|>",
|
| 1309 |
+
"lstrip": false,
|
| 1310 |
+
"normalized": false,
|
| 1311 |
+
"rstrip": false,
|
| 1312 |
+
"single_word": false,
|
| 1313 |
+
"special": true
|
| 1314 |
+
},
|
| 1315 |
+
"128164": {
|
| 1316 |
+
"content": "<|reserved_special_token_159|>",
|
| 1317 |
+
"lstrip": false,
|
| 1318 |
+
"normalized": false,
|
| 1319 |
+
"rstrip": false,
|
| 1320 |
+
"single_word": false,
|
| 1321 |
+
"special": true
|
| 1322 |
+
},
|
| 1323 |
+
"128165": {
|
| 1324 |
+
"content": "<|reserved_special_token_160|>",
|
| 1325 |
+
"lstrip": false,
|
| 1326 |
+
"normalized": false,
|
| 1327 |
+
"rstrip": false,
|
| 1328 |
+
"single_word": false,
|
| 1329 |
+
"special": true
|
| 1330 |
+
},
|
| 1331 |
+
"128166": {
|
| 1332 |
+
"content": "<|reserved_special_token_161|>",
|
| 1333 |
+
"lstrip": false,
|
| 1334 |
+
"normalized": false,
|
| 1335 |
+
"rstrip": false,
|
| 1336 |
+
"single_word": false,
|
| 1337 |
+
"special": true
|
| 1338 |
+
},
|
| 1339 |
+
"128167": {
|
| 1340 |
+
"content": "<|reserved_special_token_162|>",
|
| 1341 |
+
"lstrip": false,
|
| 1342 |
+
"normalized": false,
|
| 1343 |
+
"rstrip": false,
|
| 1344 |
+
"single_word": false,
|
| 1345 |
+
"special": true
|
| 1346 |
+
},
|
| 1347 |
+
"128168": {
|
| 1348 |
+
"content": "<|reserved_special_token_163|>",
|
| 1349 |
+
"lstrip": false,
|
| 1350 |
+
"normalized": false,
|
| 1351 |
+
"rstrip": false,
|
| 1352 |
+
"single_word": false,
|
| 1353 |
+
"special": true
|
| 1354 |
+
},
|
| 1355 |
+
"128169": {
|
| 1356 |
+
"content": "<|reserved_special_token_164|>",
|
| 1357 |
+
"lstrip": false,
|
| 1358 |
+
"normalized": false,
|
| 1359 |
+
"rstrip": false,
|
| 1360 |
+
"single_word": false,
|
| 1361 |
+
"special": true
|
| 1362 |
+
},
|
| 1363 |
+
"128170": {
|
| 1364 |
+
"content": "<|reserved_special_token_165|>",
|
| 1365 |
+
"lstrip": false,
|
| 1366 |
+
"normalized": false,
|
| 1367 |
+
"rstrip": false,
|
| 1368 |
+
"single_word": false,
|
| 1369 |
+
"special": true
|
| 1370 |
+
},
|
| 1371 |
+
"128171": {
|
| 1372 |
+
"content": "<|reserved_special_token_166|>",
|
| 1373 |
+
"lstrip": false,
|
| 1374 |
+
"normalized": false,
|
| 1375 |
+
"rstrip": false,
|
| 1376 |
+
"single_word": false,
|
| 1377 |
+
"special": true
|
| 1378 |
+
},
|
| 1379 |
+
"128172": {
|
| 1380 |
+
"content": "<|reserved_special_token_167|>",
|
| 1381 |
+
"lstrip": false,
|
| 1382 |
+
"normalized": false,
|
| 1383 |
+
"rstrip": false,
|
| 1384 |
+
"single_word": false,
|
| 1385 |
+
"special": true
|
| 1386 |
+
},
|
| 1387 |
+
"128173": {
|
| 1388 |
+
"content": "<|reserved_special_token_168|>",
|
| 1389 |
+
"lstrip": false,
|
| 1390 |
+
"normalized": false,
|
| 1391 |
+
"rstrip": false,
|
| 1392 |
+
"single_word": false,
|
| 1393 |
+
"special": true
|
| 1394 |
+
},
|
| 1395 |
+
"128174": {
|
| 1396 |
+
"content": "<|reserved_special_token_169|>",
|
| 1397 |
+
"lstrip": false,
|
| 1398 |
+
"normalized": false,
|
| 1399 |
+
"rstrip": false,
|
| 1400 |
+
"single_word": false,
|
| 1401 |
+
"special": true
|
| 1402 |
+
},
|
| 1403 |
+
"128175": {
|
| 1404 |
+
"content": "<|reserved_special_token_170|>",
|
| 1405 |
+
"lstrip": false,
|
| 1406 |
+
"normalized": false,
|
| 1407 |
+
"rstrip": false,
|
| 1408 |
+
"single_word": false,
|
| 1409 |
+
"special": true
|
| 1410 |
+
},
|
| 1411 |
+
"128176": {
|
| 1412 |
+
"content": "<|reserved_special_token_171|>",
|
| 1413 |
+
"lstrip": false,
|
| 1414 |
+
"normalized": false,
|
| 1415 |
+
"rstrip": false,
|
| 1416 |
+
"single_word": false,
|
| 1417 |
+
"special": true
|
| 1418 |
+
},
|
| 1419 |
+
"128177": {
|
| 1420 |
+
"content": "<|reserved_special_token_172|>",
|
| 1421 |
+
"lstrip": false,
|
| 1422 |
+
"normalized": false,
|
| 1423 |
+
"rstrip": false,
|
| 1424 |
+
"single_word": false,
|
| 1425 |
+
"special": true
|
| 1426 |
+
},
|
| 1427 |
+
"128178": {
|
| 1428 |
+
"content": "<|reserved_special_token_173|>",
|
| 1429 |
+
"lstrip": false,
|
| 1430 |
+
"normalized": false,
|
| 1431 |
+
"rstrip": false,
|
| 1432 |
+
"single_word": false,
|
| 1433 |
+
"special": true
|
| 1434 |
+
},
|
| 1435 |
+
"128179": {
|
| 1436 |
+
"content": "<|reserved_special_token_174|>",
|
| 1437 |
+
"lstrip": false,
|
| 1438 |
+
"normalized": false,
|
| 1439 |
+
"rstrip": false,
|
| 1440 |
+
"single_word": false,
|
| 1441 |
+
"special": true
|
| 1442 |
+
},
|
| 1443 |
+
"128180": {
|
| 1444 |
+
"content": "<|reserved_special_token_175|>",
|
| 1445 |
+
"lstrip": false,
|
| 1446 |
+
"normalized": false,
|
| 1447 |
+
"rstrip": false,
|
| 1448 |
+
"single_word": false,
|
| 1449 |
+
"special": true
|
| 1450 |
+
},
|
| 1451 |
+
"128181": {
|
| 1452 |
+
"content": "<|reserved_special_token_176|>",
|
| 1453 |
+
"lstrip": false,
|
| 1454 |
+
"normalized": false,
|
| 1455 |
+
"rstrip": false,
|
| 1456 |
+
"single_word": false,
|
| 1457 |
+
"special": true
|
| 1458 |
+
},
|
| 1459 |
+
"128182": {
|
| 1460 |
+
"content": "<|reserved_special_token_177|>",
|
| 1461 |
+
"lstrip": false,
|
| 1462 |
+
"normalized": false,
|
| 1463 |
+
"rstrip": false,
|
| 1464 |
+
"single_word": false,
|
| 1465 |
+
"special": true
|
| 1466 |
+
},
|
| 1467 |
+
"128183": {
|
| 1468 |
+
"content": "<|reserved_special_token_178|>",
|
| 1469 |
+
"lstrip": false,
|
| 1470 |
+
"normalized": false,
|
| 1471 |
+
"rstrip": false,
|
| 1472 |
+
"single_word": false,
|
| 1473 |
+
"special": true
|
| 1474 |
+
},
|
| 1475 |
+
"128184": {
|
| 1476 |
+
"content": "<|reserved_special_token_179|>",
|
| 1477 |
+
"lstrip": false,
|
| 1478 |
+
"normalized": false,
|
| 1479 |
+
"rstrip": false,
|
| 1480 |
+
"single_word": false,
|
| 1481 |
+
"special": true
|
| 1482 |
+
},
|
| 1483 |
+
"128185": {
|
| 1484 |
+
"content": "<|reserved_special_token_180|>",
|
| 1485 |
+
"lstrip": false,
|
| 1486 |
+
"normalized": false,
|
| 1487 |
+
"rstrip": false,
|
| 1488 |
+
"single_word": false,
|
| 1489 |
+
"special": true
|
| 1490 |
+
},
|
| 1491 |
+
"128186": {
|
| 1492 |
+
"content": "<|reserved_special_token_181|>",
|
| 1493 |
+
"lstrip": false,
|
| 1494 |
+
"normalized": false,
|
| 1495 |
+
"rstrip": false,
|
| 1496 |
+
"single_word": false,
|
| 1497 |
+
"special": true
|
| 1498 |
+
},
|
| 1499 |
+
"128187": {
|
| 1500 |
+
"content": "<|reserved_special_token_182|>",
|
| 1501 |
+
"lstrip": false,
|
| 1502 |
+
"normalized": false,
|
| 1503 |
+
"rstrip": false,
|
| 1504 |
+
"single_word": false,
|
| 1505 |
+
"special": true
|
| 1506 |
+
},
|
| 1507 |
+
"128188": {
|
| 1508 |
+
"content": "<|reserved_special_token_183|>",
|
| 1509 |
+
"lstrip": false,
|
| 1510 |
+
"normalized": false,
|
| 1511 |
+
"rstrip": false,
|
| 1512 |
+
"single_word": false,
|
| 1513 |
+
"special": true
|
| 1514 |
+
},
|
| 1515 |
+
"128189": {
|
| 1516 |
+
"content": "<|reserved_special_token_184|>",
|
| 1517 |
+
"lstrip": false,
|
| 1518 |
+
"normalized": false,
|
| 1519 |
+
"rstrip": false,
|
| 1520 |
+
"single_word": false,
|
| 1521 |
+
"special": true
|
| 1522 |
+
},
|
| 1523 |
+
"128190": {
|
| 1524 |
+
"content": "<|reserved_special_token_185|>",
|
| 1525 |
+
"lstrip": false,
|
| 1526 |
+
"normalized": false,
|
| 1527 |
+
"rstrip": false,
|
| 1528 |
+
"single_word": false,
|
| 1529 |
+
"special": true
|
| 1530 |
+
},
|
| 1531 |
+
"128191": {
|
| 1532 |
+
"content": "<|reserved_special_token_186|>",
|
| 1533 |
+
"lstrip": false,
|
| 1534 |
+
"normalized": false,
|
| 1535 |
+
"rstrip": false,
|
| 1536 |
+
"single_word": false,
|
| 1537 |
+
"special": true
|
| 1538 |
+
},
|
| 1539 |
+
"128192": {
|
| 1540 |
+
"content": "<|reserved_special_token_187|>",
|
| 1541 |
+
"lstrip": false,
|
| 1542 |
+
"normalized": false,
|
| 1543 |
+
"rstrip": false,
|
| 1544 |
+
"single_word": false,
|
| 1545 |
+
"special": true
|
| 1546 |
+
},
|
| 1547 |
+
"128193": {
|
| 1548 |
+
"content": "<|reserved_special_token_188|>",
|
| 1549 |
+
"lstrip": false,
|
| 1550 |
+
"normalized": false,
|
| 1551 |
+
"rstrip": false,
|
| 1552 |
+
"single_word": false,
|
| 1553 |
+
"special": true
|
| 1554 |
+
},
|
| 1555 |
+
"128194": {
|
| 1556 |
+
"content": "<|reserved_special_token_189|>",
|
| 1557 |
+
"lstrip": false,
|
| 1558 |
+
"normalized": false,
|
| 1559 |
+
"rstrip": false,
|
| 1560 |
+
"single_word": false,
|
| 1561 |
+
"special": true
|
| 1562 |
+
},
|
| 1563 |
+
"128195": {
|
| 1564 |
+
"content": "<|reserved_special_token_190|>",
|
| 1565 |
+
"lstrip": false,
|
| 1566 |
+
"normalized": false,
|
| 1567 |
+
"rstrip": false,
|
| 1568 |
+
"single_word": false,
|
| 1569 |
+
"special": true
|
| 1570 |
+
},
|
| 1571 |
+
"128196": {
|
| 1572 |
+
"content": "<|reserved_special_token_191|>",
|
| 1573 |
+
"lstrip": false,
|
| 1574 |
+
"normalized": false,
|
| 1575 |
+
"rstrip": false,
|
| 1576 |
+
"single_word": false,
|
| 1577 |
+
"special": true
|
| 1578 |
+
},
|
| 1579 |
+
"128197": {
|
| 1580 |
+
"content": "<|reserved_special_token_192|>",
|
| 1581 |
+
"lstrip": false,
|
| 1582 |
+
"normalized": false,
|
| 1583 |
+
"rstrip": false,
|
| 1584 |
+
"single_word": false,
|
| 1585 |
+
"special": true
|
| 1586 |
+
},
|
| 1587 |
+
"128198": {
|
| 1588 |
+
"content": "<|reserved_special_token_193|>",
|
| 1589 |
+
"lstrip": false,
|
| 1590 |
+
"normalized": false,
|
| 1591 |
+
"rstrip": false,
|
| 1592 |
+
"single_word": false,
|
| 1593 |
+
"special": true
|
| 1594 |
+
},
|
| 1595 |
+
"128199": {
|
| 1596 |
+
"content": "<|reserved_special_token_194|>",
|
| 1597 |
+
"lstrip": false,
|
| 1598 |
+
"normalized": false,
|
| 1599 |
+
"rstrip": false,
|
| 1600 |
+
"single_word": false,
|
| 1601 |
+
"special": true
|
| 1602 |
+
},
|
| 1603 |
+
"128200": {
|
| 1604 |
+
"content": "<|reserved_special_token_195|>",
|
| 1605 |
+
"lstrip": false,
|
| 1606 |
+
"normalized": false,
|
| 1607 |
+
"rstrip": false,
|
| 1608 |
+
"single_word": false,
|
| 1609 |
+
"special": true
|
| 1610 |
+
},
|
| 1611 |
+
"128201": {
|
| 1612 |
+
"content": "<|reserved_special_token_196|>",
|
| 1613 |
+
"lstrip": false,
|
| 1614 |
+
"normalized": false,
|
| 1615 |
+
"rstrip": false,
|
| 1616 |
+
"single_word": false,
|
| 1617 |
+
"special": true
|
| 1618 |
+
},
|
| 1619 |
+
"128202": {
|
| 1620 |
+
"content": "<|reserved_special_token_197|>",
|
| 1621 |
+
"lstrip": false,
|
| 1622 |
+
"normalized": false,
|
| 1623 |
+
"rstrip": false,
|
| 1624 |
+
"single_word": false,
|
| 1625 |
+
"special": true
|
| 1626 |
+
},
|
| 1627 |
+
"128203": {
|
| 1628 |
+
"content": "<|reserved_special_token_198|>",
|
| 1629 |
+
"lstrip": false,
|
| 1630 |
+
"normalized": false,
|
| 1631 |
+
"rstrip": false,
|
| 1632 |
+
"single_word": false,
|
| 1633 |
+
"special": true
|
| 1634 |
+
},
|
| 1635 |
+
"128204": {
|
| 1636 |
+
"content": "<|reserved_special_token_199|>",
|
| 1637 |
+
"lstrip": false,
|
| 1638 |
+
"normalized": false,
|
| 1639 |
+
"rstrip": false,
|
| 1640 |
+
"single_word": false,
|
| 1641 |
+
"special": true
|
| 1642 |
+
},
|
| 1643 |
+
"128205": {
|
| 1644 |
+
"content": "<|reserved_special_token_200|>",
|
| 1645 |
+
"lstrip": false,
|
| 1646 |
+
"normalized": false,
|
| 1647 |
+
"rstrip": false,
|
| 1648 |
+
"single_word": false,
|
| 1649 |
+
"special": true
|
| 1650 |
+
},
|
| 1651 |
+
"128206": {
|
| 1652 |
+
"content": "<|reserved_special_token_201|>",
|
| 1653 |
+
"lstrip": false,
|
| 1654 |
+
"normalized": false,
|
| 1655 |
+
"rstrip": false,
|
| 1656 |
+
"single_word": false,
|
| 1657 |
+
"special": true
|
| 1658 |
+
},
|
| 1659 |
+
"128207": {
|
| 1660 |
+
"content": "<|reserved_special_token_202|>",
|
| 1661 |
+
"lstrip": false,
|
| 1662 |
+
"normalized": false,
|
| 1663 |
+
"rstrip": false,
|
| 1664 |
+
"single_word": false,
|
| 1665 |
+
"special": true
|
| 1666 |
+
},
|
| 1667 |
+
"128208": {
|
| 1668 |
+
"content": "<|reserved_special_token_203|>",
|
| 1669 |
+
"lstrip": false,
|
| 1670 |
+
"normalized": false,
|
| 1671 |
+
"rstrip": false,
|
| 1672 |
+
"single_word": false,
|
| 1673 |
+
"special": true
|
| 1674 |
+
},
|
| 1675 |
+
"128209": {
|
| 1676 |
+
"content": "<|reserved_special_token_204|>",
|
| 1677 |
+
"lstrip": false,
|
| 1678 |
+
"normalized": false,
|
| 1679 |
+
"rstrip": false,
|
| 1680 |
+
"single_word": false,
|
| 1681 |
+
"special": true
|
| 1682 |
+
},
|
| 1683 |
+
"128210": {
|
| 1684 |
+
"content": "<|reserved_special_token_205|>",
|
| 1685 |
+
"lstrip": false,
|
| 1686 |
+
"normalized": false,
|
| 1687 |
+
"rstrip": false,
|
| 1688 |
+
"single_word": false,
|
| 1689 |
+
"special": true
|
| 1690 |
+
},
|
| 1691 |
+
"128211": {
|
| 1692 |
+
"content": "<|reserved_special_token_206|>",
|
| 1693 |
+
"lstrip": false,
|
| 1694 |
+
"normalized": false,
|
| 1695 |
+
"rstrip": false,
|
| 1696 |
+
"single_word": false,
|
| 1697 |
+
"special": true
|
| 1698 |
+
},
|
| 1699 |
+
"128212": {
|
| 1700 |
+
"content": "<|reserved_special_token_207|>",
|
| 1701 |
+
"lstrip": false,
|
| 1702 |
+
"normalized": false,
|
| 1703 |
+
"rstrip": false,
|
| 1704 |
+
"single_word": false,
|
| 1705 |
+
"special": true
|
| 1706 |
+
},
|
| 1707 |
+
"128213": {
|
| 1708 |
+
"content": "<|reserved_special_token_208|>",
|
| 1709 |
+
"lstrip": false,
|
| 1710 |
+
"normalized": false,
|
| 1711 |
+
"rstrip": false,
|
| 1712 |
+
"single_word": false,
|
| 1713 |
+
"special": true
|
| 1714 |
+
},
|
| 1715 |
+
"128214": {
|
| 1716 |
+
"content": "<|reserved_special_token_209|>",
|
| 1717 |
+
"lstrip": false,
|
| 1718 |
+
"normalized": false,
|
| 1719 |
+
"rstrip": false,
|
| 1720 |
+
"single_word": false,
|
| 1721 |
+
"special": true
|
| 1722 |
+
},
|
| 1723 |
+
"128215": {
|
| 1724 |
+
"content": "<|reserved_special_token_210|>",
|
| 1725 |
+
"lstrip": false,
|
| 1726 |
+
"normalized": false,
|
| 1727 |
+
"rstrip": false,
|
| 1728 |
+
"single_word": false,
|
| 1729 |
+
"special": true
|
| 1730 |
+
},
|
| 1731 |
+
"128216": {
|
| 1732 |
+
"content": "<|reserved_special_token_211|>",
|
| 1733 |
+
"lstrip": false,
|
| 1734 |
+
"normalized": false,
|
| 1735 |
+
"rstrip": false,
|
| 1736 |
+
"single_word": false,
|
| 1737 |
+
"special": true
|
| 1738 |
+
},
|
| 1739 |
+
"128217": {
|
| 1740 |
+
"content": "<|reserved_special_token_212|>",
|
| 1741 |
+
"lstrip": false,
|
| 1742 |
+
"normalized": false,
|
| 1743 |
+
"rstrip": false,
|
| 1744 |
+
"single_word": false,
|
| 1745 |
+
"special": true
|
| 1746 |
+
},
|
| 1747 |
+
"128218": {
|
| 1748 |
+
"content": "<|reserved_special_token_213|>",
|
| 1749 |
+
"lstrip": false,
|
| 1750 |
+
"normalized": false,
|
| 1751 |
+
"rstrip": false,
|
| 1752 |
+
"single_word": false,
|
| 1753 |
+
"special": true
|
| 1754 |
+
},
|
| 1755 |
+
"128219": {
|
| 1756 |
+
"content": "<|reserved_special_token_214|>",
|
| 1757 |
+
"lstrip": false,
|
| 1758 |
+
"normalized": false,
|
| 1759 |
+
"rstrip": false,
|
| 1760 |
+
"single_word": false,
|
| 1761 |
+
"special": true
|
| 1762 |
+
},
|
| 1763 |
+
"128220": {
|
| 1764 |
+
"content": "<|reserved_special_token_215|>",
|
| 1765 |
+
"lstrip": false,
|
| 1766 |
+
"normalized": false,
|
| 1767 |
+
"rstrip": false,
|
| 1768 |
+
"single_word": false,
|
| 1769 |
+
"special": true
|
| 1770 |
+
},
|
| 1771 |
+
"128221": {
|
| 1772 |
+
"content": "<|reserved_special_token_216|>",
|
| 1773 |
+
"lstrip": false,
|
| 1774 |
+
"normalized": false,
|
| 1775 |
+
"rstrip": false,
|
| 1776 |
+
"single_word": false,
|
| 1777 |
+
"special": true
|
| 1778 |
+
},
|
| 1779 |
+
"128222": {
|
| 1780 |
+
"content": "<|reserved_special_token_217|>",
|
| 1781 |
+
"lstrip": false,
|
| 1782 |
+
"normalized": false,
|
| 1783 |
+
"rstrip": false,
|
| 1784 |
+
"single_word": false,
|
| 1785 |
+
"special": true
|
| 1786 |
+
},
|
| 1787 |
+
"128223": {
|
| 1788 |
+
"content": "<|reserved_special_token_218|>",
|
| 1789 |
+
"lstrip": false,
|
| 1790 |
+
"normalized": false,
|
| 1791 |
+
"rstrip": false,
|
| 1792 |
+
"single_word": false,
|
| 1793 |
+
"special": true
|
| 1794 |
+
},
|
| 1795 |
+
"128224": {
|
| 1796 |
+
"content": "<|reserved_special_token_219|>",
|
| 1797 |
+
"lstrip": false,
|
| 1798 |
+
"normalized": false,
|
| 1799 |
+
"rstrip": false,
|
| 1800 |
+
"single_word": false,
|
| 1801 |
+
"special": true
|
| 1802 |
+
},
|
| 1803 |
+
"128225": {
|
| 1804 |
+
"content": "<|reserved_special_token_220|>",
|
| 1805 |
+
"lstrip": false,
|
| 1806 |
+
"normalized": false,
|
| 1807 |
+
"rstrip": false,
|
| 1808 |
+
"single_word": false,
|
| 1809 |
+
"special": true
|
| 1810 |
+
},
|
| 1811 |
+
"128226": {
|
| 1812 |
+
"content": "<|reserved_special_token_221|>",
|
| 1813 |
+
"lstrip": false,
|
| 1814 |
+
"normalized": false,
|
| 1815 |
+
"rstrip": false,
|
| 1816 |
+
"single_word": false,
|
| 1817 |
+
"special": true
|
| 1818 |
+
},
|
| 1819 |
+
"128227": {
|
| 1820 |
+
"content": "<|reserved_special_token_222|>",
|
| 1821 |
+
"lstrip": false,
|
| 1822 |
+
"normalized": false,
|
| 1823 |
+
"rstrip": false,
|
| 1824 |
+
"single_word": false,
|
| 1825 |
+
"special": true
|
| 1826 |
+
},
|
| 1827 |
+
"128228": {
|
| 1828 |
+
"content": "<|reserved_special_token_223|>",
|
| 1829 |
+
"lstrip": false,
|
| 1830 |
+
"normalized": false,
|
| 1831 |
+
"rstrip": false,
|
| 1832 |
+
"single_word": false,
|
| 1833 |
+
"special": true
|
| 1834 |
+
},
|
| 1835 |
+
"128229": {
|
| 1836 |
+
"content": "<|reserved_special_token_224|>",
|
| 1837 |
+
"lstrip": false,
|
| 1838 |
+
"normalized": false,
|
| 1839 |
+
"rstrip": false,
|
| 1840 |
+
"single_word": false,
|
| 1841 |
+
"special": true
|
| 1842 |
+
},
|
| 1843 |
+
"128230": {
|
| 1844 |
+
"content": "<|reserved_special_token_225|>",
|
| 1845 |
+
"lstrip": false,
|
| 1846 |
+
"normalized": false,
|
| 1847 |
+
"rstrip": false,
|
| 1848 |
+
"single_word": false,
|
| 1849 |
+
"special": true
|
| 1850 |
+
},
|
| 1851 |
+
"128231": {
|
| 1852 |
+
"content": "<|reserved_special_token_226|>",
|
| 1853 |
+
"lstrip": false,
|
| 1854 |
+
"normalized": false,
|
| 1855 |
+
"rstrip": false,
|
| 1856 |
+
"single_word": false,
|
| 1857 |
+
"special": true
|
| 1858 |
+
},
|
| 1859 |
+
"128232": {
|
| 1860 |
+
"content": "<|reserved_special_token_227|>",
|
| 1861 |
+
"lstrip": false,
|
| 1862 |
+
"normalized": false,
|
| 1863 |
+
"rstrip": false,
|
| 1864 |
+
"single_word": false,
|
| 1865 |
+
"special": true
|
| 1866 |
+
},
|
| 1867 |
+
"128233": {
|
| 1868 |
+
"content": "<|reserved_special_token_228|>",
|
| 1869 |
+
"lstrip": false,
|
| 1870 |
+
"normalized": false,
|
| 1871 |
+
"rstrip": false,
|
| 1872 |
+
"single_word": false,
|
| 1873 |
+
"special": true
|
| 1874 |
+
},
|
| 1875 |
+
"128234": {
|
| 1876 |
+
"content": "<|reserved_special_token_229|>",
|
| 1877 |
+
"lstrip": false,
|
| 1878 |
+
"normalized": false,
|
| 1879 |
+
"rstrip": false,
|
| 1880 |
+
"single_word": false,
|
| 1881 |
+
"special": true
|
| 1882 |
+
},
|
| 1883 |
+
"128235": {
|
| 1884 |
+
"content": "<|reserved_special_token_230|>",
|
| 1885 |
+
"lstrip": false,
|
| 1886 |
+
"normalized": false,
|
| 1887 |
+
"rstrip": false,
|
| 1888 |
+
"single_word": false,
|
| 1889 |
+
"special": true
|
| 1890 |
+
},
|
| 1891 |
+
"128236": {
|
| 1892 |
+
"content": "<|reserved_special_token_231|>",
|
| 1893 |
+
"lstrip": false,
|
| 1894 |
+
"normalized": false,
|
| 1895 |
+
"rstrip": false,
|
| 1896 |
+
"single_word": false,
|
| 1897 |
+
"special": true
|
| 1898 |
+
},
|
| 1899 |
+
"128237": {
|
| 1900 |
+
"content": "<|reserved_special_token_232|>",
|
| 1901 |
+
"lstrip": false,
|
| 1902 |
+
"normalized": false,
|
| 1903 |
+
"rstrip": false,
|
| 1904 |
+
"single_word": false,
|
| 1905 |
+
"special": true
|
| 1906 |
+
},
|
| 1907 |
+
"128238": {
|
| 1908 |
+
"content": "<|reserved_special_token_233|>",
|
| 1909 |
+
"lstrip": false,
|
| 1910 |
+
"normalized": false,
|
| 1911 |
+
"rstrip": false,
|
| 1912 |
+
"single_word": false,
|
| 1913 |
+
"special": true
|
| 1914 |
+
},
|
| 1915 |
+
"128239": {
|
| 1916 |
+
"content": "<|reserved_special_token_234|>",
|
| 1917 |
+
"lstrip": false,
|
| 1918 |
+
"normalized": false,
|
| 1919 |
+
"rstrip": false,
|
| 1920 |
+
"single_word": false,
|
| 1921 |
+
"special": true
|
| 1922 |
+
},
|
| 1923 |
+
"128240": {
|
| 1924 |
+
"content": "<|reserved_special_token_235|>",
|
| 1925 |
+
"lstrip": false,
|
| 1926 |
+
"normalized": false,
|
| 1927 |
+
"rstrip": false,
|
| 1928 |
+
"single_word": false,
|
| 1929 |
+
"special": true
|
| 1930 |
+
},
|
| 1931 |
+
"128241": {
|
| 1932 |
+
"content": "<|reserved_special_token_236|>",
|
| 1933 |
+
"lstrip": false,
|
| 1934 |
+
"normalized": false,
|
| 1935 |
+
"rstrip": false,
|
| 1936 |
+
"single_word": false,
|
| 1937 |
+
"special": true
|
| 1938 |
+
},
|
| 1939 |
+
"128242": {
|
| 1940 |
+
"content": "<|reserved_special_token_237|>",
|
| 1941 |
+
"lstrip": false,
|
| 1942 |
+
"normalized": false,
|
| 1943 |
+
"rstrip": false,
|
| 1944 |
+
"single_word": false,
|
| 1945 |
+
"special": true
|
| 1946 |
+
},
|
| 1947 |
+
"128243": {
|
| 1948 |
+
"content": "<|reserved_special_token_238|>",
|
| 1949 |
+
"lstrip": false,
|
| 1950 |
+
"normalized": false,
|
| 1951 |
+
"rstrip": false,
|
| 1952 |
+
"single_word": false,
|
| 1953 |
+
"special": true
|
| 1954 |
+
},
|
| 1955 |
+
"128244": {
|
| 1956 |
+
"content": "<|reserved_special_token_239|>",
|
| 1957 |
+
"lstrip": false,
|
| 1958 |
+
"normalized": false,
|
| 1959 |
+
"rstrip": false,
|
| 1960 |
+
"single_word": false,
|
| 1961 |
+
"special": true
|
| 1962 |
+
},
|
| 1963 |
+
"128245": {
|
| 1964 |
+
"content": "<|reserved_special_token_240|>",
|
| 1965 |
+
"lstrip": false,
|
| 1966 |
+
"normalized": false,
|
| 1967 |
+
"rstrip": false,
|
| 1968 |
+
"single_word": false,
|
| 1969 |
+
"special": true
|
| 1970 |
+
},
|
| 1971 |
+
"128246": {
|
| 1972 |
+
"content": "<|reserved_special_token_241|>",
|
| 1973 |
+
"lstrip": false,
|
| 1974 |
+
"normalized": false,
|
| 1975 |
+
"rstrip": false,
|
| 1976 |
+
"single_word": false,
|
| 1977 |
+
"special": true
|
| 1978 |
+
},
|
| 1979 |
+
"128247": {
|
| 1980 |
+
"content": "<|reserved_special_token_242|>",
|
| 1981 |
+
"lstrip": false,
|
| 1982 |
+
"normalized": false,
|
| 1983 |
+
"rstrip": false,
|
| 1984 |
+
"single_word": false,
|
| 1985 |
+
"special": true
|
| 1986 |
+
},
|
| 1987 |
+
"128248": {
|
| 1988 |
+
"content": "<|reserved_special_token_243|>",
|
| 1989 |
+
"lstrip": false,
|
| 1990 |
+
"normalized": false,
|
| 1991 |
+
"rstrip": false,
|
| 1992 |
+
"single_word": false,
|
| 1993 |
+
"special": true
|
| 1994 |
+
},
|
| 1995 |
+
"128249": {
|
| 1996 |
+
"content": "<|reserved_special_token_244|>",
|
| 1997 |
+
"lstrip": false,
|
| 1998 |
+
"normalized": false,
|
| 1999 |
+
"rstrip": false,
|
| 2000 |
+
"single_word": false,
|
| 2001 |
+
"special": true
|
| 2002 |
+
},
|
| 2003 |
+
"128250": {
|
| 2004 |
+
"content": "<|reserved_special_token_245|>",
|
| 2005 |
+
"lstrip": false,
|
| 2006 |
+
"normalized": false,
|
| 2007 |
+
"rstrip": false,
|
| 2008 |
+
"single_word": false,
|
| 2009 |
+
"special": true
|
| 2010 |
+
},
|
| 2011 |
+
"128251": {
|
| 2012 |
+
"content": "<|reserved_special_token_246|>",
|
| 2013 |
+
"lstrip": false,
|
| 2014 |
+
"normalized": false,
|
| 2015 |
+
"rstrip": false,
|
| 2016 |
+
"single_word": false,
|
| 2017 |
+
"special": true
|
| 2018 |
+
},
|
| 2019 |
+
"128252": {
|
| 2020 |
+
"content": "<|reserved_special_token_247|>",
|
| 2021 |
+
"lstrip": false,
|
| 2022 |
+
"normalized": false,
|
| 2023 |
+
"rstrip": false,
|
| 2024 |
+
"single_word": false,
|
| 2025 |
+
"special": true
|
| 2026 |
+
},
|
| 2027 |
+
"128253": {
|
| 2028 |
+
"content": "<|reserved_special_token_248|>",
|
| 2029 |
+
"lstrip": false,
|
| 2030 |
+
"normalized": false,
|
| 2031 |
+
"rstrip": false,
|
| 2032 |
+
"single_word": false,
|
| 2033 |
+
"special": true
|
| 2034 |
+
},
|
| 2035 |
+
"128254": {
|
| 2036 |
+
"content": "<|reserved_special_token_249|>",
|
| 2037 |
+
"lstrip": false,
|
| 2038 |
+
"normalized": false,
|
| 2039 |
+
"rstrip": false,
|
| 2040 |
+
"single_word": false,
|
| 2041 |
+
"special": true
|
| 2042 |
+
},
|
| 2043 |
+
"128255": {
|
| 2044 |
+
"content": "<|reserved_special_token_250|>",
|
| 2045 |
+
"lstrip": false,
|
| 2046 |
+
"normalized": false,
|
| 2047 |
+
"rstrip": false,
|
| 2048 |
+
"single_word": false,
|
| 2049 |
+
"special": true
|
| 2050 |
+
},
|
| 2051 |
+
"128256": {
|
| 2052 |
+
"content": "<|eom_id|>",
|
| 2053 |
+
"lstrip": false,
|
| 2054 |
+
"normalized": false,
|
| 2055 |
+
"rstrip": false,
|
| 2056 |
+
"single_word": false,
|
| 2057 |
+
"special": true
|
| 2058 |
+
},
|
| 2059 |
+
"128257": {
|
| 2060 |
+
"content": "<|python_tag|>",
|
| 2061 |
+
"lstrip": false,
|
| 2062 |
+
"normalized": false,
|
| 2063 |
+
"rstrip": false,
|
| 2064 |
+
"single_word": false,
|
| 2065 |
+
"special": true
|
| 2066 |
+
},
|
| 2067 |
+
"128258": {
|
| 2068 |
+
"content": "<|NONE|>",
|
| 2069 |
+
"lstrip": false,
|
| 2070 |
+
"normalized": false,
|
| 2071 |
+
"rstrip": false,
|
| 2072 |
+
"single_word": false,
|
| 2073 |
+
"special": true
|
| 2074 |
+
}
|
| 2075 |
+
},
|
| 2076 |
+
"bos_token": "<|begin_of_text|>",
|
| 2077 |
+
"chat_template": "{# version=v3-llama3.1 #}{%- macro append_new_param_info(param_declaration, comment_info, examples_info, depth) -%}\n {%- set offset = \"\" -%}\n {%- if depth >= 1 -%}\n {%- set offset = \" \" * depth -%}\n {%- endif -%}\n {%- if comment_info != \"<|NONE|>\" -%}\n {{ \"\\n\" + offset + comment_info }}\n {%- if examples_info | length > 0 -%}\n {# Append each example info #}\n {%- for example in examples_info -%}\n {{ \"\\n\" + offset + \"// \" + example|string|replace(\"'\", '\"') }}\n {%- endfor -%}\n {%- endif -%}\n {%- endif -%}\n {{ \"\\n\" + offset + param_declaration }}\n{%- endmacro -%}\n\n{%- macro convert_data_type(param_type) -%}\n {%- if param_type == \"integer\" or param_type == \"float\" -%}\n {{ \"number\" }}\n {%- else -%}\n {{ param_type }}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro get_param_type(param) -%}\n {%- set param_type = \"any\" -%}\n\n {%- if \"type\" in param -%}\n {%- set raw_param_type = param[\"type\"] -%}\n {%- if raw_param_type is iterable and raw_param_type is not string -%}\n {%- set param_type = raw_param_type | join(\" | \") -%}\n {%- else -%}\n {%- set param_type = raw_param_type -%}\n {%- endif -%}\n {{ convert_data_type(param_type) }}\n {%- elif \"oneOf\" in param -%}\n {%- set one_of_types = param[\"oneOf\"]|selectattr(\"type\", \"defined\")|list -%}\n {%- set one_of_types = one_of_types|map(attribute=\"type\")|unique|list -%}\n {{ convert_data_type(one_of_types | join(\" | \")) }}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro get_format_param(param) -%}\n {%- if \"format\" in param -%}\n {{ param[\"format\"] }}\n {%- elif \"oneOf\" in param -%}\n {%- set formats = [] -%}\n {%- for item in param[\"oneOf\"] -%}\n {%- if \"format\" in item -%}\n {%- if item[\"format\"] == param[\"oneOf\"][-1][\"format\"] -%}\n {{ item[\"format\"] }}\n {%- else -%}\n {{ item[\"format\"] + \" or \"}}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ \"<|NONE|>\" }}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro get_param_info(param) -%}\n {%- set param_type = param.get(\"type\", \"any\") -%}\n {%- set format_param = get_format_param(param) -%}\n\n {%- if \"description\" in param or \"default\" in param or format_param != \"<|NONE|>\" or param[\"maximum\"] or param[\"minimum\"] or param[\"maxLength\"] or param[\"minLength\"] -%}\n {{ \"//\" }}\n {%- if \"description\" in param -%}\n {%- set desc = param[\"description\"] -%}\n {%- if not desc.endswith(\".\") -%}\n {%- set desc = desc + \".\" -%}\n {%- endif -%}\n {{ \" \" + desc }}\n {%- endif -%}\n\n {%- if \"default\" in param -%}\n {%- set default_value = param[\"default\"] -%}\n {%- if param_type == \"string\" -%}\n {%- set default_value = '\"' ~ default_value ~ '\"' -%}\n {%- endif -%}\n {{ \" Default=\" ~ default_value ~ \".\" }}\n {%- endif -%}\n\n {%- set format_param = get_format_param(param) -%}\n {%- if format_param != \"<|NONE|>\" -%}\n {{ \" Format=\" ~ format_param }}\n {%- endif -%}\n\n {%- for field, field_name in [(\"maximum\", \"Maximum\"), (\"minimum\", \"Minimum\"), (\"maxLength\", \"Maximum length\"), (\"minLength\", \"Minimum length\")] -%}\n {%- if field in param -%}\n {{ \" \" + field_name ~ \"=\" ~ param[field] }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ \"<|NONE|>\"}}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro get_enum_option_str(enum_options) -%}\n {%- for v in enum_options -%}\n {%- if v is string -%}\n {{ '\"' + v + '\"' }}\n {%- else -%}\n {{ v }}\n {%- endif -%}\n {%- if enum_options|length > 0 and v != enum_options[-1] -%}\n {{ \" | \" }}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n\n{%- macro get_array_typescript(param_name, param_dic, depth) -%}\n {%- set offset = '' -%}\n {%- if depth >= 1 -%}\n {%- set offset = \" \" * depth -%}\n {%- endif -%}\n {%- set items_info = param_dic.get('items', {}) -%}\n\n {%- if items_info|length == 0 -%}\n {%- if param_name -%}\n {{ \"\\n\" + offset + param_name + \": []\" }}\n {%- else -%}\n {{ \"\\n\" + offset + \"[]\" }}\n {%- endif -%}\n {%- else -%}\n {%- set array_type = get_param_type(items_info) -%}\n {%- if array_type == 'object' -%}\n {%- if param_name -%}\n {{ \"\\n\" + offset + param_name + \": {\" }}\n {%- else -%}\n {{ \"\\n\" + offset + \"{\" }}\n {%- endif -%}\n {{ get_parameter_typescript(items_info.get('properties', {}), items_info.get('required', []), depth + 1) -}}\n {{- \"\\n\" + offset + \"}[]\" }}\n {%- elif array_type == 'array' -%}\n {%- set item_info = get_array_typescript(None, items_info, depth + 1) -%}\n {%- if not param_name -%}\n {{ \"\\n\" + item_info + \"[]\" }}\n {%- else -%}\n {{ \"\\n\" + offset + param_name + \": \" + item_info|trim + \"[]\" }}\n {%- endif -%}\n {%- else -%}\n {%- if 'enum' in items_info -%}\n {%- set item_type = get_enum_option_str(items_info['enum']) -%}\n {%- if param_name is none -%}\n {{ \"(\" + item_type + \")[]\"}}\n {%- else -%}\n {{ \"\\n\" + offset + param_name + \": (\" + item_type + \")[]\" }}\n {%- endif -%}\n {%- else -%}\n {%- if param_name is none -%}\n {{ \"\\n\" + array_type + \"[]\" }}\n {%- else -%}\n {{ \"\\n\" + offset + param_name + \": \" + array_type + \"[],\" }}\n {%- endif -%}\n {%- endif -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n{%- macro get_parameter_typescript(properties, required_params, depth=0) -%}\n {%- set res = \"\" -%}\n {%- for param_name, param in properties.items() -%}\n {%- if param is mapping -%}\n {%- set comment_info = get_param_info(param) -%}\n {# Param Examples #}\n {%- set examples_info = [] -%}\n {%- if \"examples\" in param -%}\n {%- set examples_info = [\"Example \" + param_name + \":\"] -%}\n {%- set examples_info = examples_info + param[\"examples\"] -%}\n {%- endif -%}\n\n {# Param Name declaration #}\n {%- set param_declaration = param_name -%}\n {%- if required_params is iterable and param_name not in required_params -%}\n {%- set param_declaration = param_declaration + \"?\" -%}\n {%- endif -%}\n\n {%- set param_type = get_param_type(param) -%}\n\n {# Handle indentation based on depth #}\n {%- set offset = \"\" -%}\n {%- if depth >= 1 -%}\n {%- set offset = \" \" * depth -%}\n {%- endif -%}\n\n {%- if param_type == \"object\" -%}\n {%- if comment_info != \"<|NONE|>\" -%}\n {{ \"\\n\" + offset + comment_info }}\n {%- endif -%}\n {%- if examples_info|length > 0 -%}\n {%- for example in examples_info -%}\n {{ \"\\n\" + offset + \"// \" + example|string|replace(\"'\", '\"') }}\n {%- endfor -%}\n {%- endif -%}\n {%- set param_declaration = param_declaration + \": {\" -%}\n {{ \"\\n\" + offset + param_declaration -}}\n {{- get_parameter_typescript(param.get(\"properties\", {}), param.get(\"required\", []), depth + 1) -}}\n {{- \"\\n\" + offset + \"},\" }}\n {%- elif param_type == \"array\" -%}\n {%- set item_info = param.get(\"items\", {}) -%}\n {%- if \"type\" not in item_info -%}\n {%- set param_declaration = param_declaration + \": [],\" -%}\n {{ append_new_param_info(param_declaration, comment_info, examples_info, depth) }}\n {%- else -%}\n {%- if comment_info != \"<|NONE|>\" -%}\n {{ \"\\n\" + offset + comment_info }}\n {%- endif -%}\n {%- if examples_info|length > 0 -%}\n {%- for example in examples_info -%}\n {{ \"\\n\" + offset + \"// \" + example|string|replace(\"'\", '\"') }}\n {%- endfor -%}\n {%- endif -%}\n {%- set array_declaration = get_array_typescript(param_declaration, param, depth) -%}\n {%- if not array_declaration.endswith(\",\") -%}\n {%- set array_declaration = array_declaration + \",\" -%}\n {%- endif -%}\n {{ array_declaration}}\n {%- endif -%}\n {%- else -%}\n {%- if \"enum\" in param -%}\n {%- set param_type = get_enum_option_str(param[\"enum\"]) -%}\n {%- endif -%}\n {%- if \"nullable\" in param and param[\"nullable\"] -%}\n {%- set param_type = param_type + \" | null\" -%}\n {%- endif -%}\n {%- set param_declaration = param_declaration + \": \" + param_type + \",\" -%}\n {{ append_new_param_info(param_declaration, comment_info, examples_info, depth) }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n\n{%- macro generate_schema_from_functions(functions, namespace='functions') -%}\n {{ \"// Supported function definitions that should be called when necessary.\\n\" -}}\n {{- \"namespace \" + namespace + \" {\\n\\n\" -}}\n\n {%- for function in functions -%}\n {%- if function.get(\"function\") -%}\n {%- set function = function.get(\"function\") -%}\n {%- endif -%}\n\n {%- set function_name = function.get(\"name\") -%}\n {%- if function_name -%}\n {%- set description = function.get('description', '') -%}\n {%- set parameters = function.get('parameters', {}) -%}\n {{- \"// \" + description + \"\\n\" -}}\n {{- \"type \" + function_name -}}\n {%- if parameters and parameters.get(\"properties\") -%}\n {{- \" = (_: {\" -}}\n {%- set required_params = parameters.get(\"required\", []) -%}\n {{ get_parameter_typescript(parameters.get(\"properties\"), required_params, 0) -}}\n {{- \"\\n}) => any;\\n\\n\" }}\n {%- else -%}\n {{ \" = () => any;\\n\\n\" }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n {{ \"} // namespace \" + namespace }}\n{%- endmacro -%}\n\n{%- if not tools is defined -%}\n {%- set tools = none -%}\n{%- endif -%}\n\n{%- set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", \"code_interpreter\") | list | length > 0 -%}\n{%- if has_code_interpreter -%}\n {%- set tools = tools | rejectattr(\"type\", \"equalto\", \"code_interpreter\") | list -%}\n{%- endif -%}\n\n{#- System message + builtin tools #}\n{{- bos_token + \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if has_code_interpreter %}\n {{- \"Environment: ipython\\n\\n\" }}\n{%- else -%}\n {{ \"\"}}\n{%- endif %}\n{%- if tools %}\n {{- \"\\nYou have access to the following functions:\\n\\n\" }}\n {%- for t in tools %}\n {%- if \"type\" in t -%}\n {{ \"Use the function '\" + t[\"function\"][\"name\"] + \"' to '\" + t[\"function\"][\"description\"] + \"'\\n\" + t[\"function\"] | tojson() }}\n {%- else -%}\n {{ \"Use the function '\" + t[\"name\"] + \"' to '\" + t[\"description\"] + \"'\\n\" + t | tojson }}\n {%- endif -%}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- '\\nThink very carefully before calling functions.\\nIf a you choose to call a function ONLY reply in the following format:\\n<{start_tag}={function_name}>{parameters}{end_tag}\\nwhere\\n\\nstart_tag => `<function`\\nparameters => a JSON dict with the function argument name as key and function argument value as value.\\nend_tag => `</function>`\\n\\nHere is an example,\\n<function=example_function_name>{\"example_name\": \"example_value\"}</function>\\n\\nReminder:\\n- If looking for real time information use relevant functions before falling back to brave_search\\n- Function calls MUST follow the specified format, start with <function= and end with </function>\\n- Required parameters MUST be specified\\n- Only call one function at a time\\n- Put the entire function call reply on one line\\n\\n' -}}\n{%- endif %}\n{{- \"<|eot_id|>\" -}}\n\n{%- for message in messages -%}\n {%- if message['role'] == 'user' or message['role'] == 'system' -%}\n {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n {%- elif message['role'] == 'tool' -%}\n {{ '<|start_header_id|>ipython<|end_header_id|>\\n\\n' + message['content'] + '<|eot_id|>' }}\n {%- else -%}\n {%- if (message['content'] and message['content']|length > 0) or ('tool_calls' in message and message['tool_calls'] and message['tool_calls']|length > 0) -%}\n {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'}}\n {%- endif -%}\n {%- if message['content'] and message['content']|length > 0 -%}\n {{ message['content'] }}\n {%- endif -%}\n {%- if 'tool_calls' in message and message['tool_calls'] and message['tool_calls']|length > 0 -%}\n {%- for tool_call in message['tool_calls'] -%}\n {%- if tool_call[\"function\"][\"name\"] == \"python\" -%}\n {{ '<|python_tag|>' + tool_call['function']['arguments'] }}\n {%- else -%}\n {{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments'] + '</function>' }}\n {%- endif -%}\n {%- endfor -%}\n {{ '<|eom_id|>' }}\n {%- elif message['content'] and message['content']|length > 0 -%}\n {{ '<|eot_id|>' }}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{ '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif -%}\n",
|
| 2078 |
+
"clean_up_tokenization_spaces": false,
|
| 2079 |
+
"eos_token": "<|eot_id|>",
|
| 2080 |
+
"extra_special_tokens": {},
|
| 2081 |
+
"max_length": 8192,
|
| 2082 |
+
"model_input_names": [
|
| 2083 |
+
"input_ids",
|
| 2084 |
+
"attention_mask"
|
| 2085 |
+
],
|
| 2086 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 2087 |
+
"pad_token": "<|end_of_text|>",
|
| 2088 |
+
"stride": 0,
|
| 2089 |
+
"tokenizer_class": "PreTrainedTokenizer",
|
| 2090 |
+
"truncation_side": "right",
|
| 2091 |
+
"truncation_strategy": "longest_first"
|
| 2092 |
+
}
|