duzx16
commited on
Commit
·
e1c8748
1
Parent(s):
10d4977
Fix apply_chat_template
Browse files- tokenization_chatglm.py +2 -2
tokenization_chatglm.py
CHANGED
|
@@ -168,7 +168,8 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
|
|
| 168 |
for item in conversation:
|
| 169 |
if item.get("tools"):
|
| 170 |
tools = item["tools"]
|
| 171 |
-
content = "你是一个名为
|
|
|
|
| 172 |
for tool in tools:
|
| 173 |
if tool["type"] == "function":
|
| 174 |
function = tool["function"]
|
|
@@ -203,7 +204,6 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
|
|
| 203 |
input_ids.extend([self.convert_tokens_to_ids("<|assistant|>")])
|
| 204 |
else:
|
| 205 |
input_message += "<|assistant|>"
|
| 206 |
-
|
| 207 |
return input_ids if tokenize else input_message
|
| 208 |
|
| 209 |
# Main logic to handle different conversation formats
|
|
|
|
| 168 |
for item in conversation:
|
| 169 |
if item.get("tools"):
|
| 170 |
tools = item["tools"]
|
| 171 |
+
content = "你是一个名为 GhatGLM 的人工智能助手。你是基于智谱AI训练的语言模型 GLM-4 模型开发的,你的任务是针对用户的问题和要求提供适当的答复和支持。"
|
| 172 |
+
content += "\n\n# 可用工具"
|
| 173 |
for tool in tools:
|
| 174 |
if tool["type"] == "function":
|
| 175 |
function = tool["function"]
|
|
|
|
| 204 |
input_ids.extend([self.convert_tokens_to_ids("<|assistant|>")])
|
| 205 |
else:
|
| 206 |
input_message += "<|assistant|>"
|
|
|
|
| 207 |
return input_ids if tokenize else input_message
|
| 208 |
|
| 209 |
# Main logic to handle different conversation formats
|