Hugging Face's logo

MTSAIR
/
ROCKET-Qwen-8b

Safetensors
qwen3
custom_code
Model card Files
xet
 Community
3
BaherMH commited on
Commit
078db80
·
verified ·
1 Parent(s): 85c80e7

Upload trained checkpoint 1500

Browse files
Files changed (23) hide show
  1. .gitattributes +2 -0
  2. added_tokens.json +28 -0
  3. chat_template.jinja +89 -0
  4. config.json +1212 -0
  5. generation_config.json +14 -0
  6. merges.txt +0 -0
  7. model-00001-of-00006.safetensors +3 -0
  8. model-00002-of-00006.safetensors +3 -0
  9. model-00003-of-00006.safetensors +3 -0
  10. model-00004-of-00006.safetensors +3 -0
  11. model-00005-of-00006.safetensors +3 -0
  12. model-00006-of-00006.safetensors +3 -0
  13. model.safetensors.index.json +731 -0
  14. modeling_qwen3_swift_sparse.py +697 -0
  15. optimizer.pt +3 -0
  16. rng_state.pth +3 -0
  17. scheduler.pt +3 -0
  18. special_tokens_map.json +31 -0
  19. tokenizer.json +3 -0
  20. tokenizer_config.json +239 -0
  21. trainer_state.json +3 -0
  22. training_args.bin +3 -0
  23. vocab.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ trainer_state.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
chat_template.jinja ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {{- messages[0].content + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0].role == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
18
+ {%- for message in messages[::-1] %}
19
+ {%- set index = (messages|length - 1) - loop.index0 %}
20
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
21
+ {%- set ns.multi_step_tool = false %}
22
+ {%- set ns.last_query_index = index %}
23
+ {%- endif %}
24
+ {%- endfor %}
25
+ {%- for message in messages %}
26
+ {%- if message.content is string %}
27
+ {%- set content = message.content %}
28
+ {%- else %}
29
+ {%- set content = '' %}
30
+ {%- endif %}
31
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
32
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
33
+ {%- elif message.role == "assistant" %}
34
+ {%- set reasoning_content = '' %}
35
+ {%- if message.reasoning_content is string %}
36
+ {%- set reasoning_content = message.reasoning_content %}
37
+ {%- else %}
38
+ {%- if '</think>' in content %}
39
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
40
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
41
+ {%- endif %}
42
+ {%- endif %}
43
+ {%- if loop.index0 > ns.last_query_index %}
44
+ {%- if loop.last or (not loop.last and reasoning_content) %}
45
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
46
+ {%- else %}
47
+ {{- '<|im_start|>' + message.role + '\n' + content }}
48
+ {%- endif %}
49
+ {%- else %}
50
+ {{- '<|im_start|>' + message.role + '\n' + content }}
51
+ {%- endif %}
52
+ {%- if message.tool_calls %}
53
+ {%- for tool_call in message.tool_calls %}
54
+ {%- if (loop.first and content) or (not loop.first) %}
55
+ {{- '\n' }}
56
+ {%- endif %}
57
+ {%- if tool_call.function %}
58
+ {%- set tool_call = tool_call.function %}
59
+ {%- endif %}
60
+ {{- '<tool_call>\n{"name": "' }}
61
+ {{- tool_call.name }}
62
+ {{- '", "arguments": ' }}
63
+ {%- if tool_call.arguments is string %}
64
+ {{- tool_call.arguments }}
65
+ {%- else %}
66
+ {{- tool_call.arguments | tojson }}
67
+ {%- endif %}
68
+ {{- '}\n</tool_call>' }}
69
+ {%- endfor %}
70
+ {%- endif %}
71
+ {{- '<|im_end|>\n' }}
72
+ {%- elif message.role == "tool" %}
73
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
74
+ {{- '<|im_start|>user' }}
75
+ {%- endif %}
76
+ {{- '\n<tool_response>\n' }}
77
+ {{- content }}
78
+ {{- '\n</tool_response>' }}
79
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
80
+ {{- '<|im_end|>\n' }}
81
+ {%- endif %}
82
+ {%- endif %}
83
+ {%- endfor %}
84
+ {%- if add_generation_prompt %}
85
+ {{- '<|im_start|>assistant\n' }}
86
+ {%- if enable_thinking is defined and enable_thinking is false %}
87
+ {{- '<think>\n\n</think>\n\n' }}
88
+ {%- endif %}
89
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,1212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoModelForCausalLM": "modeling_qwen3_swift_sparse.Qwen3ForCausalLM"
9
+ },
10
+ "bos_token_id": 151643,
11
+ "compression_ratio": {
12
+ "mlp.down_proj": {
13
+ "0": {
14
+ "cr": 0.3,
15
+ "ks": 2.4
16
+ },
17
+ "1": {
18
+ "cr": 0.35,
19
+ "ks": 2.1
20
+ },
21
+ "10": {
22
+ "cr": 0.3,
23
+ "ks": 2.3
24
+ },
25
+ "11": {
26
+ "cr": 0.35,
27
+ "ks": 2.2
28
+ },
29
+ "12": {
30
+ "cr": 0.35,
31
+ "ks": 2.2
32
+ },
33
+ "13": {
34
+ "cr": 0.3,
35
+ "ks": 2.4
36
+ },
37
+ "14": {
38
+ "cr": 0.3,
39
+ "ks": 2.4
40
+ },
41
+ "15": {
42
+ "cr": 0.3,
43
+ "ks": 2.3
44
+ },
45
+ "16": {
46
+ "cr": 0.3,
47
+ "ks": 2.3
48
+ },
49
+ "17": {
50
+ "cr": 0.3,
51
+ "ks": 2.3
52
+ },
53
+ "18": {
54
+ "cr": 0.3,
55
+ "ks": 2.4
56
+ },
57
+ "19": {
58
+ "cr": 0.3,
59
+ "ks": 2.3
60
+ },
61
+ "2": {
62
+ "cr": 0.4,
63
+ "ks": 1.9
64
+ },
65
+ "20": {
66
+ "cr": 0.3,
67
+ "ks": 2.4
68
+ },
69
+ "21": {
70
+ "cr": 0.3,
71
+ "ks": 2.4
72
+ },
73
+ "22": {
74
+ "cr": 0.3,
75
+ "ks": 2.4
76
+ },
77
+ "23": {
78
+ "cr": 0.3,
79
+ "ks": 2.4
80
+ },
81
+ "24": {
82
+ "cr": 0.3,
83
+ "ks": 2.4
84
+ },
85
+ "25": {
86
+ "cr": 0.3,
87
+ "ks": 2.4
88
+ },
89
+ "26": {
90
+ "cr": 0.3,
91
+ "ks": 2.5
92
+ },
93
+ "27": {
94
+ "cr": 0.3,
95
+ "ks": 2.5
96
+ },
97
+ "28": {
98
+ "cr": 0.3,
99
+ "ks": 2.5
100
+ },
101
+ "29": {
102
+ "cr": 0.25,
103
+ "ks": 2.7
104
+ },
105
+ "3": {
106
+ "cr": 0.3,
107
+ "ks": 2.3
108
+ },
109
+ "30": {
110
+ "cr": 0.25,
111
+ "ks": 2.7
112
+ },
113
+ "31": {
114
+ "cr": 0.25,
115
+ "ks": 2.7
116
+ },
117
+ "32": {
118
+ "cr": 0.3,
119
+ "ks": 2.6
120
+ },
121
+ "33": {
122
+ "cr": 0.3,
123
+ "ks": 2.5
124
+ },
125
+ "34": {
126
+ "cr": 0.3,
127
+ "ks": 2.5
128
+ },
129
+ "35": {
130
+ "cr": 0.3,
131
+ "ks": 2.4
132
+ },
133
+ "36": {
134
+ "cr": 0.3,
135
+ "ks": 2.4
136
+ },
137
+ "37": {
138
+ "cr": 0.25,
139
+ "ks": 2.7
140
+ },
141
+ "38": {
142
+ "cr": 0.25,
143
+ "ks": 2.9
144
+ },
145
+ "39": {
146
+ "cr": 0.25,
147
+ "ks": 3.0
148
+ },
149
+ "4": {
150
+ "cr": 0.3,
151
+ "ks": 2.4
152
+ },
153
+ "5": {
154
+ "cr": 0.25,
155
+ "ks": 2.9
156
+ },
157
+ "6": {
158
+ "cr": 0.25,
159
+ "ks": 3.0
160
+ },
161
+ "7": {
162
+ "cr": 0.25,
163
+ "ks": 2.9
164
+ },
165
+ "8": {
166
+ "cr": 0.3,
167
+ "ks": 2.6
168
+ },
169
+ "9": {
170
+ "cr": 0.3,
171
+ "ks": 2.4
172
+ }
173
+ },
174
+ "mlp.gate_proj": {
175
+ "0": {
176
+ "cr": 0.7,
177
+ "ks": 1.8
178
+ },
179
+ "1": {
180
+ "cr": 0.7,
181
+ "ks": 2.5
182
+ },
183
+ "10": {
184
+ "cr": 0.5,
185
+ "ks": 3.0
186
+ },
187
+ "11": {
188
+ "cr": 0.5,
189
+ "ks": 3.0
190
+ },
191
+ "12": {
192
+ "cr": 0.5,
193
+ "ks": 3.0
194
+ },
195
+ "13": {
196
+ "cr": 0.55,
197
+ "ks": 3.0
198
+ },
199
+ "14": {
200
+ "cr": 0.5,
201
+ "ks": 3.0
202
+ },
203
+ "15": {
204
+ "cr": 0.5,
205
+ "ks": 3.0
206
+ },
207
+ "16": {
208
+ "cr": 0.5,
209
+ "ks": 3.0
210
+ },
211
+ "17": {
212
+ "cr": 0.5,
213
+ "ks": 3.0
214
+ },
215
+ "18": {
216
+ "cr": 0.5,
217
+ "ks": 3.0
218
+ },
219
+ "19": {
220
+ "cr": 0.5,
221
+ "ks": 3.0
222
+ },
223
+ "2": {
224
+ "cr": 0.7,
225
+ "ks": 3.0
226
+ },
227
+ "20": {
228
+ "cr": 0.5,
229
+ "ks": 3.0
230
+ },
231
+ "21": {
232
+ "cr": 0.5,
233
+ "ks": 3.0
234
+ },
235
+ "22": {
236
+ "cr": 0.5,
237
+ "ks": 3.0
238
+ },
239
+ "23": {
240
+ "cr": 0.5,
241
+ "ks": 3.0
242
+ },
243
+ "24": {
244
+ "cr": 0.5,
245
+ "ks": 3.0
246
+ },
247
+ "25": {
248
+ "cr": 0.5,
249
+ "ks": 3.0
250
+ },
251
+ "26": {
252
+ "cr": 0.5,
253
+ "ks": 3.0
254
+ },
255
+ "27": {
256
+ "cr": 0.5,
257
+ "ks": 3.0
258
+ },
259
+ "28": {
260
+ "cr": 0.5,
261
+ "ks": 3.0
262
+ },
263
+ "29": {
264
+ "cr": 0.5,
265
+ "ks": 3.0
266
+ },
267
+ "3": {
268
+ "cr": 0.65,
269
+ "ks": 2.9
270
+ },
271
+ "30": {
272
+ "cr": 0.5,
273
+ "ks": 3.0
274
+ },
275
+ "31": {
276
+ "cr": 0.5,
277
+ "ks": 3.0
278
+ },
279
+ "32": {
280
+ "cr": 0.5,
281
+ "ks": 3.0
282
+ },
283
+ "33": {
284
+ "cr": 0.5,
285
+ "ks": 3.0
286
+ },
287
+ "34": {
288
+ "cr": 0.5,
289
+ "ks": 3.0
290
+ },
291
+ "35": {
292
+ "cr": 0.5,
293
+ "ks": 3.0
294
+ },
295
+ "36": {
296
+ "cr": 0.5,
297
+ "ks": 3.0
298
+ },
299
+ "37": {
300
+ "cr": 0.5,
301
+ "ks": 3.0
302
+ },
303
+ "38": {
304
+ "cr": 0.5,
305
+ "ks": 3.0
306
+ },
307
+ "39": {
308
+ "cr": 0.5,
309
+ "ks": 3.0
310
+ },
311
+ "4": {
312
+ "cr": 0.6,
313
+ "ks": 3.0
314
+ },
315
+ "5": {
316
+ "cr": 0.6,
317
+ "ks": 2.9
318
+ },
319
+ "6": {
320
+ "cr": 0.55,
321
+ "ks": 3.0
322
+ },
323
+ "7": {
324
+ "cr": 0.55,
325
+ "ks": 3.0
326
+ },
327
+ "8": {
328
+ "cr": 0.55,
329
+ "ks": 3.0
330
+ },
331
+ "9": {
332
+ "cr": 0.5,
333
+ "ks": 3.0
334
+ }
335
+ },
336
+ "mlp.up_proj": {
337
+ "0": {
338
+ "cr": 0.7,
339
+ "ks": 1.9
340
+ },
341
+ "1": {
342
+ "cr": 0.7,
343
+ "ks": 3.0
344
+ },
345
+ "10": {
346
+ "cr": 0.5,
347
+ "ks": 3.0
348
+ },
349
+ "11": {
350
+ "cr": 0.5,
351
+ "ks": 3.0
352
+ },
353
+ "12": {
354
+ "cr": 0.5,
355
+ "ks": 3.0
356
+ },
357
+ "13": {
358
+ "cr": 0.5,
359
+ "ks": 3.0
360
+ },
361
+ "14": {
362
+ "cr": 0.5,
363
+ "ks": 3.0
364
+ },
365
+ "15": {
366
+ "cr": 0.5,
367
+ "ks": 3.0
368
+ },
369
+ "16": {
370
+ "cr": 0.5,
371
+ "ks": 3.0
372
+ },
373
+ "17": {
374
+ "cr": 0.5,
375
+ "ks": 3.0
376
+ },
377
+ "18": {
378
+ "cr": 0.5,
379
+ "ks": 3.0
380
+ },
381
+ "19": {
382
+ "cr": 0.5,
383
+ "ks": 3.0
384
+ },
385
+ "2": {
386
+ "cr": 0.7,
387
+ "ks": 3.0
388
+ },
389
+ "20": {
390
+ "cr": 0.5,
391
+ "ks": 3.0
392
+ },
393
+ "21": {
394
+ "cr": 0.5,
395
+ "ks": 3.0
396
+ },
397
+ "22": {
398
+ "cr": 0.5,
399
+ "ks": 3.0
400
+ },
401
+ "23": {
402
+ "cr": 0.5,
403
+ "ks": 3.0
404
+ },
405
+ "24": {
406
+ "cr": 0.5,
407
+ "ks": 3.0
408
+ },
409
+ "25": {
410
+ "cr": 0.5,
411
+ "ks": 3.0
412
+ },
413
+ "26": {
414
+ "cr": 0.5,
415
+ "ks": 3.0
416
+ },
417
+ "27": {
418
+ "cr": 0.5,
419
+ "ks": 3.0
420
+ },
421
+ "28": {
422
+ "cr": 0.5,
423
+ "ks": 3.0
424
+ },
425
+ "29": {
426
+ "cr": 0.5,
427
+ "ks": 3.0
428
+ },
429
+ "3": {
430
+ "cr": 0.6,
431
+ "ks": 3.0
432
+ },
433
+ "30": {
434
+ "cr": 0.5,
435
+ "ks": 3.0
436
+ },
437
+ "31": {
438
+ "cr": 0.5,
439
+ "ks": 3.0
440
+ },
441
+ "32": {
442
+ "cr": 0.5,
443
+ "ks": 3.0
444
+ },
445
+ "33": {
446
+ "cr": 0.5,
447
+ "ks": 3.0
448
+ },
449
+ "34": {
450
+ "cr": 0.5,
451
+ "ks": 3.0
452
+ },
453
+ "35": {
454
+ "cr": 0.5,
455
+ "ks": 3.0
456
+ },
457
+ "36": {
458
+ "cr": 0.5,
459
+ "ks": 3.0
460
+ },
461
+ "37": {
462
+ "cr": 0.5,
463
+ "ks": 3.0
464
+ },
465
+ "38": {
466
+ "cr": 0.5,
467
+ "ks": 3.0
468
+ },
469
+ "39": {
470
+ "cr": 0.5,
471
+ "ks": 3.0
472
+ },
473
+ "4": {
474
+ "cr": 0.55,
475
+ "ks": 3.0
476
+ },
477
+ "5": {
478
+ "cr": 0.5,
479
+ "ks": 3.0
480
+ },
481
+ "6": {
482
+ "cr": 0.5,
483
+ "ks": 3.0
484
+ },
485
+ "7": {
486
+ "cr": 0.5,
487
+ "ks": 3.0
488
+ },
489
+ "8": {
490
+ "cr": 0.5,
491
+ "ks": 3.0
492
+ },
493
+ "9": {
494
+ "cr": 0.5,
495
+ "ks": 3.0
496
+ }
497
+ },
498
+ "self_attn.k_proj": {
499
+ "0": {
500
+ "cr": 0.05,
501
+ "ks": 3.0
502
+ },
503
+ "1": {
504
+ "cr": 0.05,
505
+ "ks": 2.1
506
+ },
507
+ "10": {
508
+ "cr": 0.05,
509
+ "ks": 3.0
510
+ },
511
+ "11": {
512
+ "cr": 0.05,
513
+ "ks": 3.0
514
+ },
515
+ "12": {
516
+ "cr": 0.05,
517
+ "ks": 3.0
518
+ },
519
+ "13": {
520
+ "cr": 0.05,
521
+ "ks": 2.6
522
+ },
523
+ "14": {
524
+ "cr": 0.05,
525
+ "ks": 2.9
526
+ },
527
+ "15": {
528
+ "cr": 0.05,
529
+ "ks": 2.9
530
+ },
531
+ "16": {
532
+ "cr": 0.05,
533
+ "ks": 2.6
534
+ },
535
+ "17": {
536
+ "cr": 0.05,
537
+ "ks": 2.6
538
+ },
539
+ "18": {
540
+ "cr": 0.05,
541
+ "ks": 2.6
542
+ },
543
+ "19": {
544
+ "cr": 0.05,
545
+ "ks": 2.2
546
+ },
547
+ "2": {
548
+ "cr": 0.05,
549
+ "ks": 2.6
550
+ },
551
+ "20": {
552
+ "cr": 0.05,
553
+ "ks": 2.3
554
+ },
555
+ "21": {
556
+ "cr": 0.05,
557
+ "ks": 2.1
558
+ },
559
+ "22": {
560
+ "cr": 0.05,
561
+ "ks": 2.2
562
+ },
563
+ "23": {
564
+ "cr": 0.05,
565
+ "ks": 2.4
566
+ },
567
+ "24": {
568
+ "cr": 0.05,
569
+ "ks": 2.2
570
+ },
571
+ "25": {
572
+ "cr": 0.05,
573
+ "ks": 2.6
574
+ },
575
+ "26": {
576
+ "cr": 0.05,
577
+ "ks": 2.6
578
+ },
579
+ "27": {
580
+ "cr": 0.05,
581
+ "ks": 2.6
582
+ },
583
+ "28": {
584
+ "cr": 0.05,
585
+ "ks": 2.3
586
+ },
587
+ "29": {
588
+ "cr": 0.05,
589
+ "ks": 2.6
590
+ },
591
+ "3": {
592
+ "cr": 0.05,
593
+ "ks": 2.6
594
+ },
595
+ "30": {
596
+ "cr": 0.05,
597
+ "ks": 2.6
598
+ },
599
+ "31": {
600
+ "cr": 0.05,
601
+ "ks": 2.6
602
+ },
603
+ "32": {
604
+ "cr": 0.05,
605
+ "ks": 3.0
606
+ },
607
+ "33": {
608
+ "cr": 0.05,
609
+ "ks": 2.6
610
+ },
611
+ "34": {
612
+ "cr": 0.05,
613
+ "ks": 2.6
614
+ },
615
+ "35": {
616
+ "cr": 0.05,
617
+ "ks": 2.6
618
+ },
619
+ "36": {
620
+ "cr": 0.05,
621
+ "ks": 2.9
622
+ },
623
+ "37": {
624
+ "cr": 0.05,
625
+ "ks": 2.6
626
+ },
627
+ "38": {
628
+ "cr": 0.05,
629
+ "ks": 2.6
630
+ },
631
+ "39": {
632
+ "cr": 0.05,
633
+ "ks": 3.0
634
+ },
635
+ "4": {
636
+ "cr": 0.05,
637
+ "ks": 3.0
638
+ },
639
+ "5": {
640
+ "cr": 0.05,
641
+ "ks": 2.9
642
+ },
643
+ "6": {
644
+ "cr": 0.05,
645
+ "ks": 3.0
646
+ },
647
+ "7": {
648
+ "cr": 0.05,
649
+ "ks": 3.0
650
+ },
651
+ "8": {
652
+ "cr": 0.05,
653
+ "ks": 3.0
654
+ },
655
+ "9": {
656
+ "cr": 0.05,
657
+ "ks": 2.9
658
+ }
659
+ },
660
+ "self_attn.o_proj": {
661
+ "0": {
662
+ "cr": 0.35,
663
+ "ks": 1.9
664
+ },
665
+ "1": {
666
+ "cr": 0.35,
667
+ "ks": 2.2
668
+ },
669
+ "10": {
670
+ "cr": 0.25,
671
+ "ks": 2.4
672
+ },
673
+ "11": {
674
+ "cr": 0.2,
675
+ "ks": 2.3
676
+ },
677
+ "12": {
678
+ "cr": 0.3,
679
+ "ks": 2.2
680
+ },
681
+ "13": {
682
+ "cr": 0.25,
683
+ "ks": 2.3
684
+ },
685
+ "14": {
686
+ "cr": 0.25,
687
+ "ks": 2.2
688
+ },
689
+ "15": {
690
+ "cr": 0.2,
691
+ "ks": 2.3
692
+ },
693
+ "16": {
694
+ "cr": 0.2,
695
+ "ks": 2.2
696
+ },
697
+ "17": {
698
+ "cr": 0.25,
699
+ "ks": 2.2
700
+ },
701
+ "18": {
702
+ "cr": 0.2,
703
+ "ks": 2.3
704
+ },
705
+ "19": {
706
+ "cr": 0.2,
707
+ "ks": 2.3
708
+ },
709
+ "2": {
710
+ "cr": 0.25,
711
+ "ks": 2.3
712
+ },
713
+ "20": {
714
+ "cr": 0.15,
715
+ "ks": 2.4
716
+ },
717
+ "21": {
718
+ "cr": 0.15,
719
+ "ks": 2.3
720
+ },
721
+ "22": {
722
+ "cr": 0.15,
723
+ "ks": 2.3
724
+ },
725
+ "23": {
726
+ "cr": 0.15,
727
+ "ks": 2.4
728
+ },
729
+ "24": {
730
+ "cr": 0.15,
731
+ "ks": 2.3
732
+ },
733
+ "25": {
734
+ "cr": 0.2,
735
+ "ks": 2.3
736
+ },
737
+ "26": {
738
+ "cr": 0.1,
739
+ "ks": 2.4
740
+ },
741
+ "27": {
742
+ "cr": 0.15,
743
+ "ks": 2.3
744
+ },
745
+ "28": {
746
+ "cr": 0.2,
747
+ "ks": 2.3
748
+ },
749
+ "29": {
750
+ "cr": 0.1,
751
+ "ks": 2.4
752
+ },
753
+ "3": {
754
+ "cr": 0.25,
755
+ "ks": 2.2
756
+ },
757
+ "30": {
758
+ "cr": 0.1,
759
+ "ks": 2.5
760
+ },
761
+ "31": {
762
+ "cr": 0.05,
763
+ "ks": 2.5
764
+ },
765
+ "32": {
766
+ "cr": 0.05,
767
+ "ks": 2.8
768
+ },
769
+ "33": {
770
+ "cr": 0.05,
771
+ "ks": 2.7
772
+ },
773
+ "34": {
774
+ "cr": 0.05,
775
+ "ks": 2.7
776
+ },
777
+ "35": {
778
+ "cr": 0.05,
779
+ "ks": 2.7
780
+ },
781
+ "36": {
782
+ "cr": 0.05,
783
+ "ks": 2.8
784
+ },
785
+ "37": {
786
+ "cr": 0.05,
787
+ "ks": 2.7
788
+ },
789
+ "38": {
790
+ "cr": 0.1,
791
+ "ks": 2.6
792
+ },
793
+ "39": {
794
+ "cr": 0.05,
795
+ "ks": 2.7
796
+ },
797
+ "4": {
798
+ "cr": 0.25,
799
+ "ks": 2.3
800
+ },
801
+ "5": {
802
+ "cr": 0.3,
803
+ "ks": 2.2
804
+ },
805
+ "6": {
806
+ "cr": 0.35,
807
+ "ks": 2.2
808
+ },
809
+ "7": {
810
+ "cr": 0.2,
811
+ "ks": 2.4
812
+ },
813
+ "8": {
814
+ "cr": 0.2,
815
+ "ks": 2.4
816
+ },
817
+ "9": {
818
+ "cr": 0.3,
819
+ "ks": 2.3
820
+ }
821
+ },
822
+ "self_attn.q_proj": {
823
+ "0": {
824
+ "cr": 0.2,
825
+ "ks": 2.0
826
+ },
827
+ "1": {
828
+ "cr": 0.45,
829
+ "ks": 1.8
830
+ },
831
+ "10": {
832
+ "cr": 0.25,
833
+ "ks": 2.4
834
+ },
835
+ "11": {
836
+ "cr": 0.25,
837
+ "ks": 2.3
838
+ },
839
+ "12": {
840
+ "cr": 0.3,
841
+ "ks": 2.3
842
+ },
843
+ "13": {
844
+ "cr": 0.3,
845
+ "ks": 2.3
846
+ },
847
+ "14": {
848
+ "cr": 0.3,
849
+ "ks": 2.3
850
+ },
851
+ "15": {
852
+ "cr": 0.3,
853
+ "ks": 2.3
854
+ },
855
+ "16": {
856
+ "cr": 0.35,
857
+ "ks": 2.3
858
+ },
859
+ "17": {
860
+ "cr": 0.4,
861
+ "ks": 2.2
862
+ },
863
+ "18": {
864
+ "cr": 0.35,
865
+ "ks": 2.2
866
+ },
867
+ "19": {
868
+ "cr": 0.35,
869
+ "ks": 2.2
870
+ },
871
+ "2": {
872
+ "cr": 0.35,
873
+ "ks": 2.1
874
+ },
875
+ "20": {
876
+ "cr": 0.35,
877
+ "ks": 2.3
878
+ },
879
+ "21": {
880
+ "cr": 0.35,
881
+ "ks": 2.2
882
+ },
883
+ "22": {
884
+ "cr": 0.3,
885
+ "ks": 2.3
886
+ },
887
+ "23": {
888
+ "cr": 0.35,
889
+ "ks": 2.2
890
+ },
891
+ "24": {
892
+ "cr": 0.35,
893
+ "ks": 2.2
894
+ },
895
+ "25": {
896
+ "cr": 0.3,
897
+ "ks": 2.3
898
+ },
899
+ "26": {
900
+ "cr": 0.3,
901
+ "ks": 2.3
902
+ },
903
+ "27": {
904
+ "cr": 0.25,
905
+ "ks": 2.3
906
+ },
907
+ "28": {
908
+ "cr": 0.3,
909
+ "ks": 2.3
910
+ },
911
+ "29": {
912
+ "cr": 0.2,
913
+ "ks": 2.4
914
+ },
915
+ "3": {
916
+ "cr": 0.35,
917
+ "ks": 2.2
918
+ },
919
+ "30": {
920
+ "cr": 0.2,
921
+ "ks": 2.4
922
+ },
923
+ "31": {
924
+ "cr": 0.15,
925
+ "ks": 2.5
926
+ },
927
+ "32": {
928
+ "cr": 0.1,
929
+ "ks": 2.5
930
+ },
931
+ "33": {
932
+ "cr": 0.15,
933
+ "ks": 2.5
934
+ },
935
+ "34": {
936
+ "cr": 0.2,
937
+ "ks": 2.5
938
+ },
939
+ "35": {
940
+ "cr": 0.2,
941
+ "ks": 2.4
942
+ },
943
+ "36": {
944
+ "cr": 0.15,
945
+ "ks": 2.5
946
+ },
947
+ "37": {
948
+ "cr": 0.2,
949
+ "ks": 2.4
950
+ },
951
+ "38": {
952
+ "cr": 0.2,
953
+ "ks": 2.4
954
+ },
955
+ "39": {
956
+ "cr": 0.1,
957
+ "ks": 2.4
958
+ },
959
+ "4": {
960
+ "cr": 0.3,
961
+ "ks": 2.2
962
+ },
963
+ "5": {
964
+ "cr": 0.3,
965
+ "ks": 2.2
966
+ },
967
+ "6": {
968
+ "cr": 0.3,
969
+ "ks": 2.3
970
+ },
971
+ "7": {
972
+ "cr": 0.2,
973
+ "ks": 2.4
974
+ },
975
+ "8": {
976
+ "cr": 0.2,
977
+ "ks": 2.4
978
+ },
979
+ "9": {
980
+ "cr": 0.25,
981
+ "ks": 2.3
982
+ }
983
+ },
984
+ "self_attn.v_proj": {
985
+ "0": {
986
+ "cr": 0.05,
987
+ "ks": 1.8
988
+ },
989
+ "1": {
990
+ "cr": 0.05,
991
+ "ks": 2.0
992
+ },
993
+ "10": {
994
+ "cr": 0.05,
995
+ "ks": 3.0
996
+ },
997
+ "11": {
998
+ "cr": 0.05,
999
+ "ks": 3.0
1000
+ },
1001
+ "12": {
1002
+ "cr": 0.05,
1003
+ "ks": 3.0
1004
+ },
1005
+ "13": {
1006
+ "cr": 0.05,
1007
+ "ks": 3.0
1008
+ },
1009
+ "14": {
1010
+ "cr": 0.05,
1011
+ "ks": 3.0
1012
+ },
1013
+ "15": {
1014
+ "cr": 0.05,
1015
+ "ks": 3.0
1016
+ },
1017
+ "16": {
1018
+ "cr": 0.05,
1019
+ "ks": 3.0
1020
+ },
1021
+ "17": {
1022
+ "cr": 0.05,
1023
+ "ks": 3.0
1024
+ },
1025
+ "18": {
1026
+ "cr": 0.05,
1027
+ "ks": 2.9
1028
+ },
1029
+ "19": {
1030
+ "cr": 0.05,
1031
+ "ks": 3.0
1032
+ },
1033
+ "2": {
1034
+ "cr": 0.05,
1035
+ "ks": 2.6
1036
+ },
1037
+ "20": {
1038
+ "cr": 0.05,
1039
+ "ks": 3.0
1040
+ },
1041
+ "21": {
1042
+ "cr": 0.05,
1043
+ "ks": 3.0
1044
+ },
1045
+ "22": {
1046
+ "cr": 0.05,
1047
+ "ks": 3.0
1048
+ },
1049
+ "23": {
1050
+ "cr": 0.05,
1051
+ "ks": 3.0
1052
+ },
1053
+ "24": {
1054
+ "cr": 0.05,
1055
+ "ks": 3.0
1056
+ },
1057
+ "25": {
1058
+ "cr": 0.05,
1059
+ "ks": 3.0
1060
+ },
1061
+ "26": {
1062
+ "cr": 0.05,
1063
+ "ks": 3.0
1064
+ },
1065
+ "27": {
1066
+ "cr": 0.05,
1067
+ "ks": 3.0
1068
+ },
1069
+ "28": {
1070
+ "cr": 0.05,
1071
+ "ks": 3.0
1072
+ },
1073
+ "29": {
1074
+ "cr": 0.05,
1075
+ "ks": 3.0
1076
+ },
1077
+ "3": {
1078
+ "cr": 0.05,
1079
+ "ks": 2.9
1080
+ },
1081
+ "30": {
1082
+ "cr": 0.05,
1083
+ "ks": 3.0
1084
+ },
1085
+ "31": {
1086
+ "cr": 0.05,
1087
+ "ks": 3.0
1088
+ },
1089
+ "32": {
1090
+ "cr": 0.05,
1091
+ "ks": 3.0
1092
+ },
1093
+ "33": {
1094
+ "cr": 0.05,
1095
+ "ks": 3.0
1096
+ },
1097
+ "34": {
1098
+ "cr": 0.05,
1099
+ "ks": 3.0
1100
+ },
1101
+ "35": {
1102
+ "cr": 0.05,
1103
+ "ks": 3.0
1104
+ },
1105
+ "36": {
1106
+ "cr": 0.05,
1107
+ "ks": 3.0
1108
+ },
1109
+ "37": {
1110
+ "cr": 0.05,
1111
+ "ks": 3.0
1112
+ },
1113
+ "38": {
1114
+ "cr": 0.05,
1115
+ "ks": 3.0
1116
+ },
1117
+ "39": {
1118
+ "cr": 0.05,
1119
+ "ks": 3.0
1120
+ },
1121
+ "4": {
1122
+ "cr": 0.05,
1123
+ "ks": 3.0
1124
+ },
1125
+ "5": {
1126
+ "cr": 0.05,
1127
+ "ks": 2.6
1128
+ },
1129
+ "6": {
1130
+ "cr": 0.05,
1131
+ "ks": 2.9
1132
+ },
1133
+ "7": {
1134
+ "cr": 0.05,
1135
+ "ks": 3.0
1136
+ },
1137
+ "8": {
1138
+ "cr": 0.05,
1139
+ "ks": 3.0
1140
+ },
1141
+ "9": {
1142
+ "cr": 0.05,
1143
+ "ks": 2.9
1144
+ }
1145
+ }
1146
+ },
1147
+ "dtype": "bfloat16",
1148
+ "eos_token_id": 151645,
1149
+ "head_dim": 128,
1150
+ "hidden_act": "silu",
1151
+ "hidden_size": 5120,
1152
+ "initializer_range": 0.02,
1153
+ "intermediate_size": 17408,
1154
+ "layer_types": [
1155
+ "full_attention",
1156
+ "full_attention",
1157
+ "full_attention",
1158
+ "full_attention",
1159
+ "full_attention",
1160
+ "full_attention",
1161
+ "full_attention",
1162
+ "full_attention",
1163
+ "full_attention",
1164
+ "full_attention",
1165
+ "full_attention",
1166
+ "full_attention",
1167
+ "full_attention",
1168
+ "full_attention",
1169
+ "full_attention",
1170
+ "full_attention",
1171
+ "full_attention",
1172
+ "full_attention",
1173
+ "full_attention",
1174
+ "full_attention",
1175
+ "full_attention",
1176
+ "full_attention",
1177
+ "full_attention",
1178
+ "full_attention",
1179
+ "full_attention",
1180
+ "full_attention",
1181
+ "full_attention",
1182
+ "full_attention",
1183
+ "full_attention",
1184
+ "full_attention",
1185
+ "full_attention",
1186
+ "full_attention",
1187
+ "full_attention",
1188
+ "full_attention",
1189
+ "full_attention",
1190
+ "full_attention",
1191
+ "full_attention",
1192
+ "full_attention",
1193
+ "full_attention",
1194
+ "full_attention"
1195
+ ],
1196
+ "max_position_embeddings": 40960,
1197
+ "max_window_layers": 40,
1198
+ "model_type": "qwen3",
1199
+ "num_attention_heads": 40,
1200
+ "num_hidden_layers": 40,
1201
+ "num_key_value_heads": 8,
1202
+ "rms_norm_eps": 1e-06,
1203
+ "rope_scaling": null,
1204
+ "rope_theta": 1000000,
1205
+ "sliding_window": null,
1206
+ "tie_word_embeddings": false,
1207
+ "torch_dtype": "bfloat16",
1208
+ "transformers_version": "4.55.4",
1209
+ "use_cache": true,
1210
+ "use_sliding_window": false,
1211
+ "vocab_size": 151936
1212
+ }
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "compression_path": "./cr_nested_qwen3_14b_04.json",
4
+ "do_sample": true,
5
+ "eos_token_id": [
6
+ 151645,
7
+ 151643
8
+ ],
9
+ "pad_token_id": 151643,
10
+ "temperature": 0.6,
11
+ "top_k": 20,
12
+ "top_p": 0.95,
13
+ "transformers_version": "4.55.4"
14
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1732e449de6609603d9ef58c80555f97e74f12ba8209e0e58f2e4151476ca201
3
+ size 4958473624
model-00002-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff208d5f7b326e8e8754159023f5a3fd87b8ef96bc73017a651607d5229431d4
3
+ size 4915609816
model-00003-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca2a56ae2d8666971d7cea5e7c3d0b4de128d0bb2de462f8dcc4dc99830451c1
3
+ size 4995724264
model-00004-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b22552857f6b519c00641ab76061da8493bae210219d440cebbd491274df6812
3
+ size 4987752248
model-00005-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9b641f5729b67209a055c102bc4a4ae1f751d2849459e440cfc190bdd64697b
3
+ size 4976915664
model-00006-of-00006.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65ee33c5780852a7a335523842284bde2ba30f053365b9ee8654ae1114a5dd11
3
+ size 3476921440
model.safetensors.index.json ADDED
@@ -0,0 +1,731 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 14155656192,
4
+ "total_size": 28311312384
5
+ },
6
+ "weight_map": {
7
+ "lm_head.weight": "model-00006-of-00006.safetensors",
8
+ "model.embed_tokens.weight": "model-00001-of-00006.safetensors",
9
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
10
+ "model.layers.0.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
11
+ "model.layers.0.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
12
+ "model.layers.0.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
13
+ "model.layers.0.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
14
+ "model.layers.0.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
15
+ "model.layers.0.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
16
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
17
+ "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
18
+ "model.layers.0.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
19
+ "model.layers.0.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
20
+ "model.layers.0.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
21
+ "model.layers.0.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
22
+ "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
23
+ "model.layers.0.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
24
+ "model.layers.0.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
25
+ "model.layers.0.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
26
+ "model.layers.0.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
27
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
28
+ "model.layers.1.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
29
+ "model.layers.1.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
30
+ "model.layers.1.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
31
+ "model.layers.1.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
32
+ "model.layers.1.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
33
+ "model.layers.1.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
34
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
35
+ "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
36
+ "model.layers.1.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
37
+ "model.layers.1.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
38
+ "model.layers.1.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
39
+ "model.layers.1.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
40
+ "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
41
+ "model.layers.1.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
42
+ "model.layers.1.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
43
+ "model.layers.1.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
44
+ "model.layers.1.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
45
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
46
+ "model.layers.10.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
47
+ "model.layers.10.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
48
+ "model.layers.10.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
49
+ "model.layers.10.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
50
+ "model.layers.10.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
51
+ "model.layers.10.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
52
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
53
+ "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
54
+ "model.layers.10.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
55
+ "model.layers.10.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
56
+ "model.layers.10.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
57
+ "model.layers.10.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
58
+ "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
59
+ "model.layers.10.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
60
+ "model.layers.10.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
61
+ "model.layers.10.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
62
+ "model.layers.10.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
63
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
64
+ "model.layers.11.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
65
+ "model.layers.11.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
66
+ "model.layers.11.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
67
+ "model.layers.11.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
68
+ "model.layers.11.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
69
+ "model.layers.11.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
70
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
71
+ "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
72
+ "model.layers.11.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
73
+ "model.layers.11.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
74
+ "model.layers.11.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
75
+ "model.layers.11.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
76
+ "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
77
+ "model.layers.11.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
78
+ "model.layers.11.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
79
+ "model.layers.11.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
80
+ "model.layers.11.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
81
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00006.safetensors",
82
+ "model.layers.12.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
83
+ "model.layers.12.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
84
+ "model.layers.12.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
85
+ "model.layers.12.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
86
+ "model.layers.12.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
87
+ "model.layers.12.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
88
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
89
+ "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
90
+ "model.layers.12.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
91
+ "model.layers.12.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
92
+ "model.layers.12.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
93
+ "model.layers.12.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
94
+ "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
95
+ "model.layers.12.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
96
+ "model.layers.12.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
97
+ "model.layers.12.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
98
+ "model.layers.12.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
99
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00006.safetensors",
100
+ "model.layers.13.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
101
+ "model.layers.13.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
102
+ "model.layers.13.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
103
+ "model.layers.13.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
104
+ "model.layers.13.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
105
+ "model.layers.13.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
106
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
107
+ "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
108
+ "model.layers.13.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
109
+ "model.layers.13.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
110
+ "model.layers.13.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
111
+ "model.layers.13.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
112
+ "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
113
+ "model.layers.13.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
114
+ "model.layers.13.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
115
+ "model.layers.13.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
116
+ "model.layers.13.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
117
+ "model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
118
+ "model.layers.14.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
119
+ "model.layers.14.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
120
+ "model.layers.14.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
121
+ "model.layers.14.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
122
+ "model.layers.14.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
123
+ "model.layers.14.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
124
+ "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
125
+ "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
126
+ "model.layers.14.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
127
+ "model.layers.14.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
128
+ "model.layers.14.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
129
+ "model.layers.14.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
130
+ "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
131
+ "model.layers.14.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
132
+ "model.layers.14.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
133
+ "model.layers.14.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
134
+ "model.layers.14.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
135
+ "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
136
+ "model.layers.15.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
137
+ "model.layers.15.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
138
+ "model.layers.15.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
139
+ "model.layers.15.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
140
+ "model.layers.15.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
141
+ "model.layers.15.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
142
+ "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
143
+ "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
144
+ "model.layers.15.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
145
+ "model.layers.15.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
146
+ "model.layers.15.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
147
+ "model.layers.15.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
148
+ "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
149
+ "model.layers.15.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
150
+ "model.layers.15.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
151
+ "model.layers.15.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
152
+ "model.layers.15.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
153
+ "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
154
+ "model.layers.16.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
155
+ "model.layers.16.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
156
+ "model.layers.16.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
157
+ "model.layers.16.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
158
+ "model.layers.16.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
159
+ "model.layers.16.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
160
+ "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
161
+ "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
162
+ "model.layers.16.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
163
+ "model.layers.16.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
164
+ "model.layers.16.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
165
+ "model.layers.16.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
166
+ "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
167
+ "model.layers.16.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
168
+ "model.layers.16.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
169
+ "model.layers.16.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
170
+ "model.layers.16.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
171
+ "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
172
+ "model.layers.17.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
173
+ "model.layers.17.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
174
+ "model.layers.17.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
175
+ "model.layers.17.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
176
+ "model.layers.17.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
177
+ "model.layers.17.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
178
+ "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
179
+ "model.layers.17.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
180
+ "model.layers.17.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
181
+ "model.layers.17.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
182
+ "model.layers.17.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
183
+ "model.layers.17.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
184
+ "model.layers.17.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
185
+ "model.layers.17.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
186
+ "model.layers.17.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
187
+ "model.layers.17.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
188
+ "model.layers.17.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
189
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors",
190
+ "model.layers.18.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
191
+ "model.layers.18.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
192
+ "model.layers.18.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
193
+ "model.layers.18.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
194
+ "model.layers.18.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
195
+ "model.layers.18.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
196
+ "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
197
+ "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
198
+ "model.layers.18.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
199
+ "model.layers.18.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
200
+ "model.layers.18.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
201
+ "model.layers.18.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
202
+ "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
203
+ "model.layers.18.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
204
+ "model.layers.18.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
205
+ "model.layers.18.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
206
+ "model.layers.18.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
207
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors",
208
+ "model.layers.19.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
209
+ "model.layers.19.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
210
+ "model.layers.19.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
211
+ "model.layers.19.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
212
+ "model.layers.19.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
213
+ "model.layers.19.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
214
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
215
+ "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
216
+ "model.layers.19.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
217
+ "model.layers.19.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
218
+ "model.layers.19.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
219
+ "model.layers.19.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
220
+ "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
221
+ "model.layers.19.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
222
+ "model.layers.19.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
223
+ "model.layers.19.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
224
+ "model.layers.19.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
225
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
226
+ "model.layers.2.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
227
+ "model.layers.2.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
228
+ "model.layers.2.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
229
+ "model.layers.2.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
230
+ "model.layers.2.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
231
+ "model.layers.2.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
232
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
233
+ "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
234
+ "model.layers.2.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
235
+ "model.layers.2.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
236
+ "model.layers.2.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
237
+ "model.layers.2.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
238
+ "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
239
+ "model.layers.2.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
240
+ "model.layers.2.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
241
+ "model.layers.2.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
242
+ "model.layers.2.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
243
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00006.safetensors",
244
+ "model.layers.20.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
245
+ "model.layers.20.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
246
+ "model.layers.20.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
247
+ "model.layers.20.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
248
+ "model.layers.20.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
249
+ "model.layers.20.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
250
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
251
+ "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
252
+ "model.layers.20.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
253
+ "model.layers.20.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
254
+ "model.layers.20.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
255
+ "model.layers.20.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
256
+ "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
257
+ "model.layers.20.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
258
+ "model.layers.20.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
259
+ "model.layers.20.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
260
+ "model.layers.20.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
261
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00006.safetensors",
262
+ "model.layers.21.mlp.down_proj_u.weight": "model-00003-of-00006.safetensors",
263
+ "model.layers.21.mlp.down_proj_v.weight": "model-00003-of-00006.safetensors",
264
+ "model.layers.21.mlp.gate_proj_u.weight": "model-00003-of-00006.safetensors",
265
+ "model.layers.21.mlp.gate_proj_v.weight": "model-00003-of-00006.safetensors",
266
+ "model.layers.21.mlp.up_proj_u.weight": "model-00003-of-00006.safetensors",
267
+ "model.layers.21.mlp.up_proj_v.weight": "model-00003-of-00006.safetensors",
268
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
269
+ "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
270
+ "model.layers.21.self_attn.k_proj_u.weight": "model-00003-of-00006.safetensors",
271
+ "model.layers.21.self_attn.k_proj_v.weight": "model-00003-of-00006.safetensors",
272
+ "model.layers.21.self_attn.o_proj_u.weight": "model-00003-of-00006.safetensors",
273
+ "model.layers.21.self_attn.o_proj_v.weight": "model-00003-of-00006.safetensors",
274
+ "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
275
+ "model.layers.21.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
276
+ "model.layers.21.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
277
+ "model.layers.21.self_attn.v_proj_u.weight": "model-00003-of-00006.safetensors",
278
+ "model.layers.21.self_attn.v_proj_v.weight": "model-00003-of-00006.safetensors",
279
+ "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
280
+ "model.layers.22.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
281
+ "model.layers.22.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
282
+ "model.layers.22.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
283
+ "model.layers.22.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
284
+ "model.layers.22.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
285
+ "model.layers.22.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
286
+ "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
287
+ "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
288
+ "model.layers.22.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
289
+ "model.layers.22.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
290
+ "model.layers.22.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
291
+ "model.layers.22.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
292
+ "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
293
+ "model.layers.22.self_attn.q_proj_u.weight": "model-00003-of-00006.safetensors",
294
+ "model.layers.22.self_attn.q_proj_v.weight": "model-00003-of-00006.safetensors",
295
+ "model.layers.22.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
296
+ "model.layers.22.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
297
+ "model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors",
298
+ "model.layers.23.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
299
+ "model.layers.23.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
300
+ "model.layers.23.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
301
+ "model.layers.23.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
302
+ "model.layers.23.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
303
+ "model.layers.23.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
304
+ "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
305
+ "model.layers.23.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
306
+ "model.layers.23.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
307
+ "model.layers.23.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
308
+ "model.layers.23.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
309
+ "model.layers.23.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
310
+ "model.layers.23.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
311
+ "model.layers.23.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
312
+ "model.layers.23.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
313
+ "model.layers.23.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
314
+ "model.layers.23.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
315
+ "model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors",
316
+ "model.layers.24.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
317
+ "model.layers.24.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
318
+ "model.layers.24.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
319
+ "model.layers.24.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
320
+ "model.layers.24.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
321
+ "model.layers.24.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
322
+ "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
323
+ "model.layers.24.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
324
+ "model.layers.24.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
325
+ "model.layers.24.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
326
+ "model.layers.24.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
327
+ "model.layers.24.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
328
+ "model.layers.24.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
329
+ "model.layers.24.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
330
+ "model.layers.24.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
331
+ "model.layers.24.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
332
+ "model.layers.24.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
333
+ "model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors",
334
+ "model.layers.25.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
335
+ "model.layers.25.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
336
+ "model.layers.25.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
337
+ "model.layers.25.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
338
+ "model.layers.25.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
339
+ "model.layers.25.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
340
+ "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
341
+ "model.layers.25.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
342
+ "model.layers.25.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
343
+ "model.layers.25.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
344
+ "model.layers.25.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
345
+ "model.layers.25.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
346
+ "model.layers.25.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
347
+ "model.layers.25.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
348
+ "model.layers.25.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
349
+ "model.layers.25.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
350
+ "model.layers.25.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
351
+ "model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors",
352
+ "model.layers.26.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
353
+ "model.layers.26.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
354
+ "model.layers.26.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
355
+ "model.layers.26.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
356
+ "model.layers.26.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
357
+ "model.layers.26.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
358
+ "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
359
+ "model.layers.26.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
360
+ "model.layers.26.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
361
+ "model.layers.26.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
362
+ "model.layers.26.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
363
+ "model.layers.26.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
364
+ "model.layers.26.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
365
+ "model.layers.26.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
366
+ "model.layers.26.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
367
+ "model.layers.26.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
368
+ "model.layers.26.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
369
+ "model.layers.27.input_layernorm.weight": "model-00004-of-00006.safetensors",
370
+ "model.layers.27.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
371
+ "model.layers.27.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
372
+ "model.layers.27.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
373
+ "model.layers.27.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
374
+ "model.layers.27.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
375
+ "model.layers.27.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
376
+ "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
377
+ "model.layers.27.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
378
+ "model.layers.27.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
379
+ "model.layers.27.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
380
+ "model.layers.27.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
381
+ "model.layers.27.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
382
+ "model.layers.27.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
383
+ "model.layers.27.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
384
+ "model.layers.27.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
385
+ "model.layers.27.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
386
+ "model.layers.27.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
387
+ "model.layers.28.input_layernorm.weight": "model-00004-of-00006.safetensors",
388
+ "model.layers.28.mlp.down_proj_u.weight": "model-00004-of-00006.safetensors",
389
+ "model.layers.28.mlp.down_proj_v.weight": "model-00004-of-00006.safetensors",
390
+ "model.layers.28.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
391
+ "model.layers.28.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
392
+ "model.layers.28.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
393
+ "model.layers.28.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
394
+ "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
395
+ "model.layers.28.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
396
+ "model.layers.28.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
397
+ "model.layers.28.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
398
+ "model.layers.28.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
399
+ "model.layers.28.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
400
+ "model.layers.28.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
401
+ "model.layers.28.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
402
+ "model.layers.28.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
403
+ "model.layers.28.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
404
+ "model.layers.28.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
405
+ "model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors",
406
+ "model.layers.29.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
407
+ "model.layers.29.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
408
+ "model.layers.29.mlp.gate_proj_u.weight": "model-00004-of-00006.safetensors",
409
+ "model.layers.29.mlp.gate_proj_v.weight": "model-00004-of-00006.safetensors",
410
+ "model.layers.29.mlp.up_proj_u.weight": "model-00004-of-00006.safetensors",
411
+ "model.layers.29.mlp.up_proj_v.weight": "model-00004-of-00006.safetensors",
412
+ "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
413
+ "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
414
+ "model.layers.29.self_attn.k_proj_u.weight": "model-00004-of-00006.safetensors",
415
+ "model.layers.29.self_attn.k_proj_v.weight": "model-00004-of-00006.safetensors",
416
+ "model.layers.29.self_attn.o_proj_u.weight": "model-00004-of-00006.safetensors",
417
+ "model.layers.29.self_attn.o_proj_v.weight": "model-00004-of-00006.safetensors",
418
+ "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
419
+ "model.layers.29.self_attn.q_proj_u.weight": "model-00004-of-00006.safetensors",
420
+ "model.layers.29.self_attn.q_proj_v.weight": "model-00004-of-00006.safetensors",
421
+ "model.layers.29.self_attn.v_proj_u.weight": "model-00004-of-00006.safetensors",
422
+ "model.layers.29.self_attn.v_proj_v.weight": "model-00004-of-00006.safetensors",
423
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
424
+ "model.layers.3.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
425
+ "model.layers.3.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
426
+ "model.layers.3.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
427
+ "model.layers.3.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
428
+ "model.layers.3.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
429
+ "model.layers.3.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
430
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
431
+ "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
432
+ "model.layers.3.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
433
+ "model.layers.3.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
434
+ "model.layers.3.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
435
+ "model.layers.3.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
436
+ "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
437
+ "model.layers.3.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
438
+ "model.layers.3.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
439
+ "model.layers.3.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
440
+ "model.layers.3.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
441
+ "model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors",
442
+ "model.layers.30.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
443
+ "model.layers.30.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
444
+ "model.layers.30.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
445
+ "model.layers.30.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
446
+ "model.layers.30.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
447
+ "model.layers.30.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
448
+ "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
449
+ "model.layers.30.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
450
+ "model.layers.30.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
451
+ "model.layers.30.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
452
+ "model.layers.30.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
453
+ "model.layers.30.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
454
+ "model.layers.30.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
455
+ "model.layers.30.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
456
+ "model.layers.30.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
457
+ "model.layers.30.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
458
+ "model.layers.30.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
459
+ "model.layers.31.input_layernorm.weight": "model-00005-of-00006.safetensors",
460
+ "model.layers.31.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
461
+ "model.layers.31.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
462
+ "model.layers.31.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
463
+ "model.layers.31.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
464
+ "model.layers.31.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
465
+ "model.layers.31.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
466
+ "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
467
+ "model.layers.31.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
468
+ "model.layers.31.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
469
+ "model.layers.31.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
470
+ "model.layers.31.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
471
+ "model.layers.31.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
472
+ "model.layers.31.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
473
+ "model.layers.31.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
474
+ "model.layers.31.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
475
+ "model.layers.31.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
476
+ "model.layers.31.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
477
+ "model.layers.32.input_layernorm.weight": "model-00005-of-00006.safetensors",
478
+ "model.layers.32.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
479
+ "model.layers.32.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
480
+ "model.layers.32.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
481
+ "model.layers.32.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
482
+ "model.layers.32.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
483
+ "model.layers.32.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
484
+ "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
485
+ "model.layers.32.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
486
+ "model.layers.32.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
487
+ "model.layers.32.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
488
+ "model.layers.32.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
489
+ "model.layers.32.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
490
+ "model.layers.32.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
491
+ "model.layers.32.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
492
+ "model.layers.32.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
493
+ "model.layers.32.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
494
+ "model.layers.32.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
495
+ "model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors",
496
+ "model.layers.33.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
497
+ "model.layers.33.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
498
+ "model.layers.33.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
499
+ "model.layers.33.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
500
+ "model.layers.33.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
501
+ "model.layers.33.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
502
+ "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
503
+ "model.layers.33.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
504
+ "model.layers.33.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
505
+ "model.layers.33.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
506
+ "model.layers.33.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
507
+ "model.layers.33.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
508
+ "model.layers.33.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
509
+ "model.layers.33.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
510
+ "model.layers.33.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
511
+ "model.layers.33.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
512
+ "model.layers.33.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
513
+ "model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors",
514
+ "model.layers.34.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
515
+ "model.layers.34.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
516
+ "model.layers.34.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
517
+ "model.layers.34.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
518
+ "model.layers.34.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
519
+ "model.layers.34.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
520
+ "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
521
+ "model.layers.34.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
522
+ "model.layers.34.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
523
+ "model.layers.34.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
524
+ "model.layers.34.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
525
+ "model.layers.34.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
526
+ "model.layers.34.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
527
+ "model.layers.34.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
528
+ "model.layers.34.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
529
+ "model.layers.34.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
530
+ "model.layers.34.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
531
+ "model.layers.35.input_layernorm.weight": "model-00005-of-00006.safetensors",
532
+ "model.layers.35.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
533
+ "model.layers.35.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
534
+ "model.layers.35.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
535
+ "model.layers.35.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
536
+ "model.layers.35.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
537
+ "model.layers.35.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
538
+ "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
539
+ "model.layers.35.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
540
+ "model.layers.35.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
541
+ "model.layers.35.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
542
+ "model.layers.35.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
543
+ "model.layers.35.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
544
+ "model.layers.35.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
545
+ "model.layers.35.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
546
+ "model.layers.35.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
547
+ "model.layers.35.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
548
+ "model.layers.35.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
549
+ "model.layers.36.input_layernorm.weight": "model-00005-of-00006.safetensors",
550
+ "model.layers.36.mlp.down_proj_u.weight": "model-00005-of-00006.safetensors",
551
+ "model.layers.36.mlp.down_proj_v.weight": "model-00005-of-00006.safetensors",
552
+ "model.layers.36.mlp.gate_proj_u.weight": "model-00005-of-00006.safetensors",
553
+ "model.layers.36.mlp.gate_proj_v.weight": "model-00005-of-00006.safetensors",
554
+ "model.layers.36.mlp.up_proj_u.weight": "model-00005-of-00006.safetensors",
555
+ "model.layers.36.mlp.up_proj_v.weight": "model-00005-of-00006.safetensors",
556
+ "model.layers.36.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
557
+ "model.layers.36.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
558
+ "model.layers.36.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
559
+ "model.layers.36.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
560
+ "model.layers.36.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
561
+ "model.layers.36.self_attn.o_proj_v.weight": "model-00005-of-00006.safetensors",
562
+ "model.layers.36.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
563
+ "model.layers.36.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
564
+ "model.layers.36.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
565
+ "model.layers.36.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
566
+ "model.layers.36.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
567
+ "model.layers.37.input_layernorm.weight": "model-00006-of-00006.safetensors",
568
+ "model.layers.37.mlp.down_proj_u.weight": "model-00006-of-00006.safetensors",
569
+ "model.layers.37.mlp.down_proj_v.weight": "model-00006-of-00006.safetensors",
570
+ "model.layers.37.mlp.gate_proj_u.weight": "model-00006-of-00006.safetensors",
571
+ "model.layers.37.mlp.gate_proj_v.weight": "model-00006-of-00006.safetensors",
572
+ "model.layers.37.mlp.up_proj_u.weight": "model-00006-of-00006.safetensors",
573
+ "model.layers.37.mlp.up_proj_v.weight": "model-00006-of-00006.safetensors",
574
+ "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
575
+ "model.layers.37.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
576
+ "model.layers.37.self_attn.k_proj_u.weight": "model-00005-of-00006.safetensors",
577
+ "model.layers.37.self_attn.k_proj_v.weight": "model-00005-of-00006.safetensors",
578
+ "model.layers.37.self_attn.o_proj_u.weight": "model-00005-of-00006.safetensors",
579
+ "model.layers.37.self_attn.o_proj_v.weight": "model-00006-of-00006.safetensors",
580
+ "model.layers.37.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
581
+ "model.layers.37.self_attn.q_proj_u.weight": "model-00005-of-00006.safetensors",
582
+ "model.layers.37.self_attn.q_proj_v.weight": "model-00005-of-00006.safetensors",
583
+ "model.layers.37.self_attn.v_proj_u.weight": "model-00005-of-00006.safetensors",
584
+ "model.layers.37.self_attn.v_proj_v.weight": "model-00005-of-00006.safetensors",
585
+ "model.layers.38.input_layernorm.weight": "model-00006-of-00006.safetensors",
586
+ "model.layers.38.mlp.down_proj_u.weight": "model-00006-of-00006.safetensors",
587
+ "model.layers.38.mlp.down_proj_v.weight": "model-00006-of-00006.safetensors",
588
+ "model.layers.38.mlp.gate_proj_u.weight": "model-00006-of-00006.safetensors",
589
+ "model.layers.38.mlp.gate_proj_v.weight": "model-00006-of-00006.safetensors",
590
+ "model.layers.38.mlp.up_proj_u.weight": "model-00006-of-00006.safetensors",
591
+ "model.layers.38.mlp.up_proj_v.weight": "model-00006-of-00006.safetensors",
592
+ "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
593
+ "model.layers.38.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
594
+ "model.layers.38.self_attn.k_proj_u.weight": "model-00006-of-00006.safetensors",
595
+ "model.layers.38.self_attn.k_proj_v.weight": "model-00006-of-00006.safetensors",
596
+ "model.layers.38.self_attn.o_proj_u.weight": "model-00006-of-00006.safetensors",
597
+ "model.layers.38.self_attn.o_proj_v.weight": "model-00006-of-00006.safetensors",
598
+ "model.layers.38.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
599
+ "model.layers.38.self_attn.q_proj_u.weight": "model-00006-of-00006.safetensors",
600
+ "model.layers.38.self_attn.q_proj_v.weight": "model-00006-of-00006.safetensors",
601
+ "model.layers.38.self_attn.v_proj_u.weight": "model-00006-of-00006.safetensors",
602
+ "model.layers.38.self_attn.v_proj_v.weight": "model-00006-of-00006.safetensors",
603
+ "model.layers.39.input_layernorm.weight": "model-00006-of-00006.safetensors",
604
+ "model.layers.39.mlp.down_proj_u.weight": "model-00006-of-00006.safetensors",
605
+ "model.layers.39.mlp.down_proj_v.weight": "model-00006-of-00006.safetensors",
606
+ "model.layers.39.mlp.gate_proj_u.weight": "model-00006-of-00006.safetensors",
607
+ "model.layers.39.mlp.gate_proj_v.weight": "model-00006-of-00006.safetensors",
608
+ "model.layers.39.mlp.up_proj_u.weight": "model-00006-of-00006.safetensors",
609
+ "model.layers.39.mlp.up_proj_v.weight": "model-00006-of-00006.safetensors",
610
+ "model.layers.39.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
611
+ "model.layers.39.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
612
+ "model.layers.39.self_attn.k_proj_u.weight": "model-00006-of-00006.safetensors",
613
+ "model.layers.39.self_attn.k_proj_v.weight": "model-00006-of-00006.safetensors",
614
+ "model.layers.39.self_attn.o_proj_u.weight": "model-00006-of-00006.safetensors",
615
+ "model.layers.39.self_attn.o_proj_v.weight": "model-00006-of-00006.safetensors",
616
+ "model.layers.39.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
617
+ "model.layers.39.self_attn.q_proj_u.weight": "model-00006-of-00006.safetensors",
618
+ "model.layers.39.self_attn.q_proj_v.weight": "model-00006-of-00006.safetensors",
619
+ "model.layers.39.self_attn.v_proj_u.weight": "model-00006-of-00006.safetensors",
620
+ "model.layers.39.self_attn.v_proj_v.weight": "model-00006-of-00006.safetensors",
621
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
622
+ "model.layers.4.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
623
+ "model.layers.4.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
624
+ "model.layers.4.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
625
+ "model.layers.4.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
626
+ "model.layers.4.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
627
+ "model.layers.4.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
628
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
629
+ "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
630
+ "model.layers.4.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
631
+ "model.layers.4.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
632
+ "model.layers.4.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
633
+ "model.layers.4.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
634
+ "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
635
+ "model.layers.4.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
636
+ "model.layers.4.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
637
+ "model.layers.4.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
638
+ "model.layers.4.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
639
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00006.safetensors",
640
+ "model.layers.5.mlp.down_proj_u.weight": "model-00001-of-00006.safetensors",
641
+ "model.layers.5.mlp.down_proj_v.weight": "model-00001-of-00006.safetensors",
642
+ "model.layers.5.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
643
+ "model.layers.5.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
644
+ "model.layers.5.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
645
+ "model.layers.5.mlp.up_proj_v.weight": "model-00001-of-00006.safetensors",
646
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
647
+ "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
648
+ "model.layers.5.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
649
+ "model.layers.5.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
650
+ "model.layers.5.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
651
+ "model.layers.5.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
652
+ "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
653
+ "model.layers.5.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
654
+ "model.layers.5.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
655
+ "model.layers.5.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
656
+ "model.layers.5.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
657
+ "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
658
+ "model.layers.6.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
659
+ "model.layers.6.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
660
+ "model.layers.6.mlp.gate_proj_u.weight": "model-00001-of-00006.safetensors",
661
+ "model.layers.6.mlp.gate_proj_v.weight": "model-00001-of-00006.safetensors",
662
+ "model.layers.6.mlp.up_proj_u.weight": "model-00001-of-00006.safetensors",
663
+ "model.layers.6.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
664
+ "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
665
+ "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
666
+ "model.layers.6.self_attn.k_proj_u.weight": "model-00001-of-00006.safetensors",
667
+ "model.layers.6.self_attn.k_proj_v.weight": "model-00001-of-00006.safetensors",
668
+ "model.layers.6.self_attn.o_proj_u.weight": "model-00001-of-00006.safetensors",
669
+ "model.layers.6.self_attn.o_proj_v.weight": "model-00001-of-00006.safetensors",
670
+ "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
671
+ "model.layers.6.self_attn.q_proj_u.weight": "model-00001-of-00006.safetensors",
672
+ "model.layers.6.self_attn.q_proj_v.weight": "model-00001-of-00006.safetensors",
673
+ "model.layers.6.self_attn.v_proj_u.weight": "model-00001-of-00006.safetensors",
674
+ "model.layers.6.self_attn.v_proj_v.weight": "model-00001-of-00006.safetensors",
675
+ "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
676
+ "model.layers.7.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
677
+ "model.layers.7.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
678
+ "model.layers.7.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
679
+ "model.layers.7.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
680
+ "model.layers.7.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
681
+ "model.layers.7.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
682
+ "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
683
+ "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
684
+ "model.layers.7.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
685
+ "model.layers.7.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
686
+ "model.layers.7.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
687
+ "model.layers.7.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
688
+ "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
689
+ "model.layers.7.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
690
+ "model.layers.7.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
691
+ "model.layers.7.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
692
+ "model.layers.7.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
693
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
694
+ "model.layers.8.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
695
+ "model.layers.8.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
696
+ "model.layers.8.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
697
+ "model.layers.8.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
698
+ "model.layers.8.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
699
+ "model.layers.8.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
700
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
701
+ "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
702
+ "model.layers.8.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
703
+ "model.layers.8.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
704
+ "model.layers.8.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
705
+ "model.layers.8.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
706
+ "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
707
+ "model.layers.8.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
708
+ "model.layers.8.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
709
+ "model.layers.8.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
710
+ "model.layers.8.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
711
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
712
+ "model.layers.9.mlp.down_proj_u.weight": "model-00002-of-00006.safetensors",
713
+ "model.layers.9.mlp.down_proj_v.weight": "model-00002-of-00006.safetensors",
714
+ "model.layers.9.mlp.gate_proj_u.weight": "model-00002-of-00006.safetensors",
715
+ "model.layers.9.mlp.gate_proj_v.weight": "model-00002-of-00006.safetensors",
716
+ "model.layers.9.mlp.up_proj_u.weight": "model-00002-of-00006.safetensors",
717
+ "model.layers.9.mlp.up_proj_v.weight": "model-00002-of-00006.safetensors",
718
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
719
+ "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
720
+ "model.layers.9.self_attn.k_proj_u.weight": "model-00002-of-00006.safetensors",
721
+ "model.layers.9.self_attn.k_proj_v.weight": "model-00002-of-00006.safetensors",
722
+ "model.layers.9.self_attn.o_proj_u.weight": "model-00002-of-00006.safetensors",
723
+ "model.layers.9.self_attn.o_proj_v.weight": "model-00002-of-00006.safetensors",
724
+ "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
725
+ "model.layers.9.self_attn.q_proj_u.weight": "model-00002-of-00006.safetensors",
726
+ "model.layers.9.self_attn.q_proj_v.weight": "model-00002-of-00006.safetensors",
727
+ "model.layers.9.self_attn.v_proj_u.weight": "model-00002-of-00006.safetensors",
728
+ "model.layers.9.self_attn.v_proj_v.weight": "model-00002-of-00006.safetensors",
729
+ "model.norm.weight": "model-00006-of-00006.safetensors"
730
+ }
731
+ }
modeling_qwen3_swift_sparse.py ADDED
@@ -0,0 +1,697 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
2
+ # This file was automatically generated from src/transformers/models/qwen3/modular_qwen3.py.
3
+ # Do NOT edit this file manually as any edits will be overwritten by the generation of
4
+ # the file from the modular. If any change should be done, please apply the change to the
5
+ # modular_qwen3.py file directly. One of our CI enforces this.
6
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
7
+ # coding=utf-8
8
+ # Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the "License");
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at
13
+ #
14
+ # http://www.apache.org/licenses/LICENSE-2.0
15
+ #
16
+ # Unless required by applicable law or agreed to in writing, software
17
+ # distributed under the License is distributed on an "AS IS" BASIS,
18
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
+ # See the License for the specific language governing permissions and
20
+ # limitations under the License.
21
+
22
+ from typing import Callable, Optional, Union
23
+ from sparse import SparseLinear
24
+ import torch
25
+ from torch import nn
26
+
27
+ from transformers.activations import ACT2FN
28
+ from transformers.cache_utils import Cache, DynamicCache
29
+ from transformers.generation import GenerationMixin
30
+ from transformers.integrations import use_kernel_forward_from_hub
31
+ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
32
+ from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
33
+ from transformers.modeling_layers import (
34
+ GenericForQuestionAnswering,
35
+ GenericForSequenceClassification,
36
+ GenericForTokenClassification,
37
+ GradientCheckpointingLayer,
38
+ )
39
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
40
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
41
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
42
+ from transformers.processing_utils import Unpack
43
+ from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple
44
+ from transformers.utils.deprecation import deprecate_kwarg
45
+ from transformers.utils.generic import check_model_inputs
46
+ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
47
+
48
+ def get_cr(d1, d2, k, sparsity):
49
+ vanilla = d1*d2
50
+ d = d1 * k
51
+ c = sparsity * d2
52
+ return 1 - (d+c) / vanilla
53
+
54
+ def get_k_and_sparsity(cr, d1, d2, ks_ratio):
55
+ vanilla = d1*d2
56
+ k = (1-cr) * d1 * d2 / (d1 + d2 / ks_ratio)
57
+ sparsity = k / ks_ratio
58
+ return int(k), int(sparsity)
59
+
60
+ @use_kernel_forward_from_hub("RMSNorm")
61
+ class Qwen3RMSNorm(nn.Module):
62
+ def __init__(self, hidden_size, eps: float = 1e-6) -> None:
63
+ """
64
+ Qwen3RMSNorm is equivalent to T5LayerNorm
65
+ """
66
+ super().__init__()
67
+ self.weight = nn.Parameter(torch.ones(hidden_size))
68
+ self.variance_epsilon = eps
69
+
70
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
71
+ input_dtype = hidden_states.dtype
72
+ hidden_states = hidden_states.to(torch.float32)
73
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
74
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
75
+ return self.weight * hidden_states.to(input_dtype)
76
+
77
+ def extra_repr(self):
78
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
79
+
80
+
81
+ class Qwen3MLP(nn.Module):
82
+ def __init__(self, config):
83
+ super().__init__()
84
+ self.config = config
85
+ self.hidden_size = config.hidden_size
86
+ self.intermediate_size = config.intermediate_size
87
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
88
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
89
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
90
+ self.act_fn = ACT2FN[config.hidden_act]
91
+
92
+ def forward(self, x):
93
+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
94
+ return down_proj
95
+
96
+ class Qwen3MLP_SVD(nn.Module):
97
+ def __init__(self, config, layer_idx):
98
+ super().__init__()
99
+ self.config = config
100
+ self.hidden_size = config.hidden_size
101
+ self.intermediate_size = config.intermediate_size
102
+ gate_cr = config.compression_ratio["mlp.gate_proj"][str(layer_idx)]['cr']
103
+ up_cr = config.compression_ratio["mlp.up_proj"][str(layer_idx)]['cr']
104
+ down_cr = config.compression_ratio["mlp.down_proj"][str(layer_idx)]['cr']
105
+ gate_ks = config.compression_ratio["mlp.gate_proj"][str(layer_idx)]['ks']
106
+ up_ks = config.compression_ratio["mlp.up_proj"][str(layer_idx)]['ks']
107
+ down_ks = config.compression_ratio["mlp.down_proj"][str(layer_idx)]['ks']
108
+ self.gate_low_rank, _ = get_k_and_sparsity(gate_cr, self.hidden_size, self.intermediate_size, gate_ks)
109
+ self.up_low_rank, _ = get_k_and_sparsity(up_cr, self.hidden_size, self.intermediate_size, up_ks)
110
+ self.down_low_rank, _ = get_k_and_sparsity(down_cr, self.intermediate_size, self.hidden_size, down_ks)
111
+ self.gate_proj_u = nn.Linear(self.hidden_size, self.gate_low_rank, bias=False)
112
+ self.gate_proj_v = SparseLinear(self.gate_low_rank, self.intermediate_size, bias=False)
113
+ self.up_proj_u = nn.Linear(self.hidden_size, self.up_low_rank, bias=False)
114
+ self.up_proj_v = SparseLinear(self.up_low_rank, self.intermediate_size, bias=False)
115
+ self.down_proj_u = nn.Linear(self.intermediate_size, self.down_low_rank, bias=False)
116
+ self.down_proj_v = SparseLinear(self.down_low_rank, self.hidden_size, bias=False)
117
+ self.act_fn = ACT2FN[config.hidden_act]
118
+
119
+ def forward(self, x):
120
+ gate = self.gate_proj_v(self.gate_proj_u(x))
121
+ up = self.up_proj_v(self.up_proj_u(x))
122
+ activated = self.act_fn(gate)
123
+ down = self.down_proj_v(self.down_proj_u(activated * up))
124
+ return down
125
+
126
+ def rotate_half(x):
127
+ """Rotates half the hidden dims of the input."""
128
+ x1 = x[..., : x.shape[-1] // 2]
129
+ x2 = x[..., x.shape[-1] // 2 :]
130
+ return torch.cat((-x2, x1), dim=-1)
131
+
132
+
133
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
134
+ """Applies Rotary Position Embedding to the query and key tensors.
135
+
136
+ Args:
137
+ q (`torch.Tensor`): The query tensor.
138
+ k (`torch.Tensor`): The key tensor.
139
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
140
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
141
+ position_ids (`torch.Tensor`, *optional*):
142
+ Deprecated and unused.
143
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
144
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
145
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
146
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
147
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
148
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
149
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
150
+ Returns:
151
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
152
+ """
153
+ cos = cos.unsqueeze(unsqueeze_dim)
154
+ sin = sin.unsqueeze(unsqueeze_dim)
155
+ q_embed = (q * cos) + (rotate_half(q) * sin)
156
+ k_embed = (k * cos) + (rotate_half(k) * sin)
157
+ return q_embed, k_embed
158
+
159
+
160
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
161
+ """
162
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
163
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
164
+ """
165
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
166
+ if n_rep == 1:
167
+ return hidden_states
168
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
169
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
170
+
171
+
172
+ def eager_attention_forward(
173
+ module: nn.Module,
174
+ query: torch.Tensor,
175
+ key: torch.Tensor,
176
+ value: torch.Tensor,
177
+ attention_mask: Optional[torch.Tensor],
178
+ scaling: float,
179
+ dropout: float = 0.0,
180
+ **kwargs: Unpack[TransformersKwargs],
181
+ ):
182
+ key_states = repeat_kv(key, module.num_key_value_groups)
183
+ value_states = repeat_kv(value, module.num_key_value_groups)
184
+
185
+ attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
186
+ if attention_mask is not None:
187
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
188
+ attn_weights = attn_weights + causal_mask
189
+
190
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
191
+ attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
192
+ attn_output = torch.matmul(attn_weights, value_states)
193
+ attn_output = attn_output.transpose(1, 2).contiguous()
194
+
195
+ return attn_output, attn_weights
196
+
197
+
198
+ class Qwen3Attention(nn.Module):
199
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
200
+
201
+ def __init__(self, config: Qwen3Config, layer_idx: int):
202
+ super().__init__()
203
+ self.config = config
204
+ self.layer_idx = layer_idx
205
+ self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
206
+ self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
207
+ self.scaling = self.head_dim**-0.5
208
+ self.attention_dropout = config.attention_dropout
209
+ self.is_causal = True
210
+
211
+ self.q_proj = nn.Linear(
212
+ config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
213
+ )
214
+ self.k_proj = nn.Linear(
215
+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
216
+ )
217
+ self.v_proj = nn.Linear(
218
+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
219
+ )
220
+ self.o_proj = nn.Linear(
221
+ config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
222
+ )
223
+ self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # unlike olmo, only on the head dim!
224
+ self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # thus post q_norm does not need reshape
225
+ self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None
226
+
227
+ @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
228
+ def forward(
229
+ self,
230
+ hidden_states: torch.Tensor,
231
+ position_embeddings: tuple[torch.Tensor, torch.Tensor],
232
+ attention_mask: Optional[torch.Tensor],
233
+ past_key_values: Optional[Cache] = None,
234
+ cache_position: Optional[torch.LongTensor] = None,
235
+ **kwargs: Unpack[FlashAttentionKwargs],
236
+ ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
237
+ input_shape = hidden_states.shape[:-1]
238
+ hidden_shape = (*input_shape, -1, self.head_dim)
239
+
240
+ query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
241
+ key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
242
+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
243
+
244
+ cos, sin = position_embeddings
245
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
246
+
247
+ if past_key_values is not None:
248
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
249
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
250
+ key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
251
+
252
+ attention_interface: Callable = eager_attention_forward
253
+ if self.config._attn_implementation != "eager":
254
+ attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
255
+
256
+ attn_output, attn_weights = attention_interface(
257
+ self,
258
+ query_states,
259
+ key_states,
260
+ value_states,
261
+ attention_mask,
262
+ dropout=0.0 if not self.training else self.attention_dropout,
263
+ scaling=self.scaling,
264
+ sliding_window=self.sliding_window, # diff with Llama
265
+ **kwargs,
266
+ )
267
+
268
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
269
+ attn_output = self.o_proj(attn_output)
270
+ return attn_output, attn_weights
271
+
272
+
273
+ class Qwen3Attention_SVD(nn.Module):
274
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
275
+
276
+ def __init__(self, config: Qwen3Config, layer_idx: int):
277
+ super().__init__()
278
+ self.config = config
279
+ self.layer_idx = layer_idx
280
+ self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
281
+ self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
282
+ self.scaling = self.head_dim**-0.5
283
+ self.attention_dropout = config.attention_dropout
284
+ self.is_causal = True
285
+
286
+ q_cr = config.compression_ratio["self_attn.q_proj"][str(layer_idx)]['cr']
287
+ k_cr = config.compression_ratio["self_attn.k_proj"][str(layer_idx)]['cr']
288
+ v_cr = config.compression_ratio["self_attn.v_proj"][str(layer_idx)]['cr']
289
+ o_cr = config.compression_ratio["self_attn.o_proj"][str(layer_idx)]['cr']
290
+
291
+ q_ks = config.compression_ratio["self_attn.q_proj"][str(layer_idx)]['ks']
292
+ k_ks = config.compression_ratio["self_attn.k_proj"][str(layer_idx)]['ks']
293
+ v_ks = config.compression_ratio["self_attn.v_proj"][str(layer_idx)]['ks']
294
+ o_ks = config.compression_ratio["self_attn.o_proj"][str(layer_idx)]['ks']
295
+
296
+ self.q_low_rank, _ = get_k_and_sparsity(q_cr,
297
+ config.hidden_size,
298
+ config.num_attention_heads * self.head_dim,
299
+ q_ks)
300
+ self.k_low_rank, _ = get_k_and_sparsity(k_cr,
301
+ config.hidden_size,
302
+ config.num_key_value_heads * self.head_dim,
303
+ k_ks)
304
+ self.v_low_rank, _ = get_k_and_sparsity(v_cr,
305
+ config.hidden_size,
306
+ config.num_key_value_heads * self.head_dim,
307
+ v_ks)
308
+ self.o_low_rank, _ = get_k_and_sparsity(o_cr,
309
+ config.num_attention_heads * self.head_dim,
310
+ config.hidden_size,
311
+ o_ks)
312
+
313
+
314
+ self.q_proj_u = nn.Linear(
315
+ config.hidden_size, self.q_low_rank, bias=config.attention_bias
316
+ )
317
+ self.q_proj_v = SparseLinear(
318
+ self.q_low_rank, config.num_attention_heads * self.head_dim, bias=config.attention_bias
319
+ )
320
+ self.k_proj_u = nn.Linear(
321
+ config.hidden_size, self.k_low_rank, bias=config.attention_bias
322
+ )
323
+ self.k_proj_v = SparseLinear(
324
+ self.k_low_rank, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
325
+ )
326
+ self.v_proj_u = nn.Linear(
327
+ config.hidden_size, self.v_low_rank, bias=config.attention_bias
328
+ )
329
+ self.v_proj_v = SparseLinear(
330
+ self.v_low_rank, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
331
+ )
332
+ self.o_proj_u = nn.Linear(
333
+ config.num_attention_heads * self.head_dim, self.o_low_rank, bias=config.attention_bias
334
+ )
335
+ self.o_proj_v = SparseLinear(
336
+ self.o_low_rank, config.hidden_size, bias=config.attention_bias
337
+ )
338
+ self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # unlike olmo, only on the head dim!
339
+ self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) # thus post q_norm does not need reshape
340
+ self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None
341
+
342
+ def q_proj(self, x):
343
+ return self.q_proj_v(self.q_proj_u(x))
344
+
345
+ def k_proj(self, x):
346
+ return self.k_proj_v(self.k_proj_u(x))
347
+
348
+ def v_proj(self, x):
349
+ return self.v_proj_v(self.v_proj_u(x))
350
+
351
+ def o_proj(self, x):
352
+ return self.o_proj_v(self.o_proj_u(x))
353
+
354
+ def forward(
355
+ self,
356
+ hidden_states: torch.Tensor,
357
+ position_embeddings: tuple[torch.Tensor, torch.Tensor],
358
+ attention_mask: Optional[torch.Tensor],
359
+ past_key_value: Optional[Cache] = None,
360
+ cache_position: Optional[torch.LongTensor] = None,
361
+ **kwargs: Unpack[TransformersKwargs],
362
+ ) -> tuple[torch.Tensor, torch.Tensor]:
363
+ input_shape = hidden_states.shape[:-1]
364
+ hidden_shape = (*input_shape, -1, self.head_dim)
365
+ query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
366
+ key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
367
+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
368
+
369
+ cos, sin = position_embeddings
370
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
371
+
372
+ if past_key_value is not None:
373
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
374
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
375
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
376
+
377
+ attention_interface: Callable = eager_attention_forward
378
+ if self.config._attn_implementation != "eager":
379
+ attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
380
+
381
+ attn_output, attn_weights = attention_interface(
382
+ self,
383
+ query_states,
384
+ key_states,
385
+ value_states,
386
+ attention_mask,
387
+ dropout=0.0 if not self.training else self.attention_dropout,
388
+ scaling=self.scaling,
389
+ sliding_window=self.sliding_window,
390
+ **kwargs,
391
+ )
392
+
393
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
394
+ attn_output = self.o_proj(attn_output)
395
+ return attn_output, attn_weights
396
+
397
+ class Qwen3DecoderLayer(GradientCheckpointingLayer):
398
+ def __init__(self, config: Qwen3Config, layer_idx: int):
399
+ super().__init__()
400
+ self.hidden_size = config.hidden_size
401
+
402
+ self.self_attn = Qwen3Attention_SVD(config=config, layer_idx=layer_idx)
403
+
404
+ self.mlp = Qwen3MLP_SVD(config, layer_idx=layer_idx)
405
+ self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
406
+ self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
407
+ self.attention_type = config.layer_types[layer_idx]
408
+
409
+ @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
410
+ def forward(
411
+ self,
412
+ hidden_states: torch.Tensor,
413
+ attention_mask: Optional[torch.Tensor] = None,
414
+ position_ids: Optional[torch.LongTensor] = None,
415
+ past_key_values: Optional[Cache] = None,
416
+ use_cache: Optional[bool] = False,
417
+ cache_position: Optional[torch.LongTensor] = None,
418
+ position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None, # necessary, but kept here for BC
419
+ **kwargs: Unpack[TransformersKwargs],
420
+ ) -> torch.Tensor:
421
+ residual = hidden_states
422
+ hidden_states = self.input_layernorm(hidden_states)
423
+ # Self Attention
424
+ hidden_states, _ = self.self_attn(
425
+ hidden_states=hidden_states,
426
+ attention_mask=attention_mask,
427
+ position_ids=position_ids,
428
+ past_key_values=past_key_values,
429
+ use_cache=use_cache,
430
+ cache_position=cache_position,
431
+ position_embeddings=position_embeddings,
432
+ **kwargs,
433
+ )
434
+ hidden_states = residual + hidden_states
435
+
436
+ # Fully Connected
437
+ residual = hidden_states
438
+ hidden_states = self.post_attention_layernorm(hidden_states)
439
+ hidden_states = self.mlp(hidden_states)
440
+ hidden_states = residual + hidden_states
441
+ return hidden_states
442
+
443
+
444
+ @auto_docstring
445
+ class Qwen3PreTrainedModel(PreTrainedModel):
446
+ config: Qwen3Config
447
+ base_model_prefix = "model"
448
+ supports_gradient_checkpointing = True
449
+ _no_split_modules = ["Qwen3DecoderLayer"]
450
+ _skip_keys_device_placement = ["past_key_values"]
451
+ _supports_flash_attn = True
452
+ _supports_sdpa = True
453
+ _supports_flex_attn = True
454
+
455
+ _can_compile_fullgraph = True
456
+ _supports_attention_backend = True
457
+ _can_record_outputs = {
458
+ "hidden_states": Qwen3DecoderLayer,
459
+ "attentions": Qwen3Attention,
460
+ }
461
+
462
+
463
+ class Qwen3RotaryEmbedding(nn.Module):
464
+ inv_freq: torch.Tensor # fix linting for `register_buffer`
465
+
466
+ def __init__(self, config: Qwen3Config, device=None):
467
+ super().__init__()
468
+ # BC: "rope_type" was originally "type"
469
+ if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
470
+ self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
471
+ else:
472
+ self.rope_type = "default"
473
+ self.max_seq_len_cached = config.max_position_embeddings
474
+ self.original_max_seq_len = config.max_position_embeddings
475
+
476
+ self.config = config
477
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
478
+
479
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
480
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
481
+ self.original_inv_freq = self.inv_freq
482
+
483
+ @torch.no_grad()
484
+ @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
485
+ def forward(self, x, position_ids):
486
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
487
+ position_ids_expanded = position_ids[:, None, :].float()
488
+
489
+ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
490
+ with torch.autocast(device_type=device_type, enabled=False): # Force float32
491
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
492
+ emb = torch.cat((freqs, freqs), dim=-1)
493
+ cos = emb.cos() * self.attention_scaling
494
+ sin = emb.sin() * self.attention_scaling
495
+
496
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
497
+
498
+
499
+ @auto_docstring
500
+ class Qwen3Model(Qwen3PreTrainedModel):
501
+ def __init__(self, config: Qwen3Config):
502
+ super().__init__(config)
503
+ self.padding_idx = config.pad_token_id
504
+ self.vocab_size = config.vocab_size
505
+
506
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
507
+ self.layers = nn.ModuleList(
508
+ [Qwen3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
509
+ )
510
+ self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
511
+ self.rotary_emb = Qwen3RotaryEmbedding(config=config)
512
+ self.gradient_checkpointing = False
513
+ self.has_sliding_layers = "sliding_attention" in self.config.layer_types
514
+
515
+ # Initialize weights and apply final processing
516
+ self.post_init()
517
+
518
+ @check_model_inputs
519
+ @auto_docstring
520
+ def forward(
521
+ self,
522
+ input_ids: Optional[torch.LongTensor] = None,
523
+ attention_mask: Optional[torch.Tensor] = None,
524
+ position_ids: Optional[torch.LongTensor] = None,
525
+ past_key_values: Optional[Cache] = None,
526
+ inputs_embeds: Optional[torch.FloatTensor] = None,
527
+ use_cache: Optional[bool] = None,
528
+ cache_position: Optional[torch.LongTensor] = None,
529
+ **kwargs: Unpack[TransformersKwargs],
530
+ ) -> BaseModelOutputWithPast:
531
+ if (input_ids is None) ^ (inputs_embeds is not None):
532
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
533
+
534
+ if inputs_embeds is None:
535
+ inputs_embeds = self.embed_tokens(input_ids)
536
+
537
+ if use_cache and past_key_values is None:
538
+ past_key_values = DynamicCache()
539
+
540
+ if cache_position is None:
541
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
542
+ cache_position = torch.arange(
543
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
544
+ )
545
+
546
+ if position_ids is None:
547
+ position_ids = cache_position.unsqueeze(0)
548
+
549
+ # It may already have been prepared by e.g. `generate`
550
+ if not isinstance(causal_mask_mapping := attention_mask, dict):
551
+ # Prepare mask arguments
552
+ mask_kwargs = {
553
+ "config": self.config,
554
+ "input_embeds": inputs_embeds,
555
+ "attention_mask": attention_mask,
556
+ "cache_position": cache_position,
557
+ "past_key_values": past_key_values,
558
+ "position_ids": position_ids,
559
+ }
560
+ # Create the masks
561
+ causal_mask_mapping = {
562
+ "full_attention": create_causal_mask(**mask_kwargs),
563
+ }
564
+ # The sliding window alternating layers are not always activated depending on the config
565
+ if self.has_sliding_layers:
566
+ causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs)
567
+
568
+ hidden_states = inputs_embeds
569
+
570
+ # create position embeddings to be shared across the decoder layers
571
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
572
+
573
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
574
+ hidden_states = decoder_layer(
575
+ hidden_states,
576
+ attention_mask=causal_mask_mapping[decoder_layer.attention_type],
577
+ position_ids=position_ids,
578
+ past_key_values=past_key_values,
579
+ use_cache=use_cache,
580
+ cache_position=cache_position,
581
+ position_embeddings=position_embeddings,
582
+ **kwargs,
583
+ )
584
+
585
+ hidden_states = self.norm(hidden_states)
586
+ return BaseModelOutputWithPast(
587
+ last_hidden_state=hidden_states,
588
+ past_key_values=past_key_values if use_cache else None,
589
+ )
590
+
591
+
592
+ @auto_docstring
593
+ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin):
594
+ _tied_weights_keys = ["lm_head.weight"]
595
+ _tp_plan = {"lm_head": "colwise_rep"}
596
+ _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
597
+
598
+ def __init__(self, config, compression_path=""):
599
+ if compression_path != "":
600
+ import json
601
+ with open(compression_path, "r") as f:
602
+ cr_nested = json.load(f)
603
+ config.compression_ratio = cr_nested
604
+ super().__init__(config)
605
+ self.model = Qwen3Model(config)
606
+ self.vocab_size = config.vocab_size
607
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
608
+
609
+ # Initialize weights and apply final processing
610
+ self.post_init()
611
+
612
+ @can_return_tuple
613
+ @auto_docstring
614
+ def forward(
615
+ self,
616
+ input_ids: Optional[torch.LongTensor] = None,
617
+ attention_mask: Optional[torch.Tensor] = None,
618
+ position_ids: Optional[torch.LongTensor] = None,
619
+ past_key_values: Optional[Cache] = None,
620
+ inputs_embeds: Optional[torch.FloatTensor] = None,
621
+ labels: Optional[torch.LongTensor] = None,
622
+ use_cache: Optional[bool] = None,
623
+ cache_position: Optional[torch.LongTensor] = None,
624
+ logits_to_keep: Union[int, torch.Tensor] = 0,
625
+ **kwargs: Unpack[TransformersKwargs],
626
+ ) -> CausalLMOutputWithPast:
627
+ r"""
628
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
629
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
630
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
631
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
632
+
633
+ Example:
634
+
635
+ ```python
636
+ >>> from transformers import AutoTokenizer, Qwen3ForCausalLM
637
+
638
+ >>> model = Qwen3ForCausalLM.from_pretrained("Qwen/Qwen3-8B")
639
+ >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")
640
+
641
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
642
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
643
+
644
+ >>> # Generate
645
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
646
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
647
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
648
+ ```"""
649
+ outputs: BaseModelOutputWithPast = self.model(
650
+ input_ids=input_ids,
651
+ attention_mask=attention_mask,
652
+ position_ids=position_ids,
653
+ past_key_values=past_key_values,
654
+ inputs_embeds=inputs_embeds,
655
+ use_cache=use_cache,
656
+ cache_position=cache_position,
657
+ **kwargs,
658
+ )
659
+
660
+ hidden_states = outputs.last_hidden_state
661
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
662
+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
663
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
664
+
665
+ loss = None
666
+ if labels is not None:
667
+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
668
+
669
+ return CausalLMOutputWithPast(
670
+ loss=loss,
671
+ logits=logits,
672
+ past_key_values=outputs.past_key_values,
673
+ hidden_states=outputs.hidden_states,
674
+ attentions=outputs.attentions,
675
+ )
676
+
677
+
678
+ class Qwen3ForSequenceClassification(GenericForSequenceClassification, Qwen3PreTrainedModel):
679
+ pass
680
+
681
+
682
+ class Qwen3ForTokenClassification(GenericForTokenClassification, Qwen3PreTrainedModel):
683
+ pass
684
+
685
+
686
+ class Qwen3ForQuestionAnswering(GenericForQuestionAnswering, Qwen3PreTrainedModel):
687
+ base_model_prefix = "transformer" # For BC, where `transformer` was used instead of `model`
688
+
689
+
690
+ __all__ = [
691
+ "Qwen3ForCausalLM",
692
+ "Qwen3ForQuestionAnswering",
693
+ "Qwen3PreTrainedModel",
694
+ "Qwen3Model",
695
+ "Qwen3ForSequenceClassification",
696
+ "Qwen3ForTokenClassification",
697
+ ]
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a37049ee7f6eb573c3d1553bb683bbb7f73abd5aa8496d0d28a9a1f31af6bdda
3
+ size 33397759628
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4584900ff1f3946432a17798342707a349dffb0d10686270dbedc05cbece08e7
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
3
+ size 11422654
tokenizer_config.json ADDED
@@ -0,0 +1,239 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ }
213
+ },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>"
228
+ ],
229
+ "bos_token": null,
230
+ "clean_up_tokenization_spaces": false,
231
+ "eos_token": "<|im_end|>",
232
+ "errors": "replace",
233
+ "extra_special_tokens": {},
234
+ "model_max_length": 131072,
235
+ "pad_token": "<|endoftext|>",
236
+ "split_special_tokens": false,
237
+ "tokenizer_class": "Qwen2Tokenizer",
238
+ "unk_token": null
239
+ }
trainer_state.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffe85d15eb51dbe0abb0dff0ebdbf34d5bb9e8408de20f29382cedfca71006ea
3
+ size 19386917
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7376421f3f756dc20917568802a3a94d96f1efd1dfeac84af8117e2374b153a9
3
+ size 5368
vocab.json ADDED
The diff for this file is too large to render. See raw diff