Upload folder using huggingface_hub

chat_template.jinja

@@ -1,17 +1,12 @@
-{% macro render_item_list(item_list, tag_name='required') %}
-    {%- if item_list is defined and item_list is iterable and item_list | length > 0 %}
-        {%- if tag_name %}{{- '\n<' ~ tag_name ~ '>' -}}{% endif %}
-            {{- '[' }}
-                {%- for item in item_list -%}
-                    {%- if loop.index > 1 %}{{- ", "}}{% endif -%}
-                    {%- if item is string -%}
-                        {{ "`" ~ item ~ "`" }}
-                    {%- else -%}
-                        {{ item }}
-                    {%- endif -%}
-                {%- endfor -%}
-            {{- ']' }}
-        {%- if tag_name %}{{- '</' ~ tag_name ~ '>' -}}{% endif %}
+{% macro render_extra_keys(json_dict, handled_keys) %}
+    {%- if json_dict is mapping %}
+        {%- for json_key in json_dict if json_key not in handled_keys %}
+            {%- if json_dict[json_key] is mapping %}
+                {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
+            {%- else %}
+                {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
+            {%- endif %}
+        {%- endfor %}
     {%- endif %}
 {% endmacro %}
 
@@ -41,39 +36,30 @@
             {%- set tool = tool.function %}
         {%- endif %}
         {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
-        {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
+        {%- if tool.description is defined %}
+            {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
+        {%- endif %}
         {{- '\n<parameters>' }}
-        {%- for param_name, param_fields in tool.parameters.properties|items %}
-            {{- '\n<parameter>' }}
-            {{- '\n<name>' ~ param_name ~ '</name>' }}
-            {%- if param_fields.type is defined %}
-                {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
-            {%- endif %}
-            {%- if param_fields.description is defined %}
-                {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
-            {%- endif %}
-            {{- render_item_list(param_fields.enum, 'enum') }}
-            {%- set handled_keys = ['type', 'description', 'enum', 'required'] %}
-            {%- for json_key in param_fields.keys() | reject("in", handled_keys) %}
-                {%- set normed_json_key = json_key | replace("-", "_") | replace(" ", "_") | replace("$", "") %}
-                {%- if param_fields[json_key] is mapping %}
-                    {{- '\n<' ~ normed_json_key ~ '>' ~ (param_fields[json_key] | tojson | safe) ~ '</' ~ normed_json_key ~ '>' }}
-                {%- else %}
-                    {{-'\n<' ~ normed_json_key ~ '>' ~ (param_fields[json_key] | string) ~ '</' ~ normed_json_key ~ '>' }}
+        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
+            {%- for param_name, param_fields in tool.parameters.properties|items %}
+                {{- '\n<parameter>' }}
+                {{- '\n<name>' ~ param_name ~ '</name>' }}
+                {%- if param_fields.type is defined %}
+                    {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
                 {%- endif %}
+                {%- if param_fields.description is defined %}
+                    {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
+                {%- endif %}
+                {%- set handled_keys = ['name', 'type', 'description'] %}
+                {{- render_extra_keys(param_fields, handled_keys) }}
+                {{- '\n</parameter>' }}
             {%- endfor %}
-            {{- render_item_list(param_fields.required, 'required') }}
-            {{- '\n</parameter>' }}
-        {%- endfor %}
-        {{- render_item_list(tool.parameters.required, 'required') }}
-        {{- '\n</parameters>' }}
-        {%- if tool.return is defined %}
-            {%- if tool.return is mapping %}
-                {{- '\n<return>' ~ (tool.return | tojson | safe) ~ '</return>' }}
-            {%- else %}
-                {{- '\n<return>' ~ (tool.return | string) ~ '</return>' }}
-            {%- endif %}
         {%- endif %}
+        {% set handled_keys = ['type', 'properties'] %}
+        {{- render_extra_keys(tool.parameters, handled_keys) }}
+        {{- '\n</parameters>' }}
+        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
+        {{- render_extra_keys(tool, handled_keys) }}
         {{- '\n</function>' }}
     {%- endfor %}
     {{- "\n</tools>" }}
@@ -100,7 +86,7 @@
             {%- if tool_call.arguments is defined %}
                 {%- for args_name, args_value in tool_call.arguments|items %}
                     {{- '<parameter=' + args_name + '>\n' }}
-                    {%- set args_value = args_value if args_value is string else args_value | string %}
+                    {%- set args_value = args_value | tojson | safe if args_value is mapping else args_value | string %}
                     {{- args_value }}
                     {{- '\n</parameter>\n' }}
                 {%- endfor %}

config.json

@@ -5,6 +5,7 @@
   "attention_bias": false,
   "attention_dropout": 0.0,
   "decoder_sparse_step": 1,
+  "dtype": "bfloat16",
   "eos_token_id": 151645,
   "head_dim": 128,
   "hidden_act": "silu",
@@ -27,18 +28,17 @@
   "quantization_config": {
     "config_groups": {
       "group_0": {
+        "format": "pack-quantized",
         "input_activations": null,
         "output_activations": null,
-        "targets": [
-          "Linear"
-        ],
+        "targets": ["Linear"],
         "weights": {
           "actorder": null,
           "block_structure": null,
           "dynamic": false,
-          "group_size": 128,
+          "group_size": 32,
           "num_bits": 4,
-          "observer": "minmax",
+          "observer": "mse",
           "observer_kwargs": {},
           "strategy": "group",
           "symmetric": true,
@@ -101,7 +101,10 @@
     ],
     "kv_cache_scheme": null,
     "quant_method": "compressed-tensors",
-    "quantization_status": "compressed"
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.10.3.dev47+ge463fe6"
   },
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
@@ -110,8 +113,7 @@
   "shared_expert_intermediate_size": 0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.55.0.dev0",
+  "transformers_version": "4.56.0.dev0",
   "use_cache": true,
   "use_qk_norm": true,
   "use_sliding_window": false,

generation_config.json

@@ -9,5 +9,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.55.0.dev0"
+  "transformers_version": "4.56.0.dev0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b587e8b52fcd54bffa5484858bdc6fc3a3597e9e3ecbaf8d555d9f2d01913a17
-size 5001524144
+oid sha256:e7cbf0e1cd141d81e3b206aa800b5bd2bd9bfd6f7fd5df65c44c3abf2ec39b19
+size 5001707008

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dbb437f5b2753611f155b1dbb2579c711a423d3ca5e5fc838ca774f4ed0cdfc
-size 5001803304
+oid sha256:2aa05eba343bc30f9e2ad1415bf2108c4ea58107c3beb4717a63a707f2553a0f
+size 5001283696

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23935f50ca0bc2be771d59b681fbdaa76ade736688fa83861c0acc4298c59ee2
-size 5002084152
+oid sha256:722b6bd956d867135f63b8f20ca8114a493522014fe0d511f612a2c765ee6377
+size 5001283912

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28baab4f614bf7bb38514bd453fafde6578c7587062ab381fcaa9ccf07bb171c
-size 1687667728
+oid sha256:e87552d12d99b872b152450fc61d37471b1e99daca2c1a168de14ef924dfbce1
+size 3090232736

recipe.yaml

@@ -1,9 +1,27 @@
-default_stage:
-  default_modifiers:
+quant_stage:
+  quant_modifiers:
     AWQModifier:
+      config_groups:
+        group_0:
+          targets: ['re:.*gate_proj.*', 're:.*up_proj.*', 're:.*down_proj.*', 're:.*k_proj.*',
+            're:.*q_proj.*', 're:.*v_proj.*', 're:.*o_proj.*']
+          weights:
+            num_bits: 4
+            type: int
+            symmetric: true
+            group_size: 32
+            strategy: group
+            block_structure: null
+            dynamic: false
+            actorder: null
+            observer: mse
+            observer_kwargs: {}
+          input_activations: null
+          output_activations: null
+          format: null
       targets: [Linear]
-      ignore: [lm_head, 're:.*mlp.gate$', 're:.*mlp.shared_expert_gate$']
-      scheme: W4A16
+      ignore: [lm_head, model.embed_tokens, 're:.*input_layernorm$', 're:.*post_attention_layernorm$',
+        model.norm, 're:.*shared_experts.*', 're:.*mlp.gate']
       mappings:
       - smooth_layer: re:.*input_layernorm$
         balance_layers: ['re:.*q_proj$', 're:.*k_proj$', 're:.*v_proj$']