update project

.gitattributes

@@ -71,3 +71,35 @@ main_ax650 filter=lfs diff=lfs merge=lfs -text
 main_ax650_api filter=lfs diff=lfs merge=lfs -text
 main_axcl_x86 filter=lfs diff=lfs merge=lfs -text
 main_axcl_x86_api filter=lfs diff=lfs merge=lfs -text
+image.png filter=lfs diff=lfs merge=lfs -text
+image_encoder_1024x1024.axmodel filter=lfs diff=lfs merge=lfs -text
+image_encoder_512x512.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l0_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l10_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l11_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l12_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l13_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l14_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l15_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l16_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l17_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l18_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l19_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l1_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l20_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l21_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l22_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l23_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l24_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l25_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l26_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l27_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l2_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l3_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l4_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l5_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l6_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l7_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l8_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_p128_l9_together.axmodel filter=lfs diff=lfs merge=lfs -text
+llava_qwen2_post.axmodel filter=lfs diff=lfs merge=lfs -text

README.md

@@ -40,160 +40,198 @@ How to Convert LLM from Huggingface to axmodel[TODO]
 |AXCL x86| 233.93 ms (1024x1024)| 779.51 ms (286tokens)| 12.47 tokens/sec|1.4|
 |AX650| 58.33 ms (512x512)| 128.92 ms (100tokens)| 19.87 tokens/sec|1.4|
 
-
 ## How to use
 
-Download all files from this repository to the device
+## 安装 axllm
+方式一：克隆仓库后执行安装脚本：
 
+```shell
+git clone -b axllm https://github.com/AXERA-TECH/ax-llm.git
+cd ax-llm
+./install.sh
 ```
-$ tree -L 1
-.
-|-- FastVLM_tokenizer.txt
-|-- README.md
-|-- config.json
-|-- fastvlm_ax650_context_1k_prefill_640_int4
-|-- fastvlm_tokenizer
-|-- images
-|-- infer_axmodel.py
-|-- main_ax650
-|-- main_ax650_api
-|-- main_axcl_x86
-|-- main_axcl_x86_api
-|-- post_config.json
-|-- requirements.txt
-|-- run_ax650_1024.sh
-|-- run_ax650_512.sh
-|-- run_ax650_api.sh
-|-- run_axcl_x86.sh
-|-- run_axcl_x86_api.sh
-`-- utils
-
-4 directories, 15 files
+
+方式二：一行命令安装（默认分支 `axllm`）：
+
+```shell
+curl -fsSL https://raw.githubusercontent.com/AXERA-TECH/ax-llm/axllm/install.sh | bash
 ```
 
-#### Install transformer
+方式三：下载Github Actions CI 导出的可执行程序（适合没有编译环境的用户）：
+
+如果没有编译环境，请到：
+`https://github.com/AXERA-TECH/ax-llm/actions?query=branch%3Aaxllm`
+下载 **最新 CI 导出的可执行程序**（`axllm`），然后：
 
+```shell
+chmod +x axllm
+sudo mv axllm /usr/bin/axllm
 ```
-pip install -r requirements.txt
+
+## 模型下载（Hugging Face）
+先创建模型目录并进入，然后下载到该目录：
+
+```shell
+mkdir -p AXERA-TECH/FastVLM-1.5B-GPTQ-Int4
+cd AXERA-TECH/FastVLM-1.5B-GPTQ-Int4
+hf download AXERA-TECH/FastVLM-1.5B-GPTQ-Int4 --local-dir .
+
+# structure of the downloaded files
+tree -L 3
+.
+└── AXERA-TECH
+    └── FastVLM-1.5B-GPTQ-Int4
+
+
+
+2 directories, 34 files
 ```
 
-#### Inference with AX650 Host, such as M4N-Dock(爱芯派Pro) or AX650 DEMO Board
-
-Run the following command on the Axera board to start a chat conversation:
-
-```bash
-root@ax650:~/FastVLM-1.5B-GPTQ-Int4# ./run_ax650_1024.sh
-[I][                            Init][ 134]: LLM init start
-tokenizer_type = 3
-stop_tokens size: 2
-151645
-151645
-  6% | ███                               |   2 /  31 [2.24s<34.74s, 0.89 count/s] embed_selector init ok
-100% | ████████████████████████████████ |  31 /  31 [5.96s<5.96s, 5.20 count/s] init post axmodel ok,remain_cmm(8619 MB)[I][                            Init][ 252]: IMAGE_CONTEXT_TOKEN: 151646
-[I][                            Init][ 284]: image encoder input nhwc@uint8
-[I][                            Init][ 308]: image encoder output float32
-
-[I][                            Init][ 318]: image_encoder_height : 1024, image_encoder_width: 1024
-[I][                            Init][ 320]: max_token_len : 1024
-[I][                            Init][ 323]: kv_cache_size : 256, kv_cache_num: 1024
-[I][                            Init][ 331]: prefill_token_num : 128
-[I][                            Init][ 335]: grp: 1, prefill_max_token_num : 1
-[I][                            Init][ 335]: grp: 2, prefill_max_token_num : 128
-[I][                            Init][ 335]: grp: 3, prefill_max_token_num : 256
-[I][                            Init][ 335]: grp: 4, prefill_max_token_num : 512
-[I][                            Init][ 335]: grp: 5, prefill_max_token_num : 640
-[I][                            Init][ 339]: prefill_max_token_num : 640
+## Inference with AX650 Host, such as M4N-Dock(爱芯派Pro) or AX650N DEMO Board
+
+### 运行（CLI）
+
+```shell
+(base) root@ax650:~# axllm run AXERA-TECH/Qwen3-0.6B/
+[I][                            Init][ 127]: LLM init start
+tokenizer_type = 1
+ 96% | ███████████████████████████████   |  30 /  31 [2.35s<2.42s, 12.79 count/s] init post axmodel ok,remain_cmm(8662 MB)
+[I][                            Init][ 188]: max_token_len : 2559
+[I][                            Init][ 191]: kv_cache_size : 1024, kv_cache_num: 2559
+[I][                            Init][ 194]: prefill_token_num : 128
+[I][                            Init][ 198]: grp: 1, prefill_max_kv_cache_num : 1
+[I][                            Init][ 198]: grp: 2, prefill_max_kv_cache_num : 512
+[I][                            Init][ 198]: grp: 3, prefill_max_kv_cache_num : 1024
+[I][                            Init][ 198]: grp: 4, prefill_max_kv_cache_num : 1536
+[I][                            Init][ 198]: grp: 5, prefill_max_kv_cache_num : 2048
+[I][                            Init][ 203]: prefill_max_token_num : 2048
+[I][                            Init][  27]: LLaMaEmbedSelector use mmap
+100% | ████████████████████████████████ |  31 /  31 [2.35s<2.35s, 13.21 count/s] embed_selector init ok
 [I][                     load_config][ 282]: load config:
 {
     "enable_repetition_penalty": false,
-    "enable_temperature": true,
-    "enable_top_k_sampling": true,
+    "enable_temperature": false,
+    "enable_top_k_sampling": false,
     "enable_top_p_sampling": false,
-    "penalty_window": 30,
-    "repetition_penalty": 2,
-    "temperature": 0.1,
+    "penalty_window": 20,
+    "repetition_penalty": 1.2,
+    "temperature": 0.9,
     "top_k": 10,
     "top_p": 0.8
 }
 
-[I][                            Init][ 348]: LLM init ok
-Type "q" to exit, Ctrl+c to stop current running
+[I][                            Init][ 224]: LLM init ok
+Type "q" to exit
+Ctrl+c to stop current running
+"reset" to reset kvcache
+"dd" to remove last conversation.
+"pp" to print history.
+----------------------------------------
 prompt >> who are you
-image >>
-[I][                          Encode][ 470]: input_ids size: 33
-[I][                             Run][ 604]: input token num : 33, prefill_split_num : 1
-[I][                             Run][ 619]: prefill grpid 2
-[I][                             Run][ 646]: input_num_token:33
-[I][                             Run][ 770]: ttft: 127.56 ms
-I am FastVLM, a language model developed by Apple Inc.
-
-[N][                             Run][ 879]: hit eos,avg 19.87 token/s
-
-prompt >> describe the image
-image >> ./images/ssd_horse.jpg
-[I][                          Encode][ 442]: image encode time : 237.49 ms, size : 393216
-[I][                          Encode][ 496]: imgs_embed.size() : 1, media token size : 256
-[I][                             Run][ 604]: input token num : 291, prefill_split_num : 3
-[I][                             Run][ 619]: prefill grpid 4
-[I][                             Run][ 646]: input_num_token:128
-[I][                             Run][ 646]: input_num_token:128
-[I][                             Run][ 646]: input_num_token:35
-[I][                             Run][ 770]: ttft: 418.43 ms
-The image depicts a scene in an outdoor setting, likely a farm or ranch, with a person riding a brown horse. The person is wearing a blue hoodie and jeans, and is seated on a saddle with a yellow blanket underneath. The horse has a white blaze on its face and is standing on a dirt ground. In front of the horse, there is a brown dog wearing a pink collar, looking up at the person on the horse. In the background, there is a silver pickup truck parked near a fence, and beyond the fence, there are trees and other people. The overall atmosphere of the image suggests a casual, outdoor activity, possibly involving horse riding and training.
-
-[N][                             Run][ 879]: hit eos,avg 19.85 token/s
+[I][                      SetKVCache][ 357]: prefill_grpid:2 kv_cache_num:512 precompute_len:0 input_num_token:22
+[I][                      SetKVCache][ 359]: current prefill_max_token_num:2048
+[I][                      SetKVCache][ 360]: first run
+[I][                             Run][ 412]: input token num : 22, prefill_split_num : 1
+[I][                             Run][ 474]: ttft: 586.40 ms
+<think>
+Okay, the user asked, "Who are you?" I need to respond appropriately. Since I'm an AI assistant, I should acknowledge their question and explain my purpose. I should mention that I'm here to help and that I can assist with various tasks. I should keep the response friendly and open-ended to encourage further interaction. Let me make sure the language is clear and natural.
+</think>
+
+I'm an AI assistant designed to help you with a wide range of questions and tasks. How can I assist you today? 😊
 
+[N][                             Run][ 554]: hit eos,avg 15.63 token/s
+
+[I][                      GetKVCache][ 331]: precompute_len:130, remaining:1918
 prompt >> q
 ```
 
-```sh
-$ python infer_axmodel.py -v ./fastvlm_ax650_context_1k_prefill_640_int4/image_encoder_512x512.axmodel -m ./fastvlm_ax650_context_1k_prefill_640_int4  -t ./fastvlm_tokenizer/ -i 512
+### 启动服务（OpenAI 兼容）
+
+```shell
+(base) root@ax650:~# axllm serve AXERA-TECH/Qwen3-0.6B/
+[I][                            Init][ 127]: LLM init start
+tokenizer_type = 1
+ 96% | ███████████████████████████████   |  30 /  31 [2.06s<2.13s, 14.58 count/s] init post axmodel ok,remain_cmm(8662 MB)
+[I][                            Init][ 188]: max_token_len : 2559
+[I][                            Init][ 191]: kv_cache_size : 1024, kv_cache_num: 2559
+[I][                            Init][ 194]: prefill_token_num : 128
+[I][                            Init][ 198]: grp: 1, prefill_max_kv_cache_num : 1
+[I][                            Init][ 198]: grp: 2, prefill_max_kv_cache_num : 512
+[I][                            Init][ 198]: grp: 3, prefill_max_kv_cache_num : 1024
+[I][                            Init][ 198]: grp: 4, prefill_max_kv_cache_num : 1536
+[I][                            Init][ 198]: grp: 5, prefill_max_kv_cache_num : 2048
+[I][                            Init][ 203]: prefill_max_token_num : 2048
+[I][                            Init][  27]: LLaMaEmbedSelector use mmap
+100% | █████████████████████��██████████ |  31 /  31 [2.06s<2.06s, 15.07 count/s] embed_selector init ok
+[I][                     load_config][ 282]: load config:
+{
+    "enable_repetition_penalty": false,
+    "enable_temperature": false,
+    "enable_top_k_sampling": false,
+    "enable_top_p_sampling": false,
+    "penalty_window": 20,
+    "repetition_penalty": 1.2,
+    "temperature": 0.9,
+    "top_k": 10,
+    "top_p": 0.8
+}
+
+[I][                            Init][ 224]: LLM init ok
+Starting server on port 8000 with model 'AXERA-TECH/Qwen3-0.6B'...
+OpenAI API Server starting on http://0.0.0.0:8000
+Max concurrency: 1
+Models: AXERA-TECH/Qwen3-0.6B
+```
+
+### OpenAI 调用示例
+
+```python
+from openai import OpenAI
+
+API_URL = "http://127.0.0.1:8000/v1"
+MODEL = "AXERA-TECH/Qwen3-0.6B"
+
+messages = [
+    {"role": "system", "content": [{"type": "text", "text": "you are a helpful assistant."}]},
+    {"role": "user", "content": "hello"},
+]
+
+client = OpenAI(api_key="not-needed", base_url=API_URL)
+completion = client.chat.completions.create(
+    model=MODEL,
+    messages=messages,
+)
+
+print(completion.choices[0].message.content)
 ```
-output:
-
-```bash
-[INFO] Available providers:  ['AXCLRTExecutionProvider']
-Loading config, tokenizer and init model.
-Detected prefixes: ['llava_qwen2'], chosen: llava_qwen2, layers: 28
-Init InferenceSession:   0%|                                                                                                                          | 0/28 [00:00<?, ?it/s][INFO] Using provider: AXCLRTExecutionProvider
-[INFO] SOC Name: AX650N
-[INFO] VNPU type: VNPUType.DISABLED
-[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
-Init InferenceSession:   4%|████                                                                                                              | 1/28 [00:00<00:20,  1.31it/s][INFO] Using provider: AXCLRTExecutionProvider
-[INFO] SOC Name: AX650N
-[INFO] VNPU type: VNPUType.DISABLED
-[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
-Init InferenceSession:   7%|████████▏                                                                                                         | 2/28 [00:01<00:14,  1.85it/s][INFO] Using provider: AXCLRTExecutionProvider
-...
-[INFO] SOC Name: AX650N
-[INFO] VNPU type: VNPUType.DISABLED
-[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
-Init InferenceSession: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 28/28 [00:11<00:00,  2.48it/s]
-[INFO] Using provider: AXCLRTExecutionProvider
-[INFO] SOC Name: AX650N
-[INFO] VNPU type: VNPUType.DISABLED
-[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
-Model loaded successfully!
-[INFO] Using provider: AXCLRTExecutionProvider
-[INFO] SOC Name: AX650N
-[INFO] VNPU type: VNPUType.DISABLED
-[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
-[INFO]: 输入文本进行对话，或者输入图片路径进行图片理解, 或者输入q退出对话。
-prompt<<who are you
-slice_indices: [0]
-Slice prefill done: 0
-answer >> I am an artificial intelligence language model developed by Apple Inc.
-
-prompt<<./images/ssd_horse.jpg
-slice_indices: [0]
-Slice prefill done: 0
-answer >> The image depicts a young man riding a brown horse in an outdoor setting. The horse is standing on a dirt ground, and the rider is wearing a blue jacket and jeans. The horse has a white blaze on its face and white markings on its legs. The rider is holding the reins and appears to be looking down at a brown dog standing on the ground next to the horse. The dog is wearing a collar and is looking up at the rider. In the background, there is a silver pickup truck parked on the grass, and a fence can be seen further back. The scene appears to be set in a rural or farm-like environment.
-
-prompt<<./images/image_1.jpg
-slice_indices: [0]
-Slice prefill done: 0
-answer >> The image depicts a panda bear in a natural setting. The panda is sitting on the ground, surrounded by greenery, including bamboo and other plants. The panda has a distinctive black and white fur pattern, with black fur around its eyes, ears, and limbs, and white fur on its face, chest, and legs. The panda is sitting on its hind legs, with its front paws resting on its chest. The background shows a forested area with trees and rocks, suggesting that the panda is in its natural habitat. The panda appears to be looking directly at the camera, giving the impression that it is aware of the photographer's presence. The overall scene is peaceful and serene, capturing the beauty of the panda in its natural environment.
-
-prompt<<q
-[INFO]: 对话结束，再见。
-```
+
+
+### OpenAI 流式调用示例
+
+```python
+from openai import OpenAI
+
+API_URL = "http://127.0.0.1:8000/v1"
+MODEL = "AXERA-TECH/Qwen3-0.6B"
+
+messages = [
+    {"role": "system", "content": [{"type": "text", "text": "you are a helpful assistant."}]},
+    {"role": "user", "content": "hello"},
+]
+
+client = OpenAI(api_key="not-needed", base_url=API_URL)
+stream = client.chat.completions.create(
+    model=MODEL,
+    messages=messages,
+    stream=True,
+)
+
+print("assistant:")
+for ev in stream:
+    delta = getattr(ev.choices[0], "delta", None)
+    if delta and getattr(delta, "content", None):
+        print(delta.content, end="", flush=True)
+print("
+")
+```
+

config.json

@@ -0,0 +1,27 @@
+{
+  "system_prompt": "you are a helpful assistant.",
+  "model_name": "AXERA-TECH/FastVLM-1.5B-GPTQ-Int4",
+  "url_tokenizer_model": "FastVLM_tokenizer.txt",
+  "tokenizer_type": "FastVLM",
+  "post_config_path": "post_config.json",
+  "template_filename_axmodel": "llava_qwen2_p128_l%d_together.axmodel",
+  "axmodel_num": 28,
+  "filename_post_axmodel": "llava_qwen2_post.axmodel",
+  "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
+  "tokens_embed_num": 151647,
+  "tokens_embed_size": 1536,
+  "use_mmap_load_embed": true,
+  "vlm_type": "FastVLM",
+  "filename_image_encoder_axmodel": "image_encoder_1024x1024.axmodel",
+  "vision_patch_size": 14,
+  "vision_temporal_patch_size": 1,
+  "vision_spatial_merge_size": 1,
+  "vision_fps": 1,
+  "vision_tokens_per_second": 1,
+  "vision_cache_dir": "vision_cache",
+  "use_mmap_load_layer": true,
+  "devices": [
+    0,
+    1
+  ]
+}

fastvlm_ax650_context_1k_prefill_640_int4/model.embed_tokens.weight.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1045cc5b2298c3032c2f6a3f1b2f83bc4aa69d39f32476d9bfd7b68b9d380ccd
-size 931719296

fastvlm_tokenizer/added_tokens.json

@@ -1,6 +0,0 @@
-{
-  "<image>": 151646,
-  "<|endoftext|>": 151643,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644
-}

fastvlm_tokenizer/config.json

@@ -1,79 +0,0 @@
-{
-  "architectures": [
-    "LlavaQwen2ForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "llava_qwen.LlavaConfig",
-    "AutoModelForCausalLM": "llava_qwen.LlavaQwen2ForCausalLM"
-  },
-  "bos_token_id": 151643,
-  "dtype": "float32",
-  "eos_token_id": 151645,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 1536,
-  "image_aspect_ratio": "pad",
-  "image_grid_pinpoints": null,
-  "initializer_range": 0.02,
-  "intermediate_size": 8960,
-  "layer_types": [
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention"
-  ],
-  "max_position_embeddings": 32768,
-  "max_window_layers": 28,
-  "mm_hidden_size": 3072,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "mlp2x_gelu",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "mobileclip_l_1024",
-  "model_type": "llava_qwen2",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 2,
-  "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
-  "rope_theta": 1000000.0,
-  "sliding_window": null,
-  "tie_word_embeddings": true,
-  "tokenizer_model_max_length": 8192,
-  "tokenizer_padding_side": "right",
-  "transformers_version": "4.57.0",
-  "tune_mm_mlp_adapter": false,
-  "unfreeze_mm_vision_tower": true,
-  "use_cache": true,
-  "use_mm_proj": true,
-  "use_sliding_window": false,
-  "vocab_size": 151647
-}

fastvlm_tokenizer/generation_config.json

@@ -1,6 +0,0 @@
-{
-  "do_sample": true,
-  "temperature": null,
-  "top_p": null,
-  "transformers_version": "4.57.0"
-}

fastvlm_tokenizer/special_tokens_map.json

@@ -1,20 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "eos_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

fastvlm_tokenizer/tokenizer.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b187c9fe72f04a62ed1f592418d79751ed77f5eab6c3abded85349cf97f152ea
-size 11413284

fastvlm_tokenizer/tokenizer_config.json

@@ -1,52 +0,0 @@
-{
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<image>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "bos_token": null,
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "extra_special_tokens": {},
-  "model_max_length": 8192,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "right",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}

infer_axmodel.py

@@ -1,130 +0,0 @@
-import os
-import argparse
-
-from PIL import Image
-import numpy as np
-from utils.llava_qwen import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN
-from utils.llava_qwen import tokenizer_image_token, expand2square
-from utils.infer_func import InferManager
-from transformers import AutoTokenizer, AutoConfig, CLIPImageProcessor
-import axengine as ax
-from ml_dtypes import bfloat16
-import argparse
-
-def load_model_and_tokenizer(model_path):
-    tokenizer = AutoTokenizer.from_pretrained(model_path)
-    config = AutoConfig.from_pretrained(model_path)
-
-    mm_use_im_start_end = getattr(config, "mm_use_im_start_end", False)
-    mm_use_im_patch_token = getattr(config, "mm_use_im_patch_token", True)
-    if mm_use_im_patch_token:
-        tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
-    if mm_use_im_start_end:
-        tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
-
-    return config, tokenizer
-
-def vision_encoder(image_path, ax_session, args):
-
-    image_processor = CLIPImageProcessor(size={"shortest_edge": int(args.input_size)},        # CLIP 支持 336x336
-                                         crop_size={"height": int(args.input_size), "width": int(args.input_size)},
-                                         image_mean=[0, 0, 0],
-                                         image_std=[1/255, 1/255, 1/255]
-                                         )
-
-    image = Image.open(image_path).convert('RGB')
-
-    image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
-    input_image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
-    input_image = input_image.unsqueeze(0) # add batch dimension
-
-    input_image = input_image.numpy().astype(np.uint8).transpose((0, 2, 3, 1))  # NHWC to NCHW
-    vit_output = ax_session.run(None, {"images": input_image})[0]
-
-    return vit_output
-
-def llm_infer(image_features, llm_path, config, tokenizer, imer, get_input, token_length, embeds):
-
-    prompt = "<|im_start|>system\nYou are a helpful assistant, created by apple company.<|im_end|>\n"
-    question = get_input
-    prompt += "<|im_start|>user\n" + question
-
-    if image_features is not None:
-    #     # for idx in range(len(image_features)):
-        prompt += "\n" + "<image>"*token_length + "\n"
-    prompt += "<|im_end|>\n<|im_start|>assistant\n"
-
-    token_ids = tokenizer.encode(prompt)
-
-    # 图像理解
-    prefill_data = np.take(embeds, token_ids, axis=0)
-    prefill_data = prefill_data.astype(bfloat16)
-    token_len = len(token_ids)
-
-    if image_features is not None:
-        image_start_index = np.where(np.array(token_ids) == 151646)[0][0] # <image> tag 151646
-        image_insert_index = image_start_index + 1
-        prefill_data[image_insert_index : image_insert_index + token_length] = image_features[0, :, :]
-
-    eos_token_id = None
-    if isinstance(config.eos_token_id, list) and len(config.eos_token_id) > 1:
-        eos_token_id = config.eos_token_id
-
-    slice_len = 128
-    # prefill_max_len = 640
-    max_seq_len = 1024  # prefill + decode max length
-
-    # imer = InferManager(config, llm_path, max_seq_len=max_seq_len) # prefill + decode max length
-
-    token_ids = imer.prefill(tokenizer, token_ids, prefill_data, slice_len=slice_len)
-    imer.decode(tokenizer, token_ids, embeds, slice_len=slice_len, eos_token_id=eos_token_id)
-    print("\n")
-
-if __name__ == "__main__":
-
-    args = argparse.ArgumentParser()
-    args.add_argument("--vision_model", "-v", type=str, default="./fastvlm_ax650_context_1k_prefill_640_int4/image_encoder_1024x1024.axmodel", help="Path to the vision axmodel.")
-    args.add_argument("--model_path", "-m", type=str, default="./fastvlm_ax650_context_1k_prefill_640_int4", help="Path to the llm axmodel.")
-    args.add_argument("--tokenizer_path", "-t", type=str, default="./fastvlm_tokenizer", help="Path to the tokenizer.")
-    args.add_argument("--input_size", "-i", type=str, default="1024", help="Input size of the vision encoder model.")
-    # args.add_argument("--question", type=str, default="介绍一下你自己", help="The question to ask the model.")
-    
-    args = args.parse_args()
-
-    token_len_map = {"2048": 1280,
-                     "1024": 256,
-                     "768": 144,
-                     "512": 64,
-                     "256": 16}
-    
-    token_length = token_len_map[args.input_size]
-
-    print("Loading config, tokenizer and init model.")
-    config, tokenizer = load_model_and_tokenizer(model_path=args.tokenizer_path)
-
-    slice_len = 128
-    # prefill_max_len = 640
-    max_seq_len = 1024  # prefill + decode max length
-
-    imer = InferManager(config, args.model_path, max_seq_len=max_seq_len) # prefill + decode max length
-    ax_session = ax.InferenceSession(args.vision_model)
-    embeds = np.load(os.path.join(args.model_path, "model.embed_tokens.weight.npy"))
-
-    print(f"[INFO]: 输入文本进��对话，或者输入图片路径进行图片理解, 或者输入q退出对话。")
-    while True:
-        prompt = input("prompt<<")
-        if prompt.strip() == "q":
-            print(f"[INFO]: 对话结束，再见。")
-            break
-        else:
-            get_input = prompt.strip()
-            if get_input.lower().endswith(("jpg", "jpeg", "png")):
-                if not os.path.isfile(get_input):
-                    print("[INFO]: 输入错误，请检查图片输入路径。")
-                    continue
-                image_features = vision_encoder(get_input, ax_session, args)
-                get_input = "Describe the image in detail."
-                llm_infer(image_features, args.model_path, config, tokenizer, imer, get_input, token_length, embeds)
-            else:
-                image_features = None
-                llm_infer(image_features, args.model_path, config, tokenizer, imer, get_input, token_length, embeds)

main_axcl_x86

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7205b7ade7c5efaeef89fb3e458b6da11508e6d9fe992cfb92eefe0094c4a8b5
-size 7009480

main_axcl_x86_api

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c6c2dd57d2679f349a22279a4f4b92f941ec62314b58b38052228552ff0bd70
-size 7128832

post_config.json

@@ -1,14 +1,14 @@
 {
-    "enable_temperature" : true,
-    "temperature" : 0.1,
+    "enable_temperature" : false,
+    "temperature" : 0.9,
     
     "enable_repetition_penalty" : false,
-    "repetition_penalty" : 2,
-    "penalty_window" : 30,
+    "repetition_penalty" : 1.2,
+    "penalty_window" : 20,
 
     "enable_top_p_sampling" : false,
     "top_p" : 0.8,
 
-    "enable_top_k_sampling" : true,
+    "enable_top_k_sampling" : false,
     "top_k" : 10
 }