[Test] Add accuracy nightly test for new models

Signed-off-by: hfadzxy <[email protected]>

Author: zhangxinyuehfad

.github/workflows/_e2e_nightly_single_node_models.yaml

@@ -59,7 +59,7 @@ jobs:
     name: ${{inputs.model_list}} accuracy test
     runs-on: ${{ inputs.runner }}
     container:
-      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-910b-ubuntu22.04-py3.11
+      image: "${{ inputs.image }}"
       env:
         VLLM_USE_MODELSCOPE: True
         GHA_VLLM_ASCEND_VERSION: ${{ inputs.vllm-ascend }}
@@ -114,6 +114,12 @@ jobs:
           . /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh
           python3 -m pip install "https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/triton_ascend-3.2.0.dev20250914-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl"
 
+      - name: Install tensorflow (for Molmo-7B-D-0924)
+        if: ${{ inputs.runner == 'linux-aarch64-a2-1' && contains(inputs.model_list, 'Molmo-7B-D-0924') }}
+        shell: bash -l {0}
+        run: |
+          pip install tensorflow --no-cache-dir
+
       - name: Resolve vllm-ascend version
         run: |
           VERSION_INPUT="${{ inputs.vllm-ascend }}"
@@ -175,6 +181,7 @@ jobs:
         id: report
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_DATASETS_OFFLINE: True
           VLLM_USE_MODELSCOPE: True
           VLLM_CI_RUNNER: ${{ inputs.runner }}
           VLLM_VERSION: ${{ env.GHA_VLLM_VERSION }}

.github/workflows/vllm_ascend_test_nightly_a2.yaml

@@ -114,6 +114,16 @@ jobs:
               - Qwen3-VL-8B-Instruct
               - Qwen2.5-Omni-7B
               - Meta-Llama-3.1-8B-Instruct
+          - os: linux-aarch64-a2-1
+            model_list:
+              - ERNIE-4.5-0.3B-PT
+              - gemma-2-9b-it
+              - gemma-3-4b-it
+              - internlm-7b
+              - MiniCPM-2B-dpo-bf16
+              - InternVL3_5-8B-hf
+              - llava-1.5-7b-hf
+              - Molmo-7B-D-0924
           - os: linux-aarch64-a2-2
             model_list:
               - Qwen3-30B-A3B
@@ -128,5 +138,5 @@ jobs:
       vllm: v0.11.0
       runner: ${{ matrix.test_config.os }}
       model_list: ${{ toJson(matrix.test_config.model_list) }}
-      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11
+      image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.3.rc1-910b-ubuntu22.04-py3.11'
       upload: false

tests/e2e/models/configs/ERNIE-4.5-0.3B-PT.yaml

@@ -0,0 +1,9 @@
+model_name: "PaddlePaddle/ERNIE-4.5-0.3B-PT"
+hardware: "Atlas A2 Series"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,flexible-extract"
+    value: 0.25
+num_fewshot: 5
+trust_remote_code: True

tests/e2e/models/configs/InternVL3_5-8B.yaml renamed to tests/e2e/models/configs/InternVL3_5-8B-hf.yaml

@@ -0,0 +1,10 @@
+model_name: "OpenBMB/MiniCPM-2B-dpo-bf16"
+hardware: "Atlas A2 Series"
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.5
+num_fewshot: 5
+trust_remote_code: True
+gpu_memory_utilization: 0.8

tests/e2e/models/configs/MiniCPM-2B-dpo-bf16.yaml

@@ -0,0 +1,13 @@
+model_name: "LLM-Research/Molmo-7B-D-0924"
+hardware: "Atlas A2 Series"
+model: "vllm-vlm"
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.71
+max_model_len: 4096
+trust_remote_code: True
+apply_chat_template: False
+fewshot_as_multiturn: False
+gpu_memory_utilization: 0.8

tests/e2e/models/configs/Molmo-7B-D-0924.yaml

@@ -9,4 +9,11 @@ Qwen3-VL-30B-A3B-Instruct.yaml
 Qwen3-VL-8B-Instruct.yaml
 Qwen2.5-Omni-7B.yaml
 Meta-Llama-3.1-8B-Instruct.yaml
-InternVL3_5-8B.yaml
+InternVL3_5-8B.yaml
+ERNIE-4.5-0.3B-PT.yaml
+MiniCPM-2B-dpo-bf16.yaml
+gemma-2-9b-it.yaml
+gemma-3-4b-it.yaml
+internlm-7b.yaml
+Molmo-7B-D-0924.yaml
+llava-1.5-7b-hf.yaml

tests/e2e/models/configs/accuracy.txt

@@ -0,0 +1,11 @@
+model_name: "LLM-Research/gemma-2-9b-it"
+hardware: "Atlas A2 Series"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.46
+  - name: "exact_match,flexible-extract"
+    value: 0.79
+num_fewshot: 5
+gpu_memory_utilization: 0.8

tests/e2e/models/configs/gemma-2-9b-it.yaml

@@ -0,0 +1,13 @@
+model_name: "LLM-Research/gemma-3-4b-it"
+hardware: "Atlas A2 Series"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.59
+  - name: "exact_match,flexible-extract"
+    value: 0.59
+num_fewshot: 5
+apply_chat_template: False
+fewshot_as_multiturn: False
+gpu_memory_utilization: 0.7

tests/e2e/models/configs/gemma-3-4b-it.yaml

@@ -0,0 +1,13 @@
+model_name: "Shanghai_AI_Laboratory/internlm-7b"
+hardware: "Atlas A2 Series"
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.42
+num_fewshot: 5
+max_model_len: 2048
+trust_remote_code: True
+dtype: "bfloat16"
+apply_chat_template: False
+fewshot_as_multiturn: False