【NPU】Added patch update verification for NPU CI

.buildkite/npu_suites.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""Emit Buildkite steps for the NPU suites.
+
+Piped into `buildkite-agent pipeline upload` by the npu step in pipeline.yml.
+The suites and their configurations are defined here.
+
+NPU jobs run on the ascend-a3 queue with resource_class determining NPU count
+(e.g., "npu-8" means 8 NPUs).
+
+The selection is read from the NPU_SUITES environment variable. For local
+testing, set NPU_SUITES=smk,nightly instead of having a buildkite-agent on PATH.
+
+stdlib only — runs with the agent host's python3.
+"""
+
+import json
+import os
+import subprocess
+
+NPU_QUEUE = "ascend-a3"
+CI_IMAGE = os.environ.get("CI_IMAGE", "quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1")
+IMAGE_REGISTRY = "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+IMAGE_NAME = "vime-ci-npu"
+VIME_IMAGE_TAG = os.environ.get("BUILDKITE_COMMIT", "latest")
+
+# (test_name, resource_class, extra_args, env_overrides)
+SUITES = {
+    "smk": [
+        ("qwen3-4B-npu", "npu-8", "", {}),
+    ],
+    "nightly": [
+        ("qwen3-4B-npu", "npu-8", "", {}),
+    ],
+}
+
+
+def selected_suites() -> list:
+    raw = os.environ.get("NPU_SUITES")
+    if raw is None:
+        raw = subprocess.run(
+            ["buildkite-agent", "meta-data", "get", "npu-suites"],
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout
+    values = [v.strip() for v in raw.replace(",", "\n").splitlines()]
+    unknown = [v for v in values if v and v not in SUITES]
+    if unknown:
+        raise SystemExit(f"unknown suite(s) {unknown}; expected {sorted(SUITES)}")
+    return [s for s in SUITES if s in values]
+
+
+def npu_step(suite: str, test_name: str, resource_class: str, extra_args: str, env: dict) -> dict:
+    pod_env = [
+        {"name": "VIME_TEST_ENABLE_INFINITE_RUN", "value": "false"},
+        {"name": "BUILDKITE_PULL_REQUEST", "value": os.environ.get("BUILDKITE_PULL_REQUEST", "false")},
+        {"name": "BUILDKITE_COMMIT", "value": os.environ.get("BUILDKITE_COMMIT", "")},
+        {"name": "HF_TOKEN", "value": "${HF_TOKEN}"},
+        {"name": "HF_ENDPOINT", "value": "https://hf-mirror.com"},
+        {"name": "ASCEND_RT_VISIBLE_DEVICES", "value": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"},
+        {"name": "IMAGE_REGISTRY", "value": IMAGE_REGISTRY},
+        {"name": "IMAGE_NAME", "value": IMAGE_NAME},
+        {"name": "VIME_IMAGE_TAG", "value": VIME_IMAGE_TAG},
+    ]
+    pod_env += [{"name": k, "value": v} for k, v in env.items()]
+
+    command = "\n".join(
+        [
+            'echo "INFO: update NPU environment"',
+            'if [ -n "${BUILDKITE_COMMIT}" ]; then',
+            "  source /workspace/build/buildkite/.buildkite/scripts/update-npu-environment.sh",
+            "fi",
+            'echo "INFO: download Qwen3-4B model and dapo-math-17k dataset"',
+            "pip install modelscope -q",
+            'modelscope download --model "Qwen/Qwen3-4B" --local_dir /root/.cache/huggingface/models/Qwen3-4B',
+            "hf download --repo-type dataset zhuzilin/dapo-math-17k --local-dir /root/.cache/huggingface/datasets/dapo-math-17k",
+            'echo "INFO: start training Qwen3-4B on NPU"',
+            "ls -l /dev/davinci*",
+            "npu-smi info",
+            "export MODEL_ROOT=/root/.cache/huggingface",
+            "cd /root/vime",
+            'sed -i "s/--num-rollout [0-9]\\+/--num-rollout 3/" scripts/run-qwen3-4B-npu.sh',
+            "bash scripts/run-qwen3-4B-npu.sh",
+            "sleep 3",
+            'echo "INFO: check training log"',
+            'if ! grep -q "succeeded" /root/vime/train_qwen3_4b_vllm.log; then',
+            '  echo "ERROR: train log does not contain Ray job succeeded message"',
+            "  exit 1",
+            "fi",
+            'echo "INFO: train log contains Ray job succeeded message"',
+        ]
+    )
+
+    label = f":fire: {suite}: {test_name}{' ' + extra_args if extra_args else ''}"
+    step = {
+        "label": label,
+        "command": command,
+        "agents": {
+            "queue": NPU_QUEUE,
+            "resource_class": resource_class,
+        },
+        "timeout_in_minutes": 180,
+        "retry": {"automatic": [{"exit_status": -1, "limit": 2}]},
+        "image": f"{IMAGE_REGISTRY}/{IMAGE_NAME}:{VIME_IMAGE_TAG}",
+        "plugins": [
+            {
+                "kubernetes": {
+                    "metadata": {
+                        "annotations": {
+                            "vault.hashicorp.com/agent-init-first": "true",
+                            "vault.hashicorp.com/agent-inject": "true",
+                            "vault.hashicorp.com/agent-inject-perms-ca.pem": "0400",
+                            "vault.hashicorp.com/agent-inject-perms-cert.pem": "0400",
+                            "vault.hashicorp.com/agent-inject-perms-config.json": "0400",
+                            "vault.hashicorp.com/agent-inject-perms-key.pem": "0400",
+                            "vault.hashicorp.com/agent-inject-secret-ca.pem": "internal/data/ascend/buildkitd",
+                            "vault.hashicorp.com/agent-inject-secret-cert.pem": "internal/data/ascend/buildkitd",
+                            "vault.hashicorp.com/agent-inject-secret-config.json": "internal/data/ascend/buildkitd",
+                            "vault.hashicorp.com/agent-inject-secret-key.pem": "internal/data/ascend/buildkitd",
+                            "vault.hashicorp.com/agent-inject-template-ca.pem": '{{- with secret "internal/data/ascend/buildkitd" -}}\n{{ .Data.data.RootCA }}\n{{- end }}',
+                            "vault.hashicorp.com/agent-inject-template-cert.pem": '{{- with secret "internal/data/ascend/buildkitd" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}',
+                            "vault.hashicorp.com/agent-inject-template-config.json": '{{- with secret "internal/data/ascend/buildkitd" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}',
+                            "vault.hashicorp.com/agent-inject-template-key.pem": '{{- with secret "internal/data/ascend/buildkitd" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}',
+                            "vault.hashicorp.com/agent-pre-populate-only": "true",
+                            "vault.hashicorp.com/agent-run-as-group": "1000",
+                            "vault.hashicorp.com/agent-run-as-user": "1000",
+                            "vault.hashicorp.com/agent-service-account-token-volume-name": "token-vol",
+                            "vault.hashicorp.com/role": "ascend-gha-runners",
+                            "vault.hashicorp.com/secret-volume-path": "/home/user/.docker/",
+                            "vault.hashicorp.com/tls-skip-verify": "true",
+                        }
+                    },
+                    "podSpecPatch": {
+                        "volumes": [
+                            {
+                                "name": "token-vol",
+                                "projected": {
+                                    "defaultMode": 420,
+                                    "sources": [
+                                        {
+                                            "serviceAccountToken": {
+                                                "audience": "api",
+                                                "expirationSeconds": 600,
+                                                "path": "token",
+                                            }
+                                        }
+                                    ],
+                                },
+                            }
+                        ],
+                        "imagePullSecrets": [{"name": "swr-secret"}],
+                    },
+                }
+            }
+        ],
+        "env": pod_env,
+    }
+    return step
+
+
+def main() -> None:
+    steps = [npu_step(suite, *entry) for suite in selected_suites() for entry in SUITES[suite]]
+    if not steps:
+        raise SystemExit("no NPU suites selected")
+    print(json.dumps({"steps": steps}, indent=2))
+
+
+if __name__ == "__main__":
+    main()