Skip to content

Commit 8a23525

Browse files
JacobZangJacobZang
committed
archives/filesystem: Edge2: add llm support
Signed-off-by: Jacobe Zang <[email protected]>
1 parent 735435d commit 8a23525

File tree

1 file changed

+185
-0
lines changed

1 file changed

+185
-0
lines changed

‎archives/filesystem/special/Edge2/usr/local/bin/khadas_llm.sh

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/bin/bash
2+
3+
# Ensure running as khadas user
4+
if [ "$(whoami)" != "khadas" ]; then
5+
echo "Please run this script as khadas user!"
6+
exit 1
7+
fi
8+
9+
# Check and install CMake if missing
10+
if ! command -v cmake &> /dev/null; then
11+
echo "Installing CMake..."
12+
sudo apt update
13+
sudo apt install cmake -y
14+
else
15+
echo "CMake already installed, skipping installation"
16+
fi
17+
18+
# Calculate system memory
19+
mem_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')
20+
mem_float=$(echo "scale=1; $mem_kb/1024/1024" | bc)
21+
22+
allow_7b=false
23+
if (( $(echo "$mem_float >= 8" | bc -l) )); then
24+
allow_7b=true
25+
fi
26+
27+
# Model configuration with separate directories
28+
declare -A MODEL_CONFIG=(
29+
["1"]="DeepSeek 1.5B|https://dl.khadas.com/development/llm/deepseek-r1-distill-qwen-1.5b_w8a8_rk3588.rkllm|/home/khadas/rknn-llm/examples/DeepSeek-R1-Distill-Qwen-1.5B_Demo/deploy"
30+
["2"]="DeepSeek 7B|https://dl.khadas.com/development/llm/deepseek-r1-distill-qwen-7b_w8a8_rk3588.rkllm|/home/khadas/rknn-llm/examples/DeepSeek-R1-Distill-Qwen-1.5B_Demo/deploy"
31+
["3"]="Qwen2 2B-VL|https://dl.khadas.com/development/llm/qwen2-vl-2b-instruct_w8a8_rk3588.rkllm|/home/khadas/rknn-llm/examples/Qwen2-VL-2B_Demo/deploy"
32+
["4"]="ChatGLM3 6B|https://dl.khadas.com/development/llm/chatglm3-6b_w8a8_rk3588.rkllm|/home/khadas/rknn-llm/examples/ChatGLM3-6B_Demo/deploy"
33+
)
34+
35+
# Clone repository function
36+
clone_repo() {
37+
local repo_dir="/home/khadas/rknn-llm"
38+
39+
# Only clone if directory doesn't exist
40+
if [ ! -d "$repo_dir" ]; then
41+
echo "Cloning repository to $repo_dir..."
42+
if ! git clone https://github.com/khadas/rknn-llm.git "$repo_dir"; then
43+
echo "Repository clone failed!"
44+
exit 1
45+
fi
46+
else
47+
echo "Repository exists, skipping clone"
48+
fi
49+
}
50+
51+
# Model management function
52+
manage_models() {
53+
# Check model status
54+
declare -A model_status
55+
for key in "${!MODEL_CONFIG[@]}"; do
56+
IFS='|' read -ra info <<< "${MODEL_CONFIG[$key]}"
57+
model_file="${info[2]}/$(basename "${info[1]}")"
58+
[ -f "$model_file" ] && model_status[$key]=1 || model_status[$key]=0
59+
done
60+
61+
# Display status menu
62+
echo -e "\n======================"
63+
echo "System Memory: ${mem_float}G"
64+
echo "Available Models:"
65+
for key in $(echo "${!MODEL_CONFIG[@]}" | tr ' ' '\n' | sort -n); do
66+
if [[ "$key" == "2" && "$allow_7b" != "true" ]]; then
67+
continue
68+
fi
69+
IFS='|' read -ra info <<< "${MODEL_CONFIG[$key]}"
70+
status=$([ ${model_status[$key]} -eq 1 ] && echo "[Installed]" || echo "[Missing]")
71+
printf "%2s) %-20s %s\n" "$key" "${info[0]}" "$status"
72+
done
73+
echo "======================"
74+
75+
# Handle model download
76+
read -p "Download missing models? [y/N] " dl_choice
77+
if [[ $dl_choice =~ [Yy] ]]; then
78+
echo "Available downloads:"
79+
for key in $(echo "${!model_status[@]}" | tr ' ' '\n' | sort -n); do
80+
if [[ "$key" == "2" && "$allow_7b" != "true" ]]; then
81+
continue
82+
fi
83+
[ ${model_status[$key]} -eq 0 ] && {
84+
IFS='|' read -ra info <<< "${MODEL_CONFIG[$key]}"
85+
printf "%2s) %s\n" "$key" "${info[0]}"
86+
}
87+
done
88+
89+
while :; do
90+
read -p "Select model to download: " model_choice
91+
if [[ -n "${MODEL_CONFIG[$model_choice]}" && ${model_status[$model_choice]} -eq 0 ]]; then
92+
IFS='|' read -ra selected <<< "${MODEL_CONFIG[$model_choice]}"
93+
model_url="${selected[1]}"
94+
model_dir="${selected[2]}"
95+
model_file="${model_dir}/$(basename "$model_url")"
96+
97+
echo "Downloading ${selected[0]}..."
98+
if wget --show-progress -qO "$model_file" "$model_url"; then
99+
echo "Download completed: $(basename "$model_file")"
100+
return 0
101+
else
102+
echo "Download failed! Cleaning up..."
103+
rm -f "$model_file"
104+
return 1
105+
fi
106+
else
107+
echo "Invalid selection, try again"
108+
fi
109+
done
110+
fi
111+
}
112+
113+
# Build function with path isolation
114+
build_project() {
115+
local selected_key=$1
116+
IFS='|' read -ra info <<< "${MODEL_CONFIG[$selected_key]}"
117+
118+
echo -e "\nBuilding ${info[0]} model..."
119+
cd "${info[2]}" || exit 1
120+
121+
# Copy build script if missing
122+
[ ! -f "build-linux.sh" ] && cp ../../build-linux.sh ./
123+
124+
if ! bash build-linux.sh; then
125+
echo "Build failed!"
126+
exit 1
127+
fi
128+
}
129+
130+
# Main execution flow
131+
clone_repo
132+
manage_models
133+
134+
# Model selection menu
135+
PS3="Select model to build (1-${#MODEL_CONFIG[@]}): "
136+
select choice in "${MODEL_CONFIG[@]}"; do
137+
[[ $REPLY =~ ^[1-4]$ ]] && break || echo "Invalid input, enter 1-4"
138+
done
139+
140+
# Identify selected model
141+
for key in "${!MODEL_CONFIG[@]}"; do
142+
if [[ "${MODEL_CONFIG[$key]}" == "$choice" ]]; then
143+
selected_key=$key
144+
IFS='|' read -ra selected_info <<< "${MODEL_CONFIG[$key]}"
145+
break
146+
fi
147+
done
148+
149+
# Build selected model
150+
build_project "$selected_key"
151+
152+
# Runtime preparation
153+
model_file="${selected_info[2]}/$(basename "${selected_info[1]}")"
154+
build_dir="${selected_info[2]}/install/demo_Linux_aarch64"
155+
156+
# Validate paths
157+
[ ! -f "$model_file" ] && {
158+
echo "Error: Model file not found - $model_file"
159+
exit 1
160+
}
161+
162+
[ ! -d "$build_dir" ] && {
163+
echo "Error: Build directory missing - $build_dir"
164+
exit 1
165+
}
166+
167+
# Configure runtime parameters
168+
echo -e "\nConfigure runtime parameters (press Enter for defaults)"
169+
read -p "Max new tokens (default 2048): " max_new_tokens
170+
max_new_tokens=${max_new_tokens:-2048}
171+
172+
while :; do
173+
read -p "Context length (default 4096): " max_context_len
174+
max_context_len=${max_context_len:-4096}
175+
[[ $max_context_len =~ ^[0-9]+$ ]] && [ $max_context_len -gt 0 ] && break
176+
echo "Invalid input! Must be positive integer"
177+
done
178+
179+
# Execute model
180+
cd "$build_dir" || exit 1
181+
export LD_LIBRARY_PATH=./lib
182+
export RKLLM_LOG_LEVEL=1
183+
184+
echo -e "\nLaunching: $(basename "$model_file")"
185+
./llm_demo "$model_file" "$max_new_tokens" "$max_context_len"

0 commit comments

Comments
 (0)