Skip to content

Commit ff9c730

Browse files
dianjixzdianjixz
committed
Merge branch 'dev' of github.com:m5stack/StackFlow into dev
2 parents be01d29 + 9f05b13 commit ff9c730

11 files changed

+340
-0
lines changed

‎projects/llm_framework/main_llm/models/mode_qwen2.5-0.5B-Int4-ax630c.json

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"mode":"qwen2.5-0.5B-Int4-ax630c",
3+
"type":"llm",
4+
"homepage":"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4",
5+
"capabilities":[
6+
"text_generation",
7+
"chat"
8+
],
9+
"input_type":[
10+
"llm.utf-8",
11+
"llm.utf-8.stream",
12+
"llm.chat_completion",
13+
"llm.chat_completion.stream"
14+
],
15+
"output_type":[
16+
"llm.utf-8",
17+
"llm.utf-8.stream"
18+
],
19+
"mode_param":{
20+
"tokenizer_type":2,
21+
"filename_tokenizer_model":"http://localhost:8080",
22+
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
23+
"filename_post_axmodel":"qwen2_post.axmodel",
24+
"template_filename_axmodel":"qwen2_p128_l%d_together.axmodel",
25+
"b_use_topk":false,
26+
"b_bos":false,
27+
"b_eos":false,
28+
"axmodel_num":24,
29+
"tokens_embed_num":151936,
30+
"tokens_embed_size":896,
31+
"b_use_mmap_load_embed":true,
32+
"b_dynamic_load_axmodel_layer":false,
33+
"ext_scripts":["tokenizer_qwen2.5-0.5B-Int4-ax630c.py"]
34+
}
35+
}

‎projects/llm_framework/main_llm/models/mode_qwen2.5-1.5B-Int4-ax630c.json

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"mode":"qwen2.5-1.5B-Int4-ax630c",
3+
"type":"llm",
4+
"homepage":"https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4",
5+
"capabilities":[
6+
"text_generation",
7+
"chat"
8+
],
9+
"input_type":[
10+
"llm.utf-8",
11+
"llm.utf-8.stream",
12+
"llm.chat_completion",
13+
"llm.chat_completion.stream"
14+
],
15+
"output_type":[
16+
"llm.utf-8",
17+
"llm.utf-8.stream"
18+
],
19+
"mode_param":{
20+
"tokenizer_type":2,
21+
"filename_tokenizer_model":"http://localhost:8080",
22+
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
23+
"filename_post_axmodel":"qwen2_post.axmodel",
24+
"template_filename_axmodel":"qwen2_p128_l%d_together.axmodel",
25+
"b_use_topk":false,
26+
"b_bos":false,
27+
"b_eos":false,
28+
"axmodel_num":28,
29+
"tokens_embed_num":151936,
30+
"tokens_embed_size":1536,
31+
"b_use_mmap_load_embed":true,
32+
"b_dynamic_load_axmodel_layer":false,
33+
"ext_scripts":["tokenizer_qwen2.5-1.5B-Int4-ax630c.py"]
34+
}
35+
}

‎projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-0.5B-Int4-ax630c.py

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
from transformers import AutoTokenizer, PreTrainedTokenizerFast
2+
from http.server import HTTPServer, BaseHTTPRequestHandler
3+
import json
4+
import argparse
5+
6+
class Tokenizer_Http():
7+
8+
def __init__(self, model_id):
9+
self.tokenizer = AutoTokenizer.from_pretrained(model_id)
10+
11+
def encode(self, prompt, content):
12+
messages = [
13+
{"role": "system", "content": content},
14+
{"role": "user", "content": prompt}
15+
]
16+
text = self.tokenizer.apply_chat_template(
17+
messages,
18+
tokenize=False,
19+
add_generation_prompt=True
20+
)
21+
print(text)
22+
token_ids = self.tokenizer.encode(text)
23+
return token_ids
24+
25+
def decode(self, token_ids):
26+
return self.tokenizer.decode(token_ids)
27+
28+
@property
29+
def bos_id(self):
30+
return self.tokenizer.bos_token_id
31+
32+
@property
33+
def eos_id(self):
34+
return self.tokenizer.eos_token_id
35+
36+
@property
37+
def bos_token(self):
38+
return self.tokenizer.bos_token
39+
40+
@property
41+
def eos_token(self):
42+
return self.tokenizer.eos_token
43+
44+
class Request(BaseHTTPRequestHandler):
45+
#通过类继承，新定义类
46+
timeout = 5
47+
server_version = 'Apache'
48+
49+
def do_GET(self):
50+
print(self.path)
51+
#在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
52+
self.send_response(200)
53+
self.send_header("type", "get") #设置响应头，可省略或设置多个
54+
self.end_headers()
55+
56+
if self.path == '/bos_id':
57+
bos_id = tokenizer.bos_id
58+
# print(bos_id)
59+
# to json
60+
if bos_id is None:
61+
msg = json.dumps({'bos_id': -1})
62+
else:
63+
msg = json.dumps({'bos_id': bos_id})
64+
elif self.path == '/eos_id':
65+
eos_id = tokenizer.eos_id
66+
if eos_id is None:
67+
msg = json.dumps({'eos_id': -1})
68+
else:
69+
msg = json.dumps({'eos_id': eos_id})
70+
else:
71+
msg = 'error'
72+
73+
print(msg)
74+
msg = str(msg).encode() #转为str再转为byte格式
75+
76+
self.wfile.write(msg) #将byte格式的信息返回给客户端
77+
78+
def do_POST(self):
79+
#在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
80+
data = self.rfile.read(int(
81+
self.headers['content-length'])) #获取从客户端传入的参数（byte格式）
82+
data = data.decode() #将byte格式转为str格式
83+
84+
self.send_response(200)
85+
self.send_header("type", "post") #设置响应头，可省略或设置多个
86+
self.end_headers()
87+
88+
if self.path == '/encode':
89+
req = json.loads(data)
90+
prompt = req['text']
91+
92+
token_ids = tokenizer.encode(prompt, args.content)
93+
if token_ids is None:
94+
msg = json.dumps({'token_ids': -1})
95+
else:
96+
msg = json.dumps({'token_ids': token_ids})
97+
98+
elif self.path == '/decode':
99+
req = json.loads(data)
100+
token_ids = req['token_ids']
101+
text = tokenizer.decode(token_ids)
102+
if text is None:
103+
msg = json.dumps({'text': ""})
104+
else:
105+
msg = json.dumps({'text': text})
106+
else:
107+
msg = 'error'
108+
print(msg)
109+
msg = str(msg).encode() #转为str再转为byte格式
110+
111+
self.wfile.write(msg) #将byte格式的信息返回给客户端
112+
113+
114+
if __name__ == "__main__":
115+
116+
args = argparse.ArgumentParser()
117+
args.add_argument('--host', type=str, default='localhost')
118+
args.add_argument('--port', type=int, default=8080)
119+
args.add_argument('--model_id', type=str, default='qwen2.5_coder_tokenizer')
120+
args.add_argument('--content', type=str, default='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.')
121+
args = args.parse_args()
122+
123+
tokenizer = Tokenizer_Http(args.model_id)
124+
125+
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
126+
# print(tokenizer.encode("hello world", args.content))
127+
128+
host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1'
129+
print('http://%s:%s' % host)
130+
server = HTTPServer(host, Request) #根据地址端口号和新定义的类，创建服务器实例
131+
server.serve_forever() #开启服务

‎projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-1.5B-Int4-ax630c.py

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
from transformers import AutoTokenizer, PreTrainedTokenizerFast
2+
from http.server import HTTPServer, BaseHTTPRequestHandler
3+
import json
4+
import argparse
5+
6+
class Tokenizer_Http():
7+
8+
def __init__(self, model_id):
9+
self.tokenizer = AutoTokenizer.from_pretrained(model_id)
10+
11+
def encode(self, prompt, content):
12+
messages = [
13+
{"role": "system", "content": content},
14+
{"role": "user", "content": prompt}
15+
]
16+
text = self.tokenizer.apply_chat_template(
17+
messages,
18+
tokenize=False,
19+
add_generation_prompt=True
20+
)
21+
print(text)
22+
token_ids = self.tokenizer.encode(text)
23+
return token_ids
24+
25+
def decode(self, token_ids):
26+
return self.tokenizer.decode(token_ids)
27+
28+
@property
29+
def bos_id(self):
30+
return self.tokenizer.bos_token_id
31+
32+
@property
33+
def eos_id(self):
34+
return self.tokenizer.eos_token_id
35+
36+
@property
37+
def bos_token(self):
38+
return self.tokenizer.bos_token
39+
40+
@property
41+
def eos_token(self):
42+
return self.tokenizer.eos_token
43+
44+
class Request(BaseHTTPRequestHandler):
45+
#通过类继承，新定义类
46+
timeout = 5
47+
server_version = 'Apache'
48+
49+
def do_GET(self):
50+
print(self.path)
51+
#在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
52+
self.send_response(200)
53+
self.send_header("type", "get") #设置响应头，可省略或设置多个
54+
self.end_headers()
55+
56+
if self.path == '/bos_id':
57+
bos_id = tokenizer.bos_id
58+
# print(bos_id)
59+
# to json
60+
if bos_id is None:
61+
msg = json.dumps({'bos_id': -1})
62+
else:
63+
msg = json.dumps({'bos_id': bos_id})
64+
elif self.path == '/eos_id':
65+
eos_id = tokenizer.eos_id
66+
if eos_id is None:
67+
msg = json.dumps({'eos_id': -1})
68+
else:
69+
msg = json.dumps({'eos_id': eos_id})
70+
else:
71+
msg = 'error'
72+
73+
print(msg)
74+
msg = str(msg).encode() #转为str再转为byte格式
75+
76+
self.wfile.write(msg) #将byte格式的信息返回给客户端
77+
78+
def do_POST(self):
79+
#在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
80+
data = self.rfile.read(int(
81+
self.headers['content-length'])) #获取从客户端传入的参数（byte格式）
82+
data = data.decode() #将byte格式转为str格式
83+
84+
self.send_response(200)
85+
self.send_header("type", "post") #设置响应头，可省略或设置多个
86+
self.end_headers()
87+
88+
if self.path == '/encode':
89+
req = json.loads(data)
90+
prompt = req['text']
91+
92+
token_ids = tokenizer.encode(prompt, args.content)
93+
if token_ids is None:
94+
msg = json.dumps({'token_ids': -1})
95+
else:
96+
msg = json.dumps({'token_ids': token_ids})
97+
98+
elif self.path == '/decode':
99+
req = json.loads(data)
100+
token_ids = req['token_ids']
101+
text = tokenizer.decode(token_ids)
102+
if text is None:
103+
msg = json.dumps({'text': ""})
104+
else:
105+
msg = json.dumps({'text': text})
106+
else:
107+
msg = 'error'
108+
print(msg)
109+
msg = str(msg).encode() #转为str再转为byte格式
110+
111+
self.wfile.write(msg) #将byte格式的信息返回给客户端
112+
113+
114+
if __name__ == "__main__":
115+
116+
args = argparse.ArgumentParser()
117+
args.add_argument('--host', type=str, default='localhost')
118+
args.add_argument('--port', type=int, default=8080)
119+
args.add_argument('--model_id', type=str, default='qwen2.5_coder_tokenizer')
120+
args.add_argument('--content', type=str, default='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.')
121+
args = args.parse_args()
122+
123+
tokenizer = Tokenizer_Http(args.model_id)
124+
125+
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
126+
# print(tokenizer.encode("hello world", args.content))
127+
128+
host = (args.host, args.port) #设定地址与端口号，'localhost'等价于'127.0.0.1'
129+
print('http://%s:%s' % host)
130+
server = HTTPServer(host, Request) #根据地址端口号和新定义的类，创建服务器实例
131+
server.serve_forever() #开启服务

‎projects/llm_framework/main_whisper/mode_whisper-base.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode": "whisper-base",
33
"type": "asr",
4+
"homepage":"https://huggingface.co/openai/whisper-base",
45
"capabilities": [
56
"Automatic_Speech_Recognition",
67
"English",

‎projects/llm_framework/main_whisper/mode_whisper-tiny.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode": "whisper-tiny",
33
"type": "asr",
4+
"homepage":"https://huggingface.co/openai/whisper-tiny",
45
"capabilities": [
56
"Automatic_Speech_Recognition",
67
"English",

‎projects/llm_framework/main_yolo/mode_yolo11n-hand-pose.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode":"yolo11n-hand-pose",
33
"type":"cv",
4+
"homepage":"https://github.com/ultralytics/ultralytics",
45
"capabilities":[
56
"Pose"
67
],

‎projects/llm_framework/main_yolo/mode_yolo11n-pose.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode":"yolo11n-pose",
33
"type":"cv",
4+
"homepage":"https://github.com/ultralytics/ultralytics",
45
"capabilities":[
56
"Pose"
67
],

‎projects/llm_framework/main_yolo/mode_yolo11n-seg.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode":"yolo11s-seg",
33
"type":"cv",
4+
"homepage":"https://github.com/ultralytics/ultralytics",
45
"capabilities":[
56
"Segmentation"
67
],

‎projects/llm_framework/main_yolo/mode_yolo11n.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"mode":"yolo11n",
33
"type":"cv",
4+
"homepage":"https://github.com/ultralytics/ultralytics",
45
"capabilities":[
56
"Detection"
67
],

‎projects/llm_framework/tools/llm_pack.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,8 +345,10 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'):
345345
'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.3', src_folder, revision],
346346
'llm-model-qwen2.5-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision],
347347
'llm-model-qwen2.5-0.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-p256-ax630c', '0.4', src_folder, revision],
348+
'llm-model-qwen2.5-0.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax630c', '0.4', src_folder, revision],
348349
'llm-model-qwen2.5-1.5B-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-ax630c', '0.3', src_folder, revision],
349350
'llm-model-qwen2.5-1.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-p256-ax630c', '0.4', src_folder, revision],
351+
'llm-model-qwen2.5-1.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax630c', '0.4', src_folder, revision],
350352
'llm-model-qwen2.5-coder-0.5B-ax630c':[create_data_deb,'llm-model-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision],
351353
'llm-model-llama3.2-1B-prefill-ax630c':[create_data_deb,'llm-model-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision],
352354
'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision],

0 commit comments

Comments
 (0)