[add] llm_framework

Author: dianjixz

projects/llm_framework/.gitignore

@@ -0,0 +1,10 @@
+dist
+build
+.config.mk
+.flash.conf.json
+.vscode
+static_lib**
+*.deb
+*.tar.gz
+m5stack_*
+debian*

projects/llm_framework/README.md

@@ -0,0 +1,168 @@
+# LLM
+AX630C AI large model local voice assistant program. Provided by M5STACK STACKFLOW for basic communication architecture, coordinating multiple working units to work together.
+- llm-audio: Audio unit, providing system audio support.
+- llm-kws: Keyword detection unit, providing keyword wakeup service.
+- llm-asr: Speech-to-text unit, providing speech-to-text service.
+- llm-llm: Large model unit, providing AI large model inference service.
+- llm-melotts: NPU-accelerated TTS unit, providing text-to-speech and playback service.
+- llm-tts: CPU-computed TTS unit, providing text-to-speech and playback service.
+
+## compile
+```bash
+wget https://m5share.oss-cn-shenzhen.aliyuncs.com/ax630c_llm_project/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.gz?OSSAccessKeyId=LTAI5tSGrycCku6R4TQkroEc&Expires=37727406013&Signature=U9YN9Tp6zrIhNlL%2FfjcO0xRQTzk%3D
+sudo tar zxvf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.gz -C /opt
+
+
+sudo apt install python3 python3-pip libffi-dev
+pip3 install parse scons requests 
+
+
+cd M5Stack_LLM_Framework/examples/llm_framework
+scons distclean
+scons -j22
+
+```
+
+
+## assistant mode
+llm-sys llm-audio llm-kws llm-asr llm-llm llm-melotts Collaborative work。
+
+Chinese environment:
+
+1、reset
+
+send :
+
+```json
+{
+    "request_id": "11212155", 
+    "work_id": "sys",
+    "action": "reset"
+}
+```
+waiting reset over!
+
+2、init all unit
+
+send :
+
+```json
+{
+    "request_id": "1",
+    "work_id": "kws",
+    "action": "setup",
+    "object": "kws.setup",
+    "data": {
+        "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01","response_format": "kws.bool",
+        "input": "sys.pcm",
+        "enoutput": true,
+        "kws": "你好你好"
+        }
+}
+
+{
+    "request_id": "2",
+    "work_id": "asr",
+    "action": "setup",
+    "object": "asr.setup",
+    "data": {
+        "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23",
+        "response_format": "asr.utf-8.stream",
+        "input": "sys.pcm",
+        "enoutput": true,
+        "enkws":true,
+        "rule1":2.4,
+        "rule2":1.2,
+        "rule3":30.1
+        }
+}
+
+{
+    "request_id": "3",
+    "work_id": "llm",
+    "action": "setup","object": "llm.setup",
+    "data": {
+        "model": "qwen2.5-0.5B-prefill-20e",
+        "response_format": "llm.utf-8.stream",
+        "input": "llm.utf-8",
+        "enoutput": true,
+        "max_token_len": 256,
+        "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information."
+        }
+}
+
+{
+    "request_id": "4",
+    "work_id": "melotts",
+    "action": "setup",
+    "object": "melotts.setup",
+    "data": {
+        "model": "melotts_zh-cn",
+        "response_format": "sys.pcm",
+        "input": "tts.utf-8",
+        "enoutput": false
+    }
+}
+```
+waiting return work_id:
+```json
+{"created":1731488371,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"3","work_id":"asr.1001"}
+{"created":1731488377,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"llm.1002"}
+{"created":1731488392,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"melotts.1003"}
+{"created":1731488402,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"2","work_id":"kws.1000"}
+```
+
+creat assembly line：
+```json
+{
+    "request_id": "2",
+    "work_id": "asr.1001",
+    "action": "link",
+    "object":"work_id",
+    "data":"kws.1000"
+}
+
+{
+    "request_id": "3",
+    "work_id": "llm.1002",
+    "action": "link",
+    "object":"work_id",
+    "data":"asr.1001"
+}
+
+{
+    "request_id": "4",
+    "work_id": "melotts.1003",
+    "action": "link",
+    "object":"work_id",
+    "data":"llm.1002"
+}
+
+{
+    "request_id": "3",
+    "work_id": "llm.1002",
+    "action": "link",
+    "object":"work_id",
+    "data":"kws.1000"
+}
+{
+    "request_id": "4",
+    "work_id": "melotts.1003",
+    "action": "link",
+    "object":"work_id",
+    "data":"kws.1000"
+}
+
+```
+
+waiting return status:
+
+```json
+{"created":1731488403,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"3","work_id":"llm.1002"}
+{"created":1731488403,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"melotts.1003"}
+{"created":1731488403,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"2","work_id":"asr.1001"}
+{"created":1731488403,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"4","work_id":"melotts.1003"}
+{"created":1731488403,"data":"None","error":{"code":0,"message":""},"object":"None","request_id":"3","work_id":"llm.1002"}
+```
+
+**Please enjoy the large model voice assistant.**

projects/llm_framework/SConstruct

@@ -0,0 +1,66 @@
+from pathlib import Path
+import os
+import shutil
+
+os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
+os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
+
+version = 'v0.0.5'
+static_lib = 'static_lib'
+update = False
+if not os.path.exists(static_lib):
+    update = True
+else:
+    try:
+        with open(str(Path(static_lib)/'version'), 'r') as f:
+            if f.read() != version:
+                update = True
+    except:
+        update = True
+
+if update:
+    zip_file = static_lib + '_{}.tar.gz'.format(version)
+    zip_file_extrpath = static_lib + '_tmp'
+    down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/static_lib_{}.tar.gz".format(version)
+    
+    if 'CONFIG_REPO_AUTOMATION' in os.environ:
+        down = 'y'
+    else:
+        down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('static_lib_{}.tar.gz').format(version))
+        down = down.lower()
+    if down == 'y':
+        # from git import Repo
+        import requests
+        import parse
+        import tarfile
+        import shutil
+        try:
+            # Downloading via HTTP (more common)
+            if not os.path.exists(zip_file):
+                response = requests.get(down_url)
+                if response.status_code == 200:
+                    with open(zip_file, 'wb') as file:
+                        file.write(response.content)
+                else:
+                    print("{} down failed".format(down_url))
+            try:
+                shutil.rmtree(static_lib)
+            except:
+                pass
+            with tarfile.open(zip_file, 'r:gz') as tar:
+                tar.extractall(path=zip_file_extrpath)
+            shutil.move(zip_file_extrpath, static_lib)
+            try:
+                os.remove(zip_file)
+            except:
+                pass
+            print("The {} download successful.".format(down_url))
+        except Exception as e:
+            print('Please manually download {} to {} .'.format(down_url, zip_file))
+            print("Cloning failed.: {}".format(e))
+    else:
+        print('Please manually download {} to {} .'.format(down_url, zip_file))
+
+
+with open(str(Path(os.getcwd())/'..'/'..'/'SDK'/'tools'/'scons'/'project.py')) as f:
+    exec(f.read())

projects/llm_framework/config_defaults.mk

@@ -0,0 +1,23 @@
+CONFIG_TOOLCHAIN_PATH="/opt/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin"
+CONFIG_TOOLCHAIN_PREFIX="aarch64-none-linux-gnu-"
+
+# CONFIG_COMMPILE_DEBUG=y
+CONFIG_AX_MSP_ENABLED=y
+CONFIG_AX_620E_MSP_ENABLED=y
+CONFIG_AX630C_OPENWRT_SDK_ENABLED=y
+CONFIG_DEVICE_DRIVER_ENABLED=y
+CONFIG_DEVICE_UART_ENABLED=y
+# CONFIG_LHV_ENABLED=y
+# CONFIG_LHV_WITH_KCP is not set
+# CONFIG_LHV_WITH_PROTOCOL is not set
+# CONFIG_LHV_WITH_EVPP=y
+CONFIG_UTILITIES_ENABLED=y
+CONFIG_BACKWARD_CPP_ENABLED=y
+CONFIG_TOOLCHAIN_FLAGS="-g"
+CONFIG_EVENTPP_ENABLED=y
+CONFIG_LHV_ENABLED=y
+CONFIG_LHV_WITH_EVPP=y
+CONFIG_EVENTPP_ENABLED=y
+CONFIG_STACKFLOW_ENABLED=y
+CONFIG_UTILITIES_BASE64_ENABLED=y
+CONFIG_SINGLE_HEADER_LIBS_ENABLED=y

projects/llm_framework/include/abseil-cpp/absl/algorithm/algorithm.h

@@ -0,0 +1,64 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// -----------------------------------------------------------------------------
+// File: algorithm.h
+// -----------------------------------------------------------------------------
+//
+// This header file contains Google extensions to the standard <algorithm> C++
+// header.
+
+#ifndef ABSL_ALGORITHM_ALGORITHM_H_
+#define ABSL_ALGORITHM_ALGORITHM_H_
+
+#include <algorithm>
+#include <iterator>
+#include <type_traits>
+
+#include "absl/base/config.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+// equal()
+// rotate()
+//
+// Historical note: Abseil once provided implementations of these algorithms
+// prior to their adoption in C++14. New code should prefer to use the std
+// variants.
+//
+// See the documentation for the STL <algorithm> header for more information:
+// https://en.cppreference.com/w/cpp/header/algorithm
+using std::equal;
+using std::rotate;
+
+// linear_search()
+//
+// Performs a linear search for `value` using the iterator `first` up to
+// but not including `last`, returning true if [`first`, `last`) contains an
+// element equal to `value`.
+//
+// A linear search is of O(n) complexity which is guaranteed to make at most
+// n = (`last` - `first`) comparisons. A linear search over short containers
+// may be faster than a binary search, even when the container is sorted.
+template <typename InputIterator, typename EqualityComparable>
+bool linear_search(InputIterator first, InputIterator last,
+                   const EqualityComparable& value) {
+  return std::find(first, last, value) != last;
+}
+
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_ALGORITHM_ALGORITHM_H_