feat: 图文多模态 vision chat 功能。

hkh1012 · Feb 19, 2024 · f62ed50 · f62ed50
1 parent 11a9078
commit f62ed50
Show file tree

Hide file tree

Showing 26 changed files with 539 additions and 106 deletions.
diff --git a/README.md b/README.md
@@ -17,31 +17,29 @@
 | openai库      | openai-java                                  |
 | 前端           | freemarker、bootstrap、jquery、recorder.js      | 
 | stream-chat  | SSE                                          | 
-| LLMs         | openai、chatglm2、文心一言                         | 
+| LLMs         | openai、chatglm2、文心一言 、智普AI                   | 
 | embeddings   | openai、text2vec-transformers、文心一言            |
 | vector store | weaviate、milvus                              |
 
 ## langchain 原理
 <img src="src/main/resources/assets/langchain+chatglm.png" alt="原理图"/>
 
-## 路线图
-已完成本地知识库上传、及完成openai、chatglm2两个LLMs模型流式聊天功能。未来计划会接入更多大语言模型，以满足更多需求场景。
+## 模型能力矩阵
+| 模型/能力     | 文本生成 | 流式输出 | 语音  | 函数调用 | 图片生成 | 多模态(VISION) | 嵌入EMBEDDING |
+|-----------|------|------|-----|------|---------|-------------|-------------|
+| openai    | 支持   | 支持   | 支持  | 支持   |-| 支持          | 支持          |
+| 百度(文心)    | 支持   | 支持   | -   | 支持   |-| -           | 支持          |
+| 智普(GLM-4) | 支持   | 支持   | -   | 支持   |-| 支持          | 支持          |
+| chatglm2  | 支持   | 支持   | -   | -    |-| -           | -           |
+| ... ...   | -    | -    | -   | -    |-| -           |-  |
+
+## 功能路线图
+已完成本地知识库上传、及完成openai、chatglm2、百度、智普GLM-4四个LLMs模型流式聊天功能。未来计划会接入更多大语言模型，以满足更多需求场景。
 - [ ] Langchain 知识库
     - [x] 接入非结构化文档（已支持 md、pdf、docx、txt、csv 等文件格式）
     - [ ] 搜索引擎接入
     - [ ] 结构化数据接入（如Excel、SQL 等）
     - [ ] 知识图谱/图数据库接入
-- [ ] 增加更多 LLM 模型支持
-    - [x] [OPENAI](https://platform.openai.com/docs/api-reference)
-    - [x] [THUDM/chatglm2-6b](https://huggingface.co/THUDM/chatglm2-6b)
-    - [x] [百度/文心一言](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/clntwmv7t)
-    - [ ] [THUDM/chatglm-6b](https://huggingface.co/THUDM/chatglm-6b)
-    - 
-- [ ] 增加更多 Embedding 模型支持
-    - [x] [OPENAI/embedding](https://platform.openai.com/docs/api-reference/embeddings)
-    - [x] [weaviate/text2vec-transformers](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers)
-    - [x] [百度/文心一言](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/dllz04sro)
-    - [ ] [shibing624/text2vec-base-chinese](https://huggingface.co/shibing624/text2vec-base-chinese)
 - [ ] 系统功能
     - [ ] 用户
       - [x] 用户登录

diff --git a/pom.xml b/pom.xml
@@ -104,6 +104,16 @@
             <artifactId>mysql-connector-j</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>io.projectreactor.netty</groupId>
+            <artifactId>reactor-netty-core</artifactId>
+            <version>1.1.16</version>
+        </dependency>
+        <dependency>
+            <groupId>io.projectreactor.netty</groupId>
+            <artifactId>reactor-netty-http</artifactId>
+            <version>1.1.16</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-configuration-processor</artifactId>

diff --git a/src/main/java/com/hkh/ai/chain/callback/FunctionCaller.java b/src/main/java/com/hkh/ai/chain/callback/FunctionCaller.java
@@ -10,15 +10,15 @@
 public class FunctionCaller {
 
     /**
-     * 查询天气API
+     * 查询天气API(模拟)
      * @param jsonNode
      * @return
      */
     public String get_location_weather(JSONObject jsonNode){
         String location = jsonNode.getString("location");
         String datePeriod = jsonNode.getString("datePeriod");
-        String valueByName = DatePeriod.getValueByName(datePeriod);
-        String result = "(模拟)通过查询天气API得到:" +location + valueByName + "天气为晴天";
+//        String valueByName = DatePeriod.getValueByName(datePeriod);
+        String result = "(模拟)通过查询天气API得到:" +location + datePeriod + "天气为晴天";
         log.info(result);
         return result;
     }

diff --git a/src/main/java/com/hkh/ai/chain/llm/capabilities/generation/ZhipuAiUtil.java b/src/main/java/com/hkh/ai/chain/llm/capabilities/generation/ZhipuAiUtil.java
@@ -32,8 +32,8 @@ public class ZhipuAiUtil {
     private String completionModel;
 
     /**
-     * 百度千帆开饭的embedding api model（默认：bge-large-zh）
-     * 可选模型：Embedding-V1、bge-large-zh、bge-large-en
+     * 可选模型：embedding-2
+     *
      */
     @Value("${chain.vectorization.zhipu.model}")
     private String embeddingModel;
@@ -60,12 +60,14 @@ public String getAccessToken(){
         queryWrapper.ge("expired_time",LocalDateTime.now().plusSeconds(60L));
         AccessToken accessToken = accessTokenService.getOne(queryWrapper,false);
         if (accessToken == null){
-            String[] keys = appKey.split(".");
+            String[] keys = appKey.split("\\.");
             LocalDateTime now = LocalDateTime.now();
             LocalDateTime expiredTime = now.plusSeconds(60 * 60 * 24);
+
             Map<String,String> jwtHeader = new HashMap<>();
             jwtHeader.put("alg","HS256");
             jwtHeader.put("sign_type","SIGN");
+
             Map<String,Object> payloads = new HashMap<>();
             payloads.put("api_key",keys[0]);
             payloads.put("exp",expiredTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
@@ -74,7 +76,7 @@ public String getAccessToken(){
             JWT jwt = JWT.create()
                     .addHeaders(jwtHeader)
                     .addPayloads(payloads)
-                    .setSigner(JWTSignerUtil.createSigner("HS256", DatatypeConverter.parseBase64Binary(keys[1])));
+                    .setSigner(JWTSignerUtil.createSigner("HS256", keys[1].getBytes()));
             String jwtToken = jwt.sign();
 
             AccessToken newAccessToken = new AccessToken();

diff --git a/...m/hkh/ai/chain/llm/capabilities/generation/function/openai/OpenAiFunctionChatService.java b/...m/hkh/ai/chain/llm/capabilities/generation/function/openai/OpenAiFunctionChatService.java
@@ -74,8 +74,8 @@ public List<FunctionCompletionResult> functionCompletion(String content, List<Ch
             functionCompletionResult.setType("function");
             functionCompletionResult.setName(choice.getMessage().getFunctionCall().getName());
             JsonNode arguments = choice.getMessage().getFunctionCall().getArguments();
-            JSONObject jsonObject = JSONObject.parseObject(arguments.asText());
-            functionCompletionResult.setArguments(jsonObject);
+            String argumentsStr = JSONObject.toJSONString(arguments);
+            functionCompletionResult.setArguments(JSONObject.parseObject(argumentsStr));
             functionResultList.add(functionCompletionResult);
         }
 

diff --git a/...m/hkh/ai/chain/llm/capabilities/generation/function/zhipu/ZhipuAiFunctionChatService.java b/...m/hkh/ai/chain/llm/capabilities/generation/function/zhipu/ZhipuAiFunctionChatService.java
@@ -5,6 +5,7 @@
 import cn.hutool.http.HttpRequest;
 import com.alibaba.fastjson2.JSONArray;
 import com.alibaba.fastjson2.JSONObject;
+import com.alibaba.fastjson2.JSONReader;
 import com.hkh.ai.chain.llm.capabilities.generation.ZhipuAiUtil;
 import com.hkh.ai.chain.llm.capabilities.generation.ZhipuChatApis;
 import com.hkh.ai.chain.llm.capabilities.generation.function.ChatFunctionObject;
@@ -34,6 +35,14 @@ public class ZhipuAiFunctionChatService implements FunctionChatService {
     public List<FunctionCompletionResult> functionCompletion(String content, List<ChatFunctionObject> functionObjectList) {
         String accessToken = zhipuAiUtil.getAccessToken();
 
+        JSONArray toolArray = new JSONArray();
+        for (ChatFunctionObject functionObject : functionObjectList){
+            JSONObject tool = new JSONObject();
+            tool.put("type","function");
+            tool.put("function",functionObject);
+            toolArray.add(tool);
+        }
+
         // 构建 message
         JSONArray messages = new JSONArray();
         JSONObject jsonObject = new JSONObject();
@@ -45,16 +54,24 @@ public List<FunctionCompletionResult> functionCompletion(String content, List<Ch
         JSONObject body = new JSONObject();
         body.put("messages",messages);
         body.put("model",zhipuAiUtil.getCompletionModel());
-        body.put("tools", functionObjectList);
+        body.put("tools", toolArray);
         body.put("tool_choice","auto");
 
-        HttpRequest httpRequest = new HttpRequest(UrlBuilder.of(ZhipuChatApis.COMPLETION_TEXT));
-        httpRequest.header("Authorization",accessToken);
-        httpRequest.header("content-type","application/json");
-        httpRequest.body(body.toJSONString());
-        String resultStr = httpRequest.execute().body();
+        String resultStr = HttpRequest.post(ZhipuChatApis.COMPLETION_TEXT)
+                .header("Authorization",accessToken)
+                .header("content-type","application/json")
+                .body(body.toJSONString())
+                .execute().body();
+
+        // 返回的是非标json，需要特殊处理。。。
+        resultStr = resultStr
+                .replaceAll("\\\\\\{","{")
+                .replaceAll("\\\\}\"","}")
+                .replaceAll("\\\\\"", "\"")
+                .replaceAll("\"\\{","{")
+                .replaceAll("}\"","}");
 
-        BlockCompletionResult result = JSONObject.parseObject(resultStr, BlockCompletionResult.class);
+        BlockCompletionResult result = JSONObject.parseObject(resultStr, BlockCompletionResult.class, JSONReader.Feature.AllowUnQuotedFieldNames);
         List<BlockCompletionResult.BlockCompletionResultChoiceMessageToolCall> tool_calls = result.getChoices().get(0).getMessage().getTool_calls();
         List<FunctionCompletionResult> functionResultList = new ArrayList<>();
         for (int i = 0; i < tool_calls.size(); i++) {

diff --git a/.../hkh/ai/chain/llm/capabilities/generation/text/baidu/BaiduQianFanCompletionWebClient.java b/.../hkh/ai/chain/llm/capabilities/generation/text/baidu/BaiduQianFanCompletionWebClient.java
@@ -28,7 +28,7 @@ public class BaiduQianFanCompletionWebClient {
 
     @PostConstruct
     public void init(){
-        log.info("baidu api web client init...");
+        log.info("baidu ai web client init...");
         this.webClient = WebClient.builder()
             .defaultHeader(HttpHeaders.CONTENT_TYPE, "application/json")
             .build();

diff --git a/.../com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuCompletionBizProcessor.java b/.../com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuCompletionBizProcessor.java
@@ -41,30 +41,32 @@ public class ZhipuCompletionBizProcessor {
     private final List<Integer> promptTokens;
 
     public void bizProcess(String item){
-        log.info(item);
-        StreamCompletionResult resultObj = JSONObject.parseObject(item, StreamCompletionResult.class);
-        String content = resultObj.getChoices().get(0).getDelta().getContent();
-        try {
-            if (StringUtils.isNotBlank(resultObj.getChoices().get(0).getFinish_reason())) {
-                this.getSseEmitter().send("[END]");
-                String fullContent = this.getSb().toString();
-                List<Integer> completionToken = this.getEnc().encode(fullContent);
-                System.out.println("total token costs: " + (this.getPromptTokens().size() + completionToken.size()));
-                this.getConversationService().saveConversation(this.getSysUser().getId(), this.getRequest().getSessionId(), this.getSb().toString(), "A");
-            } else {
-                if (content.contains("\n") || content.contains("\r")) {
-                    content = content.replaceAll("\n", "<br>");
-                    content = content.replaceAll("\r", "<br>");
+        System.out.println("智普流式输出:" +item);
+        if (!"[DONE]".equals(item)){
+            StreamCompletionResult resultObj = JSONObject.parseObject(item, StreamCompletionResult.class);
+            String content = resultObj.getChoices().get(0).getDelta().getContent();
+            try {
+                if (StringUtils.isNotBlank(resultObj.getChoices().get(0).getFinish_reason())) {
+                    this.getSseEmitter().send("[END]");
+                    String fullContent = this.getSb().toString();
+                    List<Integer> completionToken = this.getEnc().encode(fullContent);
+                    System.out.println("total token costs: " + (this.getPromptTokens().size() + completionToken.size()));
+                    this.getConversationService().saveConversation(this.getSysUser().getId(), this.getRequest().getSessionId(), this.getSb().toString(), "A");
+                } else {
+                    if (content.contains("\n") || content.contains("\r")) {
+                        content = content.replaceAll("\n", "<br>");
+                        content = content.replaceAll("\r", "<br>");
+                    }
+                    if (content.contains(" ")) {
+                        content = content.replaceAll(" ", "&nbsp;");
+                    }
+                    this.getSb().append(content);
+                    this.getSseEmitter().send(content);
                 }
-                if (content.contains(" ")) {
-                    content = content.replaceAll(" ", "&nbsp;");
-                }
-                this.getSb().append(content);
-                this.getSseEmitter().send(content);
+            } catch (IOException e) {
+                log.error("ZhipuCompletionBizProcessor--->>bizProcess异常", e);
+                throw new RuntimeException(e);
             }
-        } catch (IOException e) {
-            log.error("ZhipuCompletionBizProcessor--->>bizProcess异常", e);
-            throw new RuntimeException(e);
         }
 
     }

diff --git a/...ava/com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuCompletionWebClient.java b/...ava/com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuCompletionWebClient.java
@@ -3,11 +3,13 @@
 import com.alibaba.fastjson2.JSONObject;
 import com.hkh.ai.chain.llm.capabilities.generation.BaiduQianFanUtil;
 import com.hkh.ai.chain.llm.capabilities.generation.ZhipuAiUtil;
+import com.hkh.ai.chain.llm.capabilities.generation.ZhipuChatApis;
 import jakarta.annotation.PostConstruct;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatusCode;
+import org.springframework.http.MediaType;
 import org.springframework.stereotype.Component;
 import org.springframework.web.reactive.function.client.WebClient;
 import org.springframework.web.reactive.function.client.WebClientResponseException;
@@ -30,26 +32,25 @@ public class ZhipuCompletionWebClient {
     public void init(){
         log.info("zhipu ai api web client init...");
         this.webClient = WebClient.builder()
-            .defaultHeader(HttpHeaders.CONTENT_TYPE, "application/json")
+            .defaultHeader("content-type", "application/json")
             .build();
     }
 
     public Flux<String> streamChatCompletion(JSONObject requestBody){
         log.info("streamChatCompletion 参数：{}",requestBody);
-        String url = zhipuAiUtil.getCompletionModel();
         String accessToken = zhipuAiUtil.getAccessToken();
         return webClient.post()
-                .uri(url)
-                .header(HttpHeaders.CONTENT_TYPE,"application/json")
+                .uri(ZhipuChatApis.COMPLETION_TEXT)
+                .bodyValue(requestBody)
                 .header("Authorization",accessToken)
-                .bodyValue(requestBody.toJSONString())
                 .retrieve()
                 .bodyToFlux(String.class)
                 .onErrorResume(WebClientResponseException.class, ex -> {
+                    ex.printStackTrace();
                     HttpStatusCode statusCode = ex.getStatusCode();
                     String res = ex.getResponseBodyAsString();
                     log.error("ZhipuAI API error: {} {}", statusCode, res);
-                    return Mono.error(new RuntimeException(res));
+                    return Flux.error(new RuntimeException(res));
                 });
 
     }

diff --git a/...in/java/com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuTextChatService.java b/...in/java/com/hkh/ai/chain/llm/capabilities/generation/text/zhipu/ZhipuTextChatService.java
@@ -76,7 +76,7 @@ public void streamChat(CustomChatMessage request, List<String> nearestList, List
         body.put("messages",messages);
         body.put("stream",true);
         body.put("model",zhipuAiUtil.getCompletionModel());
-        body.put("request_id", UUID.fastUUID());
+        body.put("request_id", UUID.fastUUID().toString(true));
         body.put("temperature",0.95);
 
         ZhipuCompletionBizProcessor bizProcessor = ZhipuCompletionBizProcessor.builder()
@@ -111,7 +111,7 @@ public String blockCompletion(String content) {
         JSONObject body = new JSONObject();
         body.put("messages",messages);
         body.put("model",zhipuAiUtil.getCompletionModel());
-        body.put("request_id", UUID.fastUUID());
+        body.put("request_id", UUID.fastUUID().toString(true));
         body.put("stream",false);
         body.put("temperature",0.95);