feat: 图片识别功能优化

ageerle

2025-06-05 baa664ac4fd1bf72888a59f1099edcf20f22b74f

feat: 图片识别功能优化

 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageQianwenServiceImpl.java

ÎļþÒÑɾ³ý

 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/ImageServiceImpl.java

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,151 @@
package org.ruoyi.chat.service.chat.impl;

import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.ruoyi.chat.config.ChatConfig;
import org.ruoyi.chat.enums.ChatModeType;
import org.ruoyi.chat.listener.SSEEventSourceListener;
import org.ruoyi.chat.service.chat.IChatService;
import org.ruoyi.common.chat.entity.chat.ChatCompletion;
import org.ruoyi.common.chat.entity.chat.Message;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
import org.ruoyi.common.chat.request.ChatRequest;
import org.ruoyi.domain.vo.ChatModelVo;
import org.ruoyi.service.IChatModelService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;

import java.util.*;

/**
 * å›¾ç‰‡è¯†åˆ«æ¨¡åž‹
 */
@Service
@Slf4j
public class ImageServiceImpl implements IChatService {

    @Autowired
    private IChatModelService chatModelService;

    @SneakyThrows
//    @Override
//    public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) {
//        ChatModelVo chatModelVo = chatModelService.selectModelByCategory("image");
//
//        // å‘送流式消息
//
//            MultiModalConversation conv = new MultiModalConversation();
//            MultiModalMessage systemMessage = MultiModalMessage.builder().role(Role.SYSTEM.getValue())
//                    .content(Arrays.asList(
//                            Collections.singletonMap("text",chatRequest.getSysPrompt()))).build();
//            // èŽ·å–用户消息内容
//            List<Message> messages = chatRequest.getMessages();
//            MultiModalMessage userMessage = null;
//            //漫长的格式转换
//            // éåŽ†æ¶ˆæ¯åˆ—表，提取文本内容
//        if (messages != null && !messages.isEmpty()) {
//            Object content = messages.get(messages.size() - 1).getContent();
//            List<Map<String, Object>> contentList = new ArrayList<>();
//            StringBuilder textContent = new StringBuilder();
//            if (content instanceof List<?>) {
//                for (Object item : (List<?>) content) {
//                    if (item instanceof Map<?, ?> mapItem) {
//                        String type = (String) mapItem.get("type");
//                        if ("text".equals(type)) {
//                            String text = (String) mapItem.get("text");
//                            if (text != null) {
//                                textContent.append(text).append(" ");
//                            }
//                        } else if ("image_url".equals(type)) {
//                            Map<String, String> imageUrl  = (Map<String, String>) mapItem.get("image_url");
//                                contentList.add(Collections.singletonMap("image", imageUrl.get("url")));
//                        }
//                    }
//                }
//            }
//            // å°†æ‹¼æŽ¥åŽçš„文本内容添加到 contentList
//            if (textContent.length() > 0) {
//                contentList.add(Collections.singletonMap("text", textContent.toString().trim()));
//            }
//            userMessage = MultiModalMessage.builder()
//                    .role(Role.USER.getValue())
//                    .content(contentList)
//                    .build();
//        }
//            MultiModalConversationParam param = MultiModalConversationParam.builder()
//                    .apiKey(chatModelVo.getApiKey())
//                    .model(chatModelVo.getModelName())
//                    .messages(Arrays.asList(systemMessage, userMessage))
//                    .incrementalOutput(true)
//                    .build();
//
//
//        try {
//            final QwenStreamingResponseBuilder responseBuilder = new QwenStreamingResponseBuilder(param.getModel(),param.getIncrementalOutput() );
//            conv.streamCall(param, new ResultCallback<>() {
//                @SneakyThrows
//                public void onEvent(MultiModalConversationResult result) {
//
//                    String delta = responseBuilder.append(result);
//                    if (Utils.isNotNullOrEmpty(delta)) {
//
//                        emitter.send(delta);
//                        log.info("收到消息片段: {}", delta);
//                    }
//                }
//                public void onComplete() {
//                    emitter.complete();
//                    log.info("消息结束", responseBuilder.build());
//                }
//                public void onError(Exception e) {
//                    log.info("请求失败", e.getMessage());
//                }
//            });
//        } catch (NoApiKeyException e) {
//            emitter.send("请先配置API密钥");
//            throw new RuntimeException(e);
//        } catch (UploadFileException e) {
//            throw new RuntimeException(e);
//        }
//
//
//        return emitter;
//    }

    @Override
    public SseEmitter chat(ChatRequest chatRequest, SseEmitter emitter) {
        // ä»Žæ•°æ®åº“获取 image ç±»åž‹çš„模型配置
        ChatModelVo chatModelVo = chatModelService.selectModelByCategory(ChatModeType.IMAGE.getCode());
        if (chatModelVo == null) {
            log.error("未找到 image ç±»åž‹çš„模型配置");
            emitter.completeWithError(new IllegalStateException("未找到 image ç±»åž‹çš„模型配置"));
            return emitter;
        }

        // åˆ›å»º OpenAI æµå®¢æˆ·ç«¯
        OpenAiStreamClient openAiStreamClient = ChatConfig.createOpenAiStreamClient(chatModelVo.getApiHost(), chatModelVo.getApiKey());
        List<Message> messages = chatRequest.getMessages();

        // åˆ›å»º SSE äº‹ä»¶æºç›‘听器
        SSEEventSourceListener listener = new SSEEventSourceListener(emitter, chatRequest.getUserId(), chatRequest.getSessionId());

        // æž„建聊天完成请求
        ChatCompletion completion = ChatCompletion
                .builder()
                .messages(messages)
                .model(chatModelVo.getModelName()) // ä½¿ç”¨æ•°æ®åº“中配置的模型名称
                .stream(true)
                .build();

        // å‘起流式聊天完成请求
        openAiStreamClient.streamChatCompletion(completion, listener);

        return emitter;
    }

    @Override
    public String getCategory() {
        return ChatModeType.IMAGE.getCode();
    }
}

 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/QianWenAiChatServiceImpl.java

@@ -35,6 +35,8 @@
                .modelName(chatModelVo.getModelName())
                .build();



        // å‘送流式消息
        try {
            model.chat(chatRequest.getPrompt(), new StreamingChatResponseHandler() {

 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/chat/service/chat/impl/SseServiceImpl.java

@@ -131,7 +131,7 @@
            if (chatModelVo == null) {
                log.error("未找到image类型的模型配置");
                throw new IllegalStateException("未找到image类型的模型配置");
            }//            chatRequest.setModel(chatModelVo.getModelName());
            }
        }else{
            chatModelVo = chatModelService.selectModelByName(chatRequest.getModel());
        }

 script/sql/update/20250605.sql

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,7 @@

/*
 ä¸Šä¼ å›¾ç‰‡åŽä¼šè‡ªåŠ¨æŸ¥æ‰¾åˆ†ç±»ä¸ºimage的模型,要使用图片识别功能,需要执行这条sql并配置key信息
*/


INSERT INTO `chat_model` (`id`, `tenant_id`, `category`, `model_name`, `model_describe`, `model_price`, `model_type`, `model_show`, `system_prompt`, `api_host`, `api_key`, `api_url`, `create_dept`, `create_by`, `create_time`, `update_by`, `update_time`, `remark`) VALUES (1930184891812147202, '000000', 'image', 'qwen/qwen2.5-vl-72b-instruct', 'qwen/qwen2.5-vl-72b-instruct', 0.003, '2', '0', NULL, 'https://api.ppinfra.com/v3/openai/chat/completions', 'xx', NULL, 103, 1, '2025-06-04 16:48:34', 1, '2025-06-04 16:48:34', '视觉模型');