From ea09421b0e42e0f4aaefa307055afc4fb48d7522 Mon Sep 17 00:00:00 2001
From: ageerle <ageerle@163.com>
Date: 星期五, 07 三月 2025 09:56:15 +0800
Subject: [PATCH] 增加知识库配置属性
---
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java | 4
ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java | 46 +++++
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java | 18 +
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java | 6
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java | 43 +++-
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java | 39 ++++
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java | 4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java | 8
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java | 6
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java | 4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java | 4
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java | 16 +
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java | 19 +
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java | 48 +---
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java | 9
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java | 37 ++-
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java | 14
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java | 30 ++-
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java | 4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java | 4
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java | 4
script/sql/update/update20250302.sql | 1
ruoyi-admin/src/main/resources/application.yml | 30 ---
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java | 5
/dev/null | 30 ---
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java | 29 +-
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java | 45 +++++
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java | 2
ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java | 2
ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java | 7
ruoyi-admin/src/main/resources/application-dev.yml | 2
38 files changed, 332 insertions(+), 202 deletions(-)
diff --git a/.editorconfig b/.editorconfig
deleted file mode 100644
index 25b312e..0000000
--- a/.editorconfig
+++ /dev/null
@@ -1,18 +0,0 @@
-# http://editorconfig.org
-root = true
-
-# 绌烘牸鏇夸唬Tab缂╄繘鍦ㄥ悇绉嶇紪杈戝伐鍏蜂笅鏁堟灉涓�鑷�
-[*]
-indent_style = space
-indent_size = 4
-charset = utf-8
-end_of_line = lf
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.{json,yml,yaml}]
-indent_size = 2
-
-[*.md]
-insert_final_newline = false
-trim_trailing_whitespace = false
diff --git a/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java b/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java
index 76e5996..1768a69 100644
--- a/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java
+++ b/ruoyi-admin/src/main/java/org/ruoyi/controller/KnowledgeController.java
@@ -81,7 +81,7 @@
List<Message> messages = chatRequest.getMessages();
String content = messages.get(messages.size() - 1).getContent().toString();
List<String> nearestList;
- List<Double> queryVector = embeddingService.getQueryVector(content);
+ List<Double> queryVector = embeddingService.getQueryVector(content, chatRequest.getKid());
nearestList = vectorStore.nearest(queryVector,chatRequest.getKid());
for (String prompt : nearestList) {
Message sysMessage = Message.builder().content(prompt).role(Message.Role.USER).build();
diff --git a/ruoyi-admin/src/main/resources/application-dev.yml b/ruoyi-admin/src/main/resources/application-dev.yml
index 434c6f4..6608a60 100644
--- a/ruoyi-admin/src/main/resources/application-dev.yml
+++ b/ruoyi-admin/src/main/resources/application-dev.yml
@@ -27,7 +27,7 @@
driverClassName: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://43.139.70.230:3306/ruoyi-ai?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true
username: ruoyi-ai
- password: ruoyi-ai
+ password: eCaZ278N62k6fhYj
hikari:
# 鏈�澶ц繛鎺ユ睜鏁伴噺
diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml
index bbaec24..2617356 100644
--- a/ruoyi-admin/src/main/resources/application.yml
+++ b/ruoyi-admin/src/main/resources/application.yml
@@ -310,11 +310,6 @@
# 浼佷笟寰俊搴旂敤
wechat:
- # 鏄惁浣跨敤寰俊 true/false
- enable: true
- # 鐢熸垚鐨勭櫥褰曚簩缁寸爜璺緞 榛樿涓庨」鐩悓绾�
- qrPath: "./"
- # 浼佷笟寰俊搴旂敤
cp:
corpId:
appConfigs:
@@ -323,28 +318,5 @@
token: ''
aesKey: ''
-# 鐭ヨ瘑搴撻厤缃�
-chain:
- split:
- chunk:
- endspliter: "<STOP>"
- # 鍒嗗潡鏂囨湰澶у皬
- size: 200
- overlay: 30
- qaspliter: "###"
- # 鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟
- limits: 5
- vector:
- model: 'text-embedding-3-small'
- store:
- type: weaviate
- weaviate:
- protocol: http
- host: 127.0.0.1:6038
- classname: LocalKnowledge
- milvus:
- host: 127.0.0.1
- port: 19530
- dimension: 1536
- collection: LocalKnowledge
+
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java
index 0ca7ea6..5a74978 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CodeFileLoader.java
@@ -31,7 +31,7 @@
return stringBuffer.toString();
}
@Override
- public List<String> getChunkList(String content){
- return textSplitter.split(content);
+ public List<String> getChunkList(String content, String kid){
+ return textSplitter.split(content, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java
index 5bedf5b..cec04fe 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/CsvFileLoader.java
@@ -10,7 +10,7 @@
}
@Override
- public List<String> getChunkList(String content) {
+ public List<String> getChunkList(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java
index 4682fee..1bbbc7c 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/FolderLoader.java
@@ -10,7 +10,7 @@
}
@Override
- public List<String> getChunkList(String content) {
+ public List<String> getChunkList(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java
index ae86ac4..dd6b449 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/GithubLoader.java
@@ -10,7 +10,7 @@
}
@Override
- public List<String> getChunkList(String content) {
+ public List<String> getChunkList(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java
index 5cd1d6e..5c69389 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/JsonFileLoader.java
@@ -10,7 +10,7 @@
}
@Override
- public List<String> getChunkList(String content) {
+ public List<String> getChunkList(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java
index cecdf05..24d43ff 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/MarkDownFileLoader.java
@@ -31,7 +31,7 @@
return stringBuffer.toString();
}
@Override
- public List<String> getChunkList(String content){
- return textSplitter.split(content);
+ public List<String> getChunkList(String content, String kid){
+ return textSplitter.split(content, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java
index 83566de..0acf355 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/PdfFileLoader.java
@@ -28,7 +28,7 @@
}
@Override
- public List<String> getChunkList(String content) {
- return characterTextSplitter.split(content);
+ public List<String> getChunkList(String content, String kid) {
+ return characterTextSplitter.split(content, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java
index 32f4130..9ef4b11 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/ResourceLoader.java
@@ -7,6 +7,8 @@
* 璧勬簮杞藉叆
*/
public interface ResourceLoader {
+
String getContent(InputStream inputStream);
- List<String> getChunkList(String content);
+
+ List<String> getChunkList(String content, String kid);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java
index a2e74fa..95b8929 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/TextFileLoader.java
@@ -31,7 +31,7 @@
return stringBuffer.toString();
}
@Override
- public List<String> getChunkList(String content){
- return textSplitter.split(content);
+ public List<String> getChunkList(String content, String kid){
+ return textSplitter.split(content, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java
index 7b9d920..c31b9e6 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/loader/WordLoader.java
@@ -30,8 +30,8 @@
}
@Override
- public List<String> getChunkList(String content) {
- return textSplitter.split(content);
+ public List<String> getChunkList(String content, String kid) {
+ return textSplitter.split(content, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java
deleted file mode 100644
index 3b62c6b..0000000
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/retrieve/PromptRetrieverProperties.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.ruoyi.knowledge.chain.retrieve;
-
-import lombok.Data;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Data
-@Component
-public class PromptRetrieverProperties {
- /**
- * 浠庣煡璇嗗簱涓绱㈢殑鏉℃暟锛宭imits 搴斿ぇ浜� num
- */
- @Value("${chain.limits}")
- private int limits;
-
-}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java
index 12db513..cf86b34 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CharacterTextSplitter.java
@@ -1,7 +1,10 @@
package org.ruoyi.knowledge.chain.split;
-import lombok.AllArgsConstructor;
+import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
+import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
+import org.ruoyi.knowledge.service.IKnowledgeInfoService;
+import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
@@ -10,38 +13,46 @@
import java.util.List;
@Component
-@AllArgsConstructor
@Slf4j
@Primary
-public class CharacterTextSplitter implements TextSplitter{
- private final SplitterProperties splitterProperties;
+public class CharacterTextSplitter implements TextSplitter {
+
+ @Lazy
+ @Resource
+ private IKnowledgeInfoService knowledgeInfoService;
+
@Override
- public List<String> split(String content) {
+ public List<String> split(String content, String kid) {
+ // 浠庣煡璇嗗簱琛ㄤ腑鑾峰彇閰嶇疆
+ KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
+ String knowledgeSeparator = knowledgeInfoVo.getKnowledgeSeparator();
+ int textBlockSize = knowledgeInfoVo.getTextBlockSize();
+ int overlapChar = knowledgeInfoVo.getOverlapChar();
List<String> chunkList = new ArrayList<>();
- if (content.contains(splitterProperties.getEndspliter())){
+ if (content.contains(knowledgeSeparator)) {
// 鎸夎嚜瀹氫箟鍒嗛殧绗﹀垏鍒�
- String[] chunks = content.split(splitterProperties.getEndspliter());
+ String[] chunks = content.split(knowledgeSeparator);
chunkList.addAll(Arrays.asList(chunks));
- }else {
+ } else {
int indexMin = 0;
int len = content.length();
int i = 0;
int right = 0;
while (true) {
- if (len > right ){
- int begin = i*splitterProperties.getSize() - splitterProperties.getOverlay();
- if (begin < indexMin){
+ if (len > right) {
+ int begin = i * textBlockSize - overlapChar;
+ if (begin < indexMin) {
begin = indexMin;
}
- int end = splitterProperties.getSize()*(i+1) + splitterProperties.getOverlay();
- if (end > len){
+ int end = textBlockSize * (i + 1) + overlapChar;
+ if (end > len) {
end = len;
}
- String chunk = content.substring(begin,end);
+ String chunk = content.substring(begin, end);
chunkList.add(chunk);
i++;
- right = right + splitterProperties.getSize();
- }else {
+ right = right + textBlockSize;
+ } else {
break;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java
index e68d61c..209f5de 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/CodeTextSplitter.java
@@ -11,7 +11,7 @@
@Slf4j
public class CodeTextSplitter implements TextSplitter{
@Override
- public List<String> split(String content) {
+ public List<String> split(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java
index 5c44781..89ea653 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/MarkdownTextSplitter.java
@@ -11,7 +11,7 @@
@Slf4j
public class MarkdownTextSplitter implements TextSplitter{
@Override
- public List<String> split(String content) {
+ public List<String> split(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java
deleted file mode 100644
index d0d9f5f..0000000
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/SplitterProperties.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.ruoyi.knowledge.chain.split;
-
-import lombok.Data;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@Data
-@Component
-@ConfigurationProperties(prefix = "chain.split.chunk")
-public class SplitterProperties {
- /**
- * 鍒嗘鏍囪瘑绗�
- */
- private String endspliter;
-
- /**
- * 鎻愰棶鍒嗘鏍囪瘑绗�
- */
- private String qaspliter;
-
- /**
- * 鍒嗗潡鏂囨湰澶у皬
- */
- private int size;
-
- /**
- * 鐩搁偦鍧椾箣闂撮噸鍙犵殑瀛楃鏁�(閬垮厤杈圭晫淇℃伅涓㈠け)
- */
- private int overlay;
-}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java
index 5d3d71b..86b3aaf 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TextSplitter.java
@@ -7,5 +7,12 @@
*/
public interface TextSplitter {
- List<String> split(String content);
+ /**
+ * 鏂囨湰鍒囧垎
+ *
+ * @param content 鏂囨湰鍐呭
+ * @param kid 鐭ヨ瘑搴搃d
+ * @return 鍒囧垎鍚庣殑鏂囨湰鍒楄〃
+ */
+ List<String> split(String content, String kid);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java
index dedc631..696f261 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/split/TokenTextSplitter.java
@@ -11,7 +11,7 @@
@Slf4j
public class TokenTextSplitter implements TextSplitter{
@Override
- public List<String> split(String content) {
+ public List<String> split(String content, String kid) {
return null;
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java
index 2ac0143..0f2d0ba 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/OpenAiVectorization.java
@@ -1,5 +1,6 @@
package org.ruoyi.knowledge.chain.vectorizer;
+import jakarta.annotation.Resource;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -8,7 +9,10 @@
import org.ruoyi.common.chat.entity.embeddings.EmbeddingResponse;
import org.ruoyi.common.chat.openai.OpenAiStreamClient;
+import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
+import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
@@ -20,8 +24,9 @@
@RequiredArgsConstructor
public class OpenAiVectorization implements Vectorization {
- @Value("${chain.vector.model}")
- private String embeddingModel;
+ @Lazy
+ @Resource
+ private IKnowledgeInfoService knowledgeInfoService;
@Getter
private OpenAiStreamClient openAiStreamClient;
@@ -29,12 +34,12 @@
private final ChatConfig chatConfig;
@Override
- public List<List<Double>> batchVectorization(List<String> chunkList) {
+ public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
openAiStreamClient = chatConfig.getOpenAiStreamClient();
-
+ KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Embedding embedding = Embedding.builder()
.input(chunkList)
- .model(embeddingModel)
+ .model(knowledgeInfoVo.getVectorModel())
.build();
EmbeddingResponse embeddings = openAiStreamClient.embeddings(embedding);
List<List<Double>> vectorList = new ArrayList<>();
@@ -50,10 +55,10 @@
}
@Override
- public List<Double> singleVectorization(String chunk) {
+ public List<Double> singleVectorization(String chunk, String kid) {
List<String> chunkList = new ArrayList<>();
chunkList.add(chunk);
- List<List<Double>> vectorList = batchVectorization(chunkList);
+ List<List<Double>> vectorList = batchVectorization(chunkList, kid);
return vectorList.get(0);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java
index c770a16..1c25dde 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/Vectorization.java
@@ -6,6 +6,7 @@
* 鍚戦噺鍖�
*/
public interface Vectorization {
- List<List<Double>> batchVectorization(List<String> chunkList);
- List<Double> singleVectorization(String chunk);
+ List<List<Double>> batchVectorization(List<String> chunkList, String kid);
+
+ List<Double> singleVectorization(String chunk, String kid);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java
index 1522148..e41797f 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorizer/VectorizationWrapper.java
@@ -15,14 +15,14 @@
private final VectorizationFactory vectorizationFactory;
@Override
- public List<List<Double>> batchVectorization(List<String> chunkList) {
+ public List<List<Double>> batchVectorization(List<String> chunkList, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding();
- return embedding.batchVectorization(chunkList);
+ return embedding.batchVectorization(chunkList, kid);
}
@Override
- public List<Double> singleVectorization(String chunk) {
+ public List<Double> singleVectorization(String chunk, String kid) {
Vectorization embedding = vectorizationFactory.getEmbedding();
- return embedding.singleVectorization(chunk);
+ return embedding.singleVectorization(chunk, kid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java
index 74bef6f..c083ef4 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/MilvusVectorStore.java
@@ -19,8 +19,10 @@
import io.milvus.response.QueryResultsWrapper;
import io.milvus.response.SearchResultsWrapper;
import jakarta.annotation.PostConstruct;
+import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.ruoyi.common.core.service.ConfigService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@@ -32,26 +34,27 @@
@Slf4j
public class MilvusVectorStore implements VectorStore{
-
- @Value("${chain.vector.store.milvus.host}")
- private String milvusHost;
- @Value("${chain.vector.store.milvus.port}")
- private Integer milvausPort;
-
- @Value("${chain.vector.store.milvus.dimension}")
- private Integer dimension;
-
- @Value("${chain.vector.store.milvus.collection}")
- private String collectionName;
-
+ private volatile Integer dimension;
+ private volatile String collectionName;
private MilvusServiceClient milvusServiceClient;
+
+ @Resource
+ private ConfigService configService;
+
+ @PostConstruct
+ public void loadConfig() {
+ this.dimension = Integer.parseInt(configService.getConfigValue("milvus", "dimension"));
+ this.collectionName = configService.getConfigValue("milvus", "collection");
+ }
@PostConstruct
public void init(){
+ String milvusHost = configService.getConfigValue("milvus", "host");
+ String milvausPort = configService.getConfigValue("milvus", "port");
milvusServiceClient = new MilvusServiceClient(
ConnectParam.newBuilder()
.withHost(milvusHost)
- .withPort(milvausPort)
+ .withPort(Integer.parseInt(milvausPort))
.withDatabaseName("default")
.build()
);
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java
index 3478229..ae6e73c 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreFactory.java
@@ -1,29 +1,37 @@
package org.ruoyi.knowledge.chain.vectorstore;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
+import org.ruoyi.knowledge.domain.KnowledgeInfo;
+import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
+import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
@Component
@Slf4j
+@RequiredArgsConstructor
public class VectorStoreFactory {
-
- @Value("${chain.vector.store.type}")
- private String type;
private final WeaviateVectorStore weaviateVectorStore;
private final MilvusVectorStore milvusVectorStore;
- public VectorStoreFactory(WeaviateVectorStore weaviateVectorStore, MilvusVectorStore milvusVectorStore) {
- this.weaviateVectorStore = weaviateVectorStore;
- this.milvusVectorStore = milvusVectorStore;
- }
+ private final KnowledgeInfoMapper knowledgeInfoMapper;
- public VectorStore getVectorStore(){
- if ("weaviate".equals(type)){
+ public VectorStore getVectorStore(String kid){
+ KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoMapper.selectVoOne(
+ new LambdaQueryWrapper<KnowledgeInfo>().eq(KnowledgeInfo::getKid,kid)
+ );
+ String vectorModel = knowledgeInfoVo.getVector();
+ if ("weaviate".equals(vectorModel)){
return weaviateVectorStore;
- }else if ("milvus".equals(type)){
+ }else if ("milvus".equals(vectorModel)){
return milvusVectorStore;
}
return null;
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java
index 2c0a732..eb6b1f4 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/VectorStoreWrapper.java
@@ -16,43 +16,43 @@
private final VectorStoreFactory vectorStoreFactory;
@Override
public void storeEmbeddings(List<String> chunkList, List<List<Double>> vectorList, String kid, String docId, List<String> fidList) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.storeEmbeddings(chunkList, vectorList, kid, docId, fidList);
}
@Override
public void removeByDocId(String kid, String docId) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByDocId(kid,docId);
}
@Override
public void removeByKid(String kid) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKid(kid);
}
@Override
public List<String> nearest(List<Double> queryVector, String kid) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(queryVector,kid);
}
@Override
public List<String> nearest(String query, String kid) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
return vectorStore.nearest(query, kid);
}
@Override
public void newSchema(String kid) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.newSchema(kid);
}
@Override
public void removeByKidAndFid(String kid, String fid) {
- VectorStore vectorStore = vectorStoreFactory.getVectorStore();
+ VectorStore vectorStore = vectorStoreFactory.getVectorStore(kid);
vectorStore.removeByKidAndFid(kid, fid);
}
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java
index 335e4ed..8895a25 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/chain/vectorstore/WeaviateVectorStore.java
@@ -22,10 +22,15 @@
import io.weaviate.client.v1.schema.model.Property;
import io.weaviate.client.v1.schema.model.Schema;
import io.weaviate.client.v1.schema.model.WeaviateClass;
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
-import org.ruoyi.knowledge.chain.retrieve.PromptRetrieverProperties;
+import org.ruoyi.common.core.service.ConfigService;
+import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
+import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@@ -37,18 +42,23 @@
@Slf4j
public class WeaviateVectorStore implements VectorStore{
- @Value("${chain.vector.store.weaviate.protocol}")
- private String protocol;
- @Value("${chain.vector.store.weaviate.host}")
- private String host;
+ private volatile String protocol;
+ private volatile String host;
+ private volatile String className;
- @Value("${chain.vector.store.weaviate.classname}")
- private String className;
+ @Lazy
+ @Resource
+ private IKnowledgeInfoService knowledgeInfoService;
- private final PromptRetrieverProperties promptRetrieverProperties;
+ @Lazy
+ @Resource
+ private ConfigService configService;
- public WeaviateVectorStore(PromptRetrieverProperties promptRetrieverProperties) {
- this.promptRetrieverProperties = promptRetrieverProperties;
+ @PostConstruct
+ public void loadConfig() {
+ this.protocol = configService.getConfigValue("weaviate", "protocol");
+ this.host = configService.getConfigValue("weaviate", "host");
+ this.className = configService.getConfigValue("weaviate", "classname");
}
public WeaviateClient getClient(){
@@ -309,11 +319,12 @@
.vector(vf)
.distance(1.6f) // certainty = 1f - distance /2f
.build();
+ KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Result<GraphQLResponse> result = client.graphQL().get()
.withClassName(className + kid)
.withFields(contentField,_additional)
.withNearVector(nearVector)
- .withLimit(promptRetrieverProperties.getLimits())
+ .withLimit(knowledgeInfoVo.getRetrieveLimit())
.run();
LinkedTreeMap<String,Object> t = (LinkedTreeMap<String, Object>) result.getResult().getData();
LinkedTreeMap<String,ArrayList<LinkedTreeMap>> l = (LinkedTreeMap<String, ArrayList<LinkedTreeMap>>) t.get("Get");
@@ -342,12 +353,12 @@
.concepts(new String[]{ query })
.distance(1.6f) // certainty = 1f - distance /2f
.build();
-
+ KnowledgeInfoVo knowledgeInfoVo = knowledgeInfoService.queryById(Long.valueOf(kid));
Result<GraphQLResponse> result = client.graphQL().get()
.withClassName(className + kid)
.withFields(contentField,_additional)
.withNearText(nearText)
- .withLimit(promptRetrieverProperties.getLimits())
+ .withLimit(knowledgeInfoVo.getRetrieveLimit())
.run();
LinkedTreeMap<String,Object> t = (LinkedTreeMap<String, Object>) result.getResult().getData();
LinkedTreeMap<String,ArrayList<LinkedTreeMap>> l = (LinkedTreeMap<String, ArrayList<LinkedTreeMap>>) t.get("Get");
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java
index 0d9b51e..fb8407d 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/KnowledgeInfo.java
@@ -2,6 +2,7 @@
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
+import jakarta.validation.constraints.NotBlank;
import lombok.Data;
import java.io.Serial;
@@ -44,6 +45,11 @@
private String kname;
/**
+ * 鐭ヨ瘑搴撳悕绉�
+ */
+ private String share;
+
+ /**
* 鎻忚堪
*/
private String description;
@@ -59,5 +65,38 @@
*/
private Date createTime;
+ /**
+ * 鐭ヨ瘑鍒嗛殧绗�
+ */
+ private String knowledgeSeparator;
+ /**
+ * 鎻愰棶鍒嗛殧绗�
+ */
+ private String questionSeparator;
+
+ /**
+ * 閲嶅彔瀛楃鏁�
+ */
+ private Integer overlapChar;
+
+ /**
+ * 鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟
+ */
+ private Integer retrieveLimit;
+
+ /**
+ * 鏂囨湰鍧楀ぇ灏�
+ */
+ private Integer textBlockSize;
+
+ /**
+ * 鍚戦噺搴�
+ */
+ private String vector;
+
+ /**
+ * 鍚戦噺妯″瀷
+ */
+ private String vectorModel;
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java
index bea7b25..b606d82 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/bo/KnowledgeInfoBo.java
@@ -44,10 +44,56 @@
private String kname;
/**
+ * 鐭ヨ瘑搴撳悕绉�
+ */
+ @NotBlank(message = "鏄惁鍏紑鐭ヨ瘑搴�")
+ private String share;
+
+ /**
* 鎻忚堪
*/
@NotBlank(message = "鎻忚堪涓嶈兘涓虹┖")
private String description;
+ /**
+ * 鐭ヨ瘑鍒嗛殧绗�
+ */
+ @NotBlank(message = "鐭ヨ瘑鍒嗛殧绗︿笉鑳戒负绌�")
+ private String knowledgeSeparator;
+ /**
+ * 鎻愰棶鍒嗛殧绗�
+ */
+ @NotBlank(message = "鎻愰棶鍒嗛殧绗︿笉鑳戒负绌�")
+ private String questionSeparator;
+
+ /**
+ * 閲嶅彔瀛楃鏁�
+ */
+ @NotNull(message = "閲嶅彔瀛楃鏁颁笉鑳戒负绌�")
+ private Integer overlapChar;
+
+ /**
+ * 鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟
+ */
+ @NotNull(message = "鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟涓嶈兘涓虹┖")
+ private Integer retrieveLimit;
+
+ /**
+ * 鏂囨湰鍧楀ぇ灏�
+ */
+ @NotNull(message = "鏂囨湰鍧楀ぇ灏忎笉鑳戒负绌�")
+ private Integer textBlockSize;
+
+ /**
+ * 鍚戦噺搴�
+ */
+ @NotBlank(message = "鍚戦噺搴撲笉鑳戒负绌�")
+ private String vector;
+
+ /**
+ * 鍚戦噺妯″瀷
+ */
+ @NotBlank(message = "鍚戦噺妯″瀷涓嶈兘涓虹┖")
+ private String vectorModel;
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java
index c541a82..a47b5d6 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/domain/vo/KnowledgeInfoVo.java
@@ -48,10 +48,55 @@
private String kname;
/**
+ * 鐭ヨ瘑搴撳悕绉�
+ */
+ private String share;
+
+ /**
* 鎻忚堪
*/
@ExcelProperty(value = "鎻忚堪")
private String description;
+ /**
+ * 鐭ヨ瘑鍒嗛殧绗�
+ */
+ @ExcelProperty(value = "鐭ヨ瘑鍒嗛殧绗�")
+ private String knowledgeSeparator;
+ /**
+ * 鎻愰棶鍒嗛殧绗�
+ */
+ @ExcelProperty(value = "鎻愰棶鍒嗛殧绗�")
+ private String questionSeparator;
+
+ /**
+ * 閲嶅彔瀛楃鏁�
+ */
+ @ExcelProperty(value = "閲嶅彔瀛楃鏁�")
+ private Integer overlapChar;
+
+ /**
+ * 鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟
+ */
+ @ExcelProperty(value = "鐭ヨ瘑搴撲腑妫�绱㈢殑鏉℃暟")
+ private Integer retrieveLimit;
+
+ /**
+ * 鏂囨湰鍧楀ぇ灏�
+ */
+ @ExcelProperty(value = "鏂囨湰鍧楀ぇ灏�")
+ private Integer textBlockSize;
+
+ /**
+ * 鍚戦噺搴�
+ */
+ @ExcelProperty(value = "鍚戦噺搴�")
+ private String vector;
+
+ /**
+ * 鍚戦噺妯″瀷
+ */
+ @ExcelProperty(value = "鍚戦噺妯″瀷")
+ private String vectorModel;
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java
index 739c7e6..a4dd30d 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/EmbeddingService.java
@@ -10,7 +10,7 @@
void removeByKid(String kid);
- List<Double> getQueryVector(String query);
+ List<Double> getQueryVector(String query, String kid);
void createSchema(String kid);
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java
index 0c1b090..121f8fb 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/IKnowledgeInfoService.java
@@ -55,4 +55,10 @@
* 鍒犻櫎鐭ヨ瘑搴�
*/
void removeKnowledge(String id);
+
+ /**
+ * 妫�鏌ユ槸鍚︽湁鍒犻櫎鏉冮檺
+ * @param knowledgeInfoList 鐭ヨ瘑鍒楄〃
+ */
+ void check(List<KnowledgeInfoVo> knowledgeInfoList);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java
index 8850311..73503a4 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/EmbeddingServiceImpl.java
@@ -24,7 +24,7 @@
*/
@Override
public void storeEmbeddings(List<String> chunkList, String kid, String docId,List<String> fidList) {
- List<List<Double>> vectorList = vectorization.batchVectorization(chunkList);
+ List<List<Double>> vectorList = vectorization.batchVectorization(chunkList, kid);
vectorStore.storeEmbeddings(chunkList,vectorList,kid,docId,fidList);
}
@@ -39,8 +39,8 @@
}
@Override
- public List<Double> getQueryVector(String query) {
- List<Double> queryVector = vectorization.singleVectorization(query);
+ public List<Double> getQueryVector(String query, String kid) {
+ List<Double> queryVector = vectorization.singleVectorization(query,kid);
return queryVector;
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java
index 07812d4..9bea5dd 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeAttachServiceImpl.java
@@ -4,16 +4,21 @@
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.RequiredArgsConstructor;
+import org.ruoyi.common.core.domain.model.LoginUser;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.common.mybatis.core.page.PageQuery;
import org.ruoyi.common.mybatis.core.page.TableDataInfo;
+import org.ruoyi.common.satoken.utils.LoginHelper;
import org.ruoyi.knowledge.domain.KnowledgeAttach;
import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo;
import org.ruoyi.knowledge.domain.vo.KnowledgeAttachVo;
+import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
import org.ruoyi.knowledge.mapper.KnowledgeAttachMapper;
import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper;
+import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.ruoyi.knowledge.service.IKnowledgeAttachService;
+import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import java.util.Collection;
@@ -34,6 +39,11 @@
private final KnowledgeAttachMapper baseMapper;
private final KnowledgeFragmentMapper fragmentMapper;
+
+ private final KnowledgeInfoMapper knowledgeInfoMapper;
+
+ private final IKnowledgeInfoService knowledgeInfoService;
+
/**
* 鏌ヨ鐭ヨ瘑搴撻檮浠�
@@ -117,8 +127,12 @@
@Override
public void removeKnowledgeAttach(String kid) {
- HashMap<String, Object> map = new HashMap<>();
- map.put("kid", kid);
+ LoginUser loginUser = LoginHelper.getLoginUser();
+ Map<String,Object> map = new HashMap<>();
+ map.put("kid",kid);
+ List<KnowledgeInfoVo> knowledgeInfoList = knowledgeInfoMapper.selectVoByMap(map);
+ knowledgeInfoService.check(knowledgeInfoList);
+
baseMapper.deleteByMap(map);
fragmentMapper.deleteByMap(map);
}
diff --git a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java
index 0f2b842..3bd9390 100644
--- a/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java
+++ b/ruoyi-modules/ruoyi-knowledge/src/main/java/org/ruoyi/knowledge/service/impl/KnowledgeInfoServiceImpl.java
@@ -1,16 +1,16 @@
package org.ruoyi.knowledge.service.impl;
import cn.hutool.core.util.RandomUtil;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.github.ollama4j.OllamaAPI;
-import io.github.ollama4j.exceptions.OllamaBaseException;
import io.github.ollama4j.models.chat.OllamaChatMessageRole;
import io.github.ollama4j.models.chat.OllamaChatRequestBuilder;
import io.github.ollama4j.models.chat.OllamaChatRequestModel;
import io.github.ollama4j.models.chat.OllamaChatResult;
import lombok.RequiredArgsConstructor;
+import org.ruoyi.common.core.domain.model.LoginUser;
import org.ruoyi.common.core.utils.MapstructUtils;
import org.ruoyi.common.core.utils.StringUtils;
import org.ruoyi.common.mybatis.core.page.PageQuery;
@@ -21,8 +21,6 @@
import org.ruoyi.knowledge.domain.KnowledgeAttach;
import org.ruoyi.knowledge.domain.KnowledgeFragment;
import org.ruoyi.knowledge.domain.KnowledgeInfo;
-import org.ruoyi.knowledge.domain.bo.KnowledgeAttachBo;
-import org.ruoyi.knowledge.domain.bo.KnowledgeFragmentBo;
import org.ruoyi.knowledge.domain.bo.KnowledgeInfoBo;
import org.ruoyi.knowledge.domain.req.KnowledgeInfoUploadRequest;
import org.ruoyi.knowledge.domain.vo.KnowledgeInfoVo;
@@ -30,14 +28,11 @@
import org.ruoyi.knowledge.mapper.KnowledgeFragmentMapper;
import org.ruoyi.knowledge.mapper.KnowledgeInfoMapper;
import org.ruoyi.knowledge.service.EmbeddingService;
-import org.ruoyi.knowledge.service.IKnowledgeAttachService;
-import org.ruoyi.knowledge.service.IKnowledgeFragmentService;
import org.ruoyi.knowledge.service.IKnowledgeInfoService;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
-import java.time.LocalDateTime;
import java.util.*;
/**
@@ -88,12 +83,13 @@
}
private LambdaQueryWrapper<KnowledgeInfo> buildQueryWrapper(KnowledgeInfoBo bo) {
- Map<String, Object> params = bo.getParams();
LambdaQueryWrapper<KnowledgeInfo> lqw = Wrappers.lambdaQuery();
lqw.eq(StringUtils.isNotBlank(bo.getKid()), KnowledgeInfo::getKid, bo.getKid());
lqw.eq(bo.getUid() != null, KnowledgeInfo::getUid, bo.getUid());
lqw.like(StringUtils.isNotBlank(bo.getKname()), KnowledgeInfo::getKname, bo.getKname());
lqw.eq(StringUtils.isNotBlank(bo.getDescription()), KnowledgeInfo::getDescription, bo.getDescription());
+ // 鏌ヨ鍏紑鐨勭煡璇嗗簱
+ lqw.or(wrapper -> wrapper.eq(KnowledgeInfo::getShare, "1"));
return lqw;
}
@@ -151,7 +147,7 @@
List<String> fids = new ArrayList<>();
try {
content = resourceLoader.getContent(file.getInputStream());
- chunkList = resourceLoader.getChunkList(content);
+ chunkList = resourceLoader.getChunkList(content, kid);
for (int i = 0; i < chunkList.size(); i++) {
String fid = RandomUtil.randomString(16);
fids.add(fid);
@@ -179,6 +175,8 @@
Map<String,Object> map = new HashMap<>();
map.put("kid",id);
+ List<KnowledgeInfoVo> knowledgeInfoList = baseMapper.selectVoByMap(map);
+ check(knowledgeInfoList);
// 鍒犻櫎鐭ヨ瘑搴�
baseMapper.deleteByMap(map);
// 鍒犻櫎闄勪欢鍜岀煡璇嗙墖娈�
@@ -188,31 +186,13 @@
embeddingService.removeByKid(id);
}
- /**
- * 灏嗘枃鏈潡杞崲涓洪璁粌鏁版嵁
- * @param chunk 瑙ｆ瀽鏂囨湰鍧�
- */
- public String convertTextBlockToPretrainData(String chunk){
- String host = "http://localhost:11434/";
- OllamaAPI ollama = new OllamaAPI(host);
- OllamaChatRequestBuilder builder = OllamaChatRequestBuilder.getInstance("qwen2.5:7b");
- // 璁剧疆瓒呮椂鏃堕棿
- ollama.setRequestTimeoutSeconds(100);
- // create first user question
- String json = "instruction:鐢ㄦ埛鎸囦护,鏍规嵁璇箟鎻愬彇涓�涓叧閿瘝;input:鐢ㄦ埛杈撳叆,鏍规嵁璇箟鎻愬彇澶氫釜鍏抽敭璇�;output:杈撳嚭鏂囨湰鍐呭";
-
- OllamaChatRequestModel requestModel = builder.withMessage
- (OllamaChatMessageRole.USER, "鏂囨湰锛�"+chunk+"鐞嗚В鏂囨湰鍐呭锛屽苟涓斿皢鏂囨湰鍐呭杞崲涓�:"+json+",杈撳嚭JSON鏍煎紡锛屼笉瑕佸寘鍚叾浠栨棤鍏冲唴瀹�,鍐呴儴浣跨敤鏃犻渶鑴辨晱")
- .build();
-
- // start conversation with model
- OllamaChatResult chatResult = null;
- try {
- chatResult = ollama.chat(requestModel);
- } catch (Exception e) {
- System.out.println("瑙ｆ瀽澶辫触!");
+ @Override
+ public void check(List<KnowledgeInfoVo> knowledgeInfoList){
+ LoginUser loginUser = LoginHelper.getLoginUser();
+ for (KnowledgeInfoVo knowledgeInfoVo : knowledgeInfoList) {
+ if(!knowledgeInfoVo.getUid().equals(loginUser.getUserId())){
+ throw new SecurityException("鏉冮檺涓嶈冻");
}
- return chatResult.getResponse();
+ }
}
-
}
diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java
index 4af2348..a77a0d3 100644
--- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java
+++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/controller/system/ChatConfigController.java
@@ -36,8 +36,8 @@
*/
@GetMapping("/list")
@SaCheckPermission("system:config:list")
- public List<ChatConfigVo> list(ChatConfigBo bo) {
- return chatConfigService.queryList(bo);
+ public R<List<ChatConfigVo>> list(ChatConfigBo bo) {
+ return R.ok(chatConfigService.queryList(bo));
}
/**
diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java
index 5148f34..04fa785 100644
--- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java
+++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/domain/request/translation/TranslationRequest.java
@@ -22,13 +22,8 @@
private String model;
/**
- * 婧愯瑷�
- */
- private String sourceLanguage;
-
- /**
* 鐩爣璇█
*/
- private String targetLanguage;
+ private String targetLanguage;
}
diff --git a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java
index 7ada900..e2f02b5 100644
--- a/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java
+++ b/ruoyi-modules/ruoyi-system/src/main/java/org/ruoyi/system/service/impl/SseServiceImpl.java
@@ -553,7 +553,21 @@
List<Message> messageList = new ArrayList<>();
- Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("浣滀负鑻辨眽缈昏瘧锛屾偍鐨勪换鍔℃槸鍑嗙‘鍦板湪涓ょ璇█涔嬮棿缈昏瘧鏂囨湰銆傜炕璇戞椂锛岃娉ㄦ剰涓婁笅鏂囷紝鍑嗙‘瑙ｉ噴鎴愯鍜岃皻璇�傚鏋滆繛缁敹鍒板涓嫳鏂囧崟璇嶏紝璇烽粯璁ゅ皢鍏剁炕璇戞垚涓枃鍙ュ瓙銆備絾濡傛灉鍓嶉潰鏈�'phrase:鈥欙紝鍒欏簲缈昏瘧涓虹煭璇�;濡傛灉鏈�'norma!:'锛屽垯缈昏瘧涓哄涓棤鍏崇殑鍗曡瘝銆傛偍鐨勭炕璇戝簲鎺ヨ繎姣嶈鑰呯殑姘村钩锛屽苟鑰冭檻鐢ㄦ埛瑕佹眰鐨勭壒瀹氳瑷�椋庢牸鎴栬姘斻�傞伩鍏嶄娇鐢ㄥ啋鐘�ц瘝姹囷紝蹇呰鏃剁敤x鏇挎崲銆傛彁渚涚炕璇戞椂锛岃鐢ㄤ腑鏂囪В閲婃瘡鍙ヨ瘽鐨勬椂鎬併�佷粠鍙ャ�佷富璇�佽皳璇�佸璇�佺壒娈婄煭璇拰璋氳瀵逛簬闇�瑕佺炕璇戠殑鐭鎴栧崟璇嶏紝璇锋彁渚涙潵婧�(璇嶅吀)銆傚鏋滆姹傜炕璇戝涓煭璇紝璇风敤|绗﹀彿鍒嗛殧銆傝璁颁綇:鎮ㄦ槸鑻辨眽缈昏瘧锛屼笉鏄眽姹夌炕璇戞垨鑻辫嫳缈昏瘧銆傛彁浜ゅ墠璇蜂粩缁嗘鏌ュ拰淇绛旀,鍥炲鎺у埗鍦�50瀛椾互鍐�").build();
+ Message sysMessage = Message.builder().role(Message.Role.SYSTEM).content("浣犳槸涓�鍚嶇炕璇戣�佸笀\n" +
+ "\n" +
+ "璇峰皢鐢ㄦ埛杈撳叆璇嶈缈昏瘧鎴恵" + translationRequest.getTargetLanguage() + "}\n" +
+ "\n" +
+ "璁╂垜浠竴姝ヤ竴姝ユ潵鎬濊�僜n" +
+ "==绀轰緥杈撳嚭==\n" +
+ "**缈昏瘧** : <杩欓噷鏄剧ず缈昏瘧鎴愯嫳璇殑缁撴灉>\n" +
+ "\n" +
+ "**閫犲彞** : What's the weather like today? Use the 'Weather Query' plugin to find out instantly! <閫犱竴涓嫳璇彞瀛�>\n" +
+ "\n" +
+ "**鍚屼箟璇�** : Add-on銆丒xtension銆丮odule <杩欓噷鏄剧ず1-3涓嫳鏂囩殑鍚屼箟璇�>\n" +
+ "\n" +
+ "==绀轰緥缁撴潫==\n" +
+ "\n" +
+ "娉ㄦ剰锛氳涓ユ牸鎸夌ず渚嬭繘琛岃緭鍑�").build();
messageList.add(sysMessage);
Message message = Message.builder().role(Message.Role.USER).content(translationRequest.getPrompt()).build();
messageList.add(message);
diff --git a/script/sql/update/update20250302.sql b/script/sql/update/update20250302.sql
new file mode 100644
index 0000000..b3aa42b
--- /dev/null
+++ b/script/sql/update/update20250302.sql
@@ -0,0 +1 @@
+ALTER TABLE `knowledge_info` ADD COLUMN `share` tinyint(4) NULL DEFAULT NULL COMMENT '鏄惁鍏紑鐭ヨ瘑搴擄紙0 鍚� 1鏄級' AFTER `kname`;
--
Gitblit v1.9.3