python脚本

ageer

2025-03-15 fe62ae4d5e9f5bb49d32323606a1312c381953c2

python脚本

 README.md

@@ -158,7 +158,7 @@
- [Naive UI](https://www.naiveui.com)
- [RuoYi-Vue-Plus](https://gitee.com/dromara/RuoYi-Vue-Plus)

## Community
## è´¡çŒ®è€…
<a href="https://github.com/ageerle/ruoyi-ai/graphs/contributors">
  <img src="https://contrib.rocks/image?repo=ageerle/ruoyi-ai" />
</a>

 script/docker/localModels/Dockerfile

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,21 @@
# ä½¿ç”¨å®˜æ–¹ Python ä½œä¸ºåŸºç¡€é•œåƒ
FROM python:3.8-slim

# è®¾ç½®å·¥ä½œç›®å½•ä¸º /app
WORKDIR /app

# å¤åˆ¶å½“前目录下的所有文件到 Docker å®¹å™¨çš„ /app ç›®å½•
COPY . /app

# å®‰è£…应用依赖
RUN pip install --no-cache-dir -r requirements.txt

# æš´éœ² Flask åº”用使用的端口
EXPOSE 5000

# è®¾ç½®çŽ¯å¢ƒå˜é‡
ENV FLASK_APP=app.py
ENV FLASK_RUN_HOST=0.0.0.0

# å¯åŠ¨ Flask åº”用
CMD ["flask", "run", "--host=0.0.0.0"]

 script/docker/localModels/app.py

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,116 @@
from flask import Flask, request, jsonify
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import json

app = Flask(__name__)

# åˆ›å»ºä¸€ä¸ªå…¨å±€çš„模型缓存字典
model_cache = {}

# åˆ†å‰²æ–‡æœ¬å—
def split_text(text, block_size, overlap_chars, delimiter):
    chunks = text.split(delimiter)
    text_blocks = []
    current_block = ""

    for chunk in chunks:
        if len(current_block) + len(chunk) + 1 <= block_size:
            if current_block:
                current_block += " " + chunk
            else:
                current_block = chunk
        else:
            text_blocks.append(current_block)
            current_block = chunk
    if current_block:
        text_blocks.append(current_block)

    overlap_blocks = []
    for i in range(len(text_blocks)):
        if i > 0:
            overlap_block = text_blocks[i - 1][-overlap_chars:] + text_blocks[i]
            overlap_blocks.append(overlap_block)
        overlap_blocks.append(text_blocks[i])

    return overlap_blocks

# æ–‡æœ¬å‘量化
def vectorize_text_blocks(text_blocks, model):
    return model.encode(text_blocks)

# æ–‡æœ¬æ£€ç´¢
def retrieve_top_k(query, knowledge_base, k, block_size, overlap_chars, delimiter, model):
    # å°†çŸ¥è¯†åº“拆分为文本块
    text_blocks = split_text(knowledge_base, block_size, overlap_chars, delimiter)
    # å‘量化文本块
    knowledge_vectors = vectorize_text_blocks(text_blocks, model)
    # å‘量化查询文本
    query_vector = model.encode([query]).reshape(1, -1)
    # è®¡ç®—相似度
    similarities = cosine_similarity(query_vector, knowledge_vectors)
    # èŽ·å–相似度最高的 k ä¸ªæ–‡æœ¬å—的索引
    top_k_indices = similarities[0].argsort()[-k:][::-1]

    # è¿”回文本块和它们的向量
    top_k_texts = [text_blocks[i] for i in top_k_indices]
    top_k_embeddings = [knowledge_vectors[i] for i in top_k_indices]

    return top_k_texts, top_k_embeddings

@app.route('/vectorize', methods=['POST'])
def vectorize_text():
    # ä»Žè¯·æ±‚中获取 JSON æ•°æ®
    data = request.json
    print(f"Received request data: {data}")  # è°ƒè¯•è¾“出请求数据

    text_list = data.get("text", [])
    model_name = data.get("model_name", "msmarco-distilbert-base-tas-b")  # é»˜è®¤æ¨¡åž‹

    delimiter = data.get("delimiter", "\n")  # é»˜è®¤åˆ†éš”符
    k = int(data.get("k", 3))  # é»˜è®¤æ£€ç´¢æ¡æ•°
    block_size = int(data.get("block_size", 500))  # é»˜è®¤æ–‡æœ¬å—大小
    overlap_chars = int(data.get("overlap_chars", 50))  # é»˜è®¤é‡å å­—符数

    if not text_list:
        return jsonify({"error": "Text is required."}), 400

    # æ£€æŸ¥æ¨¡åž‹æ˜¯å¦å·²ç»åŠ è½½
    if model_name not in model_cache:
        try:
            model = SentenceTransformer(model_name)
            model_cache[model_name] = model  # ç¼“存模型
        except Exception as e:
            return jsonify({"error": f"Failed to load model: {e}"}), 500

    model = model_cache[model_name]

    top_k_texts_all = []
    top_k_embeddings_all = []

    # å¦‚果只有一个查询文本
    if len(text_list) == 1:
        top_k_texts, top_k_embeddings = retrieve_top_k(text_list[0], text_list[0], k, block_size, overlap_chars, delimiter, model)
        top_k_texts_all.append(top_k_texts)
        top_k_embeddings_all.append(top_k_embeddings)
    elif len(text_list) > 1:
        # å¦‚果多个查询文本，依次处理
        for query in text_list:
            top_k_texts, top_k_embeddings = retrieve_top_k(query, text_list[0], k, block_size, overlap_chars, delimiter, model)
            top_k_texts_all.append(top_k_texts)
            top_k_embeddings_all.append(top_k_embeddings)

    # å°†åµŒå…¥å‘量（ndarray）转换为可序列化的列表
    top_k_embeddings_all = [[embedding.tolist() for embedding in embeddings] for embeddings in top_k_embeddings_all]

    print(f"Top K texts: {top_k_texts_all}")  # æ‰“印检索到的文本
    print(f"Top K embeddings: {top_k_embeddings_all}")  # æ‰“印检索到的向量

    # è¿”回 JSON æ ¼å¼çš„数据
    return jsonify({

        "topKEmbeddings": top_k_embeddings_all  # è¿”回嵌入向量
    })

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=5000, debug=True)

 script/docker/localModels/requirements.txt

¶Ô±ÈÐÂÎļþ
@@ -0,0 +1,3 @@
Flask==2.0.3
sentence-transformers==2.2.0
scikit-learn==0.24.2