增强检索（RAG）

import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.DocumentSplitter;
import dev.langchain4j.data.document.loader.FileSystemDocumentLoader;
import dev.langchain4j.data.document.splitter.DocumentSplitters;
import dev.langchain4j.model.openai.OpenAiTokenizer;

import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;

public class LangChain4JLoadDocumentSplitTest {

    public static void main(String[] args) throws Exception {
        // 下载文件
        String fileUrl = "https://raw.githubusercontent.com/litongjava/tio-boot-docs/main/docs/zh/01_introduction/02.md";
        Path tempFilePath = Paths.get("downloaded.md");

        try (InputStream in = new URL(fileUrl).openStream()) {
            Files.copy(in, tempFilePath);
        }

        // 加载文档
        Document document = FileSystemDocumentLoader.loadDocument(tempFilePath);

        // 切分文档
        DocumentSplitter splitter = DocumentSplitters.recursive(150, 10, new OpenAiTokenizer());
        List<TextSegment> segments = splitter.split(document);

        // 输出切分后的文档段落数量
        System.out.println("切分后的段落数量: " + segments.size());

        // 删除临时文件
        Files.delete(tempFilePath);
    }
}

DocumentSplitter splitter = DocumentSplitters.recursive(150, 10, new OpenAiTokenizer());
List<TextSegment> segments = splitter.split(document);

import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.model.openai.OpenAiEmbeddingModel;
import dev.langchain4j.model.openai.OpenAiEmbeddingModelName;

public class LangChain4JEmbeddingTest {

    public static void main(String[] args) {
        // 配置 OpenAI API 密钥
        String openApiKey = "YOUR_OPENAI_API_KEY";

        // 创建嵌入模型
        OpenAiEmbeddingModel embeddingModel = new OpenAiEmbeddingModel.OpenAiEmbeddingModelBuilder()
                .apiKey(openApiKey)
                .modelName(OpenAiEmbeddingModelName.TEXT_EMBEDDING_3_LARGE)
                .build();

        // 需要嵌入的文本
        String text = "两只眼睛";

        // 生成嵌入
        Embedding embedding = embeddingModel.embed(text).content();

        // 输出嵌入结果
        System.out.println("文本的嵌入结果: " + embedding.vectorAsList());
        System.out.println("向量维度: " + embedding.dimension());
    }
}

package nexus.io.test;

import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.junit.Test;

import nexus.io.openai.chat.ChatResponseVo;
import nexus.io.openai.client.OpenAiClient;
import nexus.io.tio.utils.environment.EnvUtils;
import nexus.io.tio.utils.json.JsonUtils;

import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.DocumentSplitter;
import dev.langchain4j.data.document.loader.FileSystemDocumentLoader;
import dev.langchain4j.data.document.splitter.DocumentSplitters;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.message.UserMessage;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.input.Prompt;
import dev.langchain4j.model.input.PromptTemplate;
import dev.langchain4j.model.openai.OpenAiTokenizer;
import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest.EmbeddingSearchRequestBuilder;
import dev.langchain4j.store.embedding.EmbeddingSearchResult;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.chroma.ChromaEmbeddingStore;
import dev.langchain4j.store.embedding.filter.Filter;
import dev.langchain4j.store.embedding.filter.comparison.IsEqualTo;

public class Langchain4jChromDbTest {

  @Test
  public void test() throws NoSuchAlgorithmException {
    // 加载环境变量
    EnvUtils.load();

    // 下载待处理的文件
    String fileUrl = "https://raw.githubusercontent.com/litongjava/tio-boot-docs/main/docs/zh/01_introduction/02.md";
    Path tempFilePath = Paths.get("downloaded.md");

    // 下载并保存文件
    try {
      Files.deleteIfExists(tempFilePath); // 若文件已存在，先删除
    } catch (IOException e) {
      e.printStackTrace();
    }
    try (InputStream in = new URL(fileUrl).openStream()) {
      Files.copy(in, tempFilePath);
    } catch (MalformedURLException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }

    // 加载文档
    Document document = FileSystemDocumentLoader.loadDocument(tempFilePath);

    // 使用 OpenAI Tokenizer 切分文档
    DocumentSplitter splitter = DocumentSplitters.recursive(150, 10, new OpenAiTokenizer());
    List<TextSegment> segments = splitter.split(document);

    System.out.println("拆分后的片段数量:" + segments.size());

    // 设置 Chroma 向量数据库 URL 和集合名
    String CHROMA_URL = EnvUtils.get("CHROMA_URL");
    String CHROMA_DB_DEFAULT_COLLECTION_NAME = "tio-boot";

    // 创建向量存储器并将向量与文本段存储
    EmbeddingStore<TextSegment> embeddingStore = ChromaEmbeddingStore.builder().baseUrl(CHROMA_URL).collectionName(CHROMA_DB_DEFAULT_COLLECTION_NAME).build();

    // MD5 哈希生成器
    MessageDigest md = MessageDigest.getInstance("MD5");
    EmbeddingSearchRequestBuilder builder = EmbeddingSearchRequest.builder();
    for (TextSegment segment : segments) {
      // 为文本段生成 MD5 哈希
      String segmentText = segment.text();
      String segmentMd5 = new BigInteger(1, md.digest(segmentText.getBytes(StandardCharsets.UTF_8))).toString(16);

      // 在 Chroma 中通过 MD5 进行搜索，而不是生成新的嵌入
      Map<String, String> metadataFilter = new HashMap<>();
      metadataFilter.put("md5", segmentMd5);
      // 创建一个 Filter 来搜索 MD5 匹配的条目
      Filter md5Filter = new IsEqualTo("md5", segmentMd5);

      float[] dummyVector = new float[3072]; // 创建一个 3072 维的空向量，默认值为 0.0f
      Embedding dummyEmbedding = new Embedding(dummyVector); // 用这个向量创建一个虚拟的 Embedding

      EmbeddingSearchRequest searchRequest = builder.filter(md5Filter) // 通过元数据 (MD5) 进行搜索
          .queryEmbedding(dummyEmbedding) // 传入虚拟 Embedding
          .maxResults(1) // 搜索最多返回1个结果
          .build();
      EmbeddingSearchResult<TextSegment> searchResult = embeddingStore.search(searchRequest);

      // 如果存在相同的 MD5，跳过添加
      boolean isDuplicate = !searchResult.matches().isEmpty();
      if (isDuplicate) {
        System.out.println("跳过重复片段: " + segmentMd5);
        continue;
      }

      // 将文本段嵌入并存储，同时存储 MD5 作为元数据
      float[] vector = OpenAiClient.embeddingArrayByLargeModel(segmentText);
      Embedding embedding = new Embedding(vector); // 使用 MD5 作为元数据
      segment.metadata().put("md5", segmentMd5);
      embeddingStore.add(embedding, segment);
    }

    // 用户查询问题
    String question = "如何启动tio-boot";
    // 使用 API 生成用户查询的嵌入向量
    Embedding queryEmbedding = new Embedding(OpenAiClient.embeddingArrayByLargeModel(question));

    // 在 Chroma 中检索最相关的文档段
    EmbeddingSearchRequest embeddingSearchRequest = builder.queryEmbedding(queryEmbedding).maxResults(1).build();
    EmbeddingSearchResult<TextSegment> embeddedEmbeddingSearchResult = embeddingStore.search(embeddingSearchRequest);

    // 获取最相关的文档段
    List<EmbeddingMatch<TextSegment>> embeddingMatches = embeddedEmbeddingSearchResult.matches();
    List<String> context = new ArrayList<>();
    for (EmbeddingMatch<TextSegment> match : embeddingMatches) {
      TextSegment textSegment = match.embedded();
      context.add(textSegment.text());
    }

    // 构造 Prompt 模板
    PromptTemplate promptTemplate = PromptTemplate.from("基于如下信息进行回答:\n{{context}}\n提问:\n{{question}}");
    Map<String, Object> variables = new HashMap<>();
    variables.put("context", JsonUtils.toJson(context));
    variables.put("question", question);
    Prompt prompt = promptTemplate.apply(variables);

    // 构建用户消息并向 OpenAI 提交请求
    UserMessage userMessage = prompt.toUserMessage();
    String singleText = userMessage.singleText();
    // 输出生成的回答
    ChatResponseVo chatResponse = OpenAiClient.chatCompletionsWithRole("user", singleText);
    // 输出信息
    System.out.println(singleText);
    System.out.println(JsonUtils.toJson(chatResponse));
  }
}