文档解析优化

import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Locale;

public Parsed parse(byte[] data, String filename) throws Exception {
  if (data == null || data.length == 0) {
    throw new IllegalArgumentException("文件为空");
  }
  if (data.length > 100 * 1024 * 1024) {
    throw new IllegalArgumentException("文件不能超过100MB");
  }
  String name = filename.toLowerCase(Locale.ROOT);
  Parsed result;
  if (new String(data, 0, Math.min(5, data.length), StandardCharsets.US_ASCII).equals("%PDF-")) {
    // 见下一节：PDF 逐页选择原生文本或 OCR
  } else if (data.length > 4 && data[0] == 'P' && data[1] == 'K' && name.endsWith(".docx")) {
    String extracted = docx(data);
    if (extracted.replaceAll("\\s", "").length() < 40) {
      String images = docxImages(data);
      result = new Parsed(images.isBlank() ? "docx-structure" : "docx-ocr", extracted + images, 0);
    } else {
      result = new Parsed("docx-structure", extracted, 0);
    }
  } else if (name.endsWith(".xlsx") || name.endsWith(".xls")) {
    result = new Parsed("spreadsheet-structure", spreadsheet(data), 0);
  } else if (javax.imageio.ImageIO.read(new ByteArrayInputStream(data)) != null) {
    result = new Parsed("image-ocr", ocr(data, filename), 1);
  } else {
    throw new IllegalArgumentException("不支持的文件格式，请转换为PDF、DOCX、TXT或图片后上传");
  }
  if (result.text().isBlank()) {
    throw new IllegalArgumentException("未提取到内容，请检查文档或使用扫描PDF");
  }
  return result;
}

import java.io.ByteArrayOutputStream;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import nexus.io.tio.utils.crypto.Md5Utils;

try (PDDocument pdf = PDDocument.load(data)) {
  if (pdf.isEncrypted()) {
    throw new IllegalArgumentException("请先解密PDF文件");
  }
  PDFTextStripper stripper = new PDFTextStripper();
  stripper.setSortByPosition(true);
  String sourceHash = Md5Utils.md5Hex(data);
  StringBuilder text = new StringBuilder();
  int nativePages = 0;
  for (int i = 1; i <= pdf.getNumberOfPages(); i++) {
    stripper.setStartPage(i);
    stripper.setEndPage(i);
    String page = stripper.getText(pdf);
    if (page.replaceAll("\\s", "").length() >= 40 && !page.contains("\uFFFD")) {
      text.append("\n\n> Page ").append(i).append("\n\n").append(page);
      nativePages++;
    } else {
      try (PDDocument single = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        single.importPage(pdf.getPage(i - 1));
        single.save(out);
        text.append("\n\n> Page ").append(i).append("\n\n")
            .append(ocrPdfPage(sourceHash, i, out.toByteArray()));
      }
    }
  }
  result = new Parsed(
      nativePages == pdf.getNumberOfPages() ? "pdf-text"
          : nativePages == 0 ? "pdf-ocr" : "pdf-mixed",
      text.toString(), pdf.getNumberOfPages());
}

import java.io.IOException;
import java.util.Arrays;
import java.util.Locale;
import java.util.concurrent.TimeUnit;

protected String recognize(byte[] data, String filename) throws Exception {
  GiteeClient client = new GiteeClient();
  GiteeDocumentParseRequest request = new GiteeDocumentParseRequest();
  request.setModel("PaddleOCR-VL-1.5");
  request.setPrompt(GiteePromptConst.pdf_to_markdown_prompt);
  request.setInclude_image(false);
  request.setInclude_image_base64(false);
  GiteeTaskResponse task = client.parseDocument(data, filename, request);

  long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(8);
  while (!Arrays.asList("success", "succeeded", "completed")
      .contains(String.valueOf(task.getStatus()).toLowerCase(Locale.ROOT))) {
    if (task.getStatus() == null || task.getTask_id() == null) {
      throw new IOException("OCR服务未返回有效任务状态");
    }
    if (Arrays.asList("failed", "failure", "cancelled", "canceled")
        .contains(task.getStatus().toLowerCase(Locale.ROOT))) {
      throw new IOException("OCR任务失败: " + task.getTask_id());
    }
    if (System.nanoTime() > deadline) {
      throw new IOException("OCR任务超时: " + task.getTask_id());
    }
    Thread.sleep(2000);
    task = client.getTask(task.getTask_id());
  }

  String markdown = textOnly(GiteeSimpleMarkdownUtils.toMarkdown(task.getOutput(), null));
  if (markdown == null || markdown.isBlank()) {
    throw new IOException("OCR未返回正文");
  }
  return markdown;
}

import okhttp3.MultipartBody;
import okhttp3.Request;
import okhttp3.RequestBody;

private GiteeTaskResponse parseDocument(RequestBody fileBody, String filename,
    GiteeDocumentParseRequest requestEntity) {
  String url = baseUrl + "/v1/async/documents/parse";
  MultipartBody.Builder bodyBuilder = new MultipartBody.Builder().setType(MultipartBody.FORM)
      .addFormDataPart("file", filename, fileBody)
      .addFormDataPart("model", requestEntity.getModel());
  if (requestEntity.getInclude_image() != null) {
    bodyBuilder.addFormDataPart("include_image", requestEntity.getInclude_image().toString());
  }
  if (requestEntity.getInclude_image_base64() != null) {
    bodyBuilder.addFormDataPart("include_image_base64", requestEntity.getInclude_image_base64().toString());
  }
  if (requestEntity.getPrompt() != null && !requestEntity.getPrompt().trim().isEmpty()) {
    bodyBuilder.addFormDataPart("prompt", requestEntity.getPrompt());
  }
  Request request = new Request.Builder().url(url)
      .addHeader("Authorization", "Bearer " + apiKey)
      .addHeader("X-Failover-Enabled", "true")
      .post(bodyBuilder.build()).build();
  return execute(request);
}

// 只使用 java.lang.String，无需额外 import
private String textOnly(String markdown) {
  if (markdown == null) {
    return "";
  }
  return markdown.replaceAll("(?is)<img\\b[^>]*>", "")
      .replaceAll("(?is)<div[^>]*>\\s*</div>", "")
      .replaceAll("!\\[[^\\]]*\\]\\([^)]*\\)", "");
}

import nexus.io.db.activerecord.Db;

private String ocrWithCache(byte[] data, String filename, String hash) throws Exception {
  String cached = readOcrCache(hash);
  if (cached != null && !cached.isBlank()) {
    return textOnly(cached);
  }
  String markdown = recognize(data, filename);
  writeOcrCache(hash, markdown);
  return markdown;
}

protected String readOcrCache(String hash) {
  return Db.queryStr("select content from moss_kb_document_markdown_cache where id=?", hash);
}

protected void writeOcrCache(String hash, String markdown) {
  Db.update("insert into moss_kb_document_markdown_cache(id,content) values(?,?) "
      + "on conflict(id) do update set content=excluded.content", hash, markdown);
}

// 复用上一节已导入的 Md5Utils
protected String ocr(byte[] data, String filename) throws Exception {
  return ocrWithCache(data, filename, Md5Utils.md5Hex(data) + ":paddle15");
}

protected String ocrPdfPage(String sourceHash, int page, byte[] pageData) throws Exception {
  // Serializing a PDF page can generate a fresh internal document ID. Cache by
  // original file content and page number instead of the transient PDF bytes.
  return ocrWithCache(pageData, "page.pdf", sourceHash + ":page:" + page + ":paddle15");
}

import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilderFactory;

DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");

import java.util.ArrayList;
import java.util.List;
import nexus.io.annotation.RequestPath;
import nexus.io.jfinal.aop.Aop;
import nexus.io.mosskb.service.SystemFileService;
import nexus.io.mosskb.service.kb.MossKbDocumentSplitService;
import nexus.io.model.result.ResultVo;
import nexus.io.model.upload.UploadFile;
import nexus.io.model.upload.UploadResult;
import nexus.io.tio.http.common.HttpRequest;

@RequestPath("/api/dataset/document")
public class ApiDatasetDocumentController {

  public ResultVo split(HttpRequest request) throws Exception {
    Object[] files = request.getParams().get("file");
    if (files == null || files.length == 0) {
      return ResultVo.fail("请求体中未找到文件");
    }
    List<Object> results = new ArrayList<>();
    for (Object item : files) {
      if (!(item instanceof UploadFile)) {
        return ResultVo.fail("文件格式不正确");
      }
      UploadFile file = (UploadFile) item;
      // 先落盘拿到文件 id，再解析正文
      UploadResult uploaded = Aop.get(SystemFileService.class).upload(file, "default", "default");
      ResultVo result = Aop.get(MossKbDocumentSplitService.class).split(file.getData(), uploaded);
      results.addAll((List<?>) result.getData());
    }
    return ResultVo.ok(results);
  }
}

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import com.jfinal.kit.Kv;
import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.DocumentSplitter;
import dev.langchain4j.data.document.splitter.DocumentSplitters;
import dev.langchain4j.data.segment.TextSegment;
import nexus.io.model.result.ResultVo;
import nexus.io.model.upload.UploadResult;
import nexus.io.openai.token.OpenAiTokenizer;

public ResultVo split(byte[] data, UploadResult vo)
    throws IOException, InterruptedException, ExecutionException {
  String filename = vo.getName();
  DocumentParsingService.Parsed parsed;
  try {
    parsed = new DocumentParsingService().parse(data, filename);
  } catch (Exception e) {
    throw new IOException(e.getMessage(), e);
  }
  String markdown = parsed.text();
  List<TextSegment> segments = split(markdown);
  Kv fileSplitResult = Kv.by("name", filename).set("id", vo.getId());
  List<Kv> contents = new ArrayList<>();
  for (TextSegment textSegment : segments) {
    contents.add(Kv.by("title", "").set("content", textSegment.text()));
  }
  fileSplitResult.set("content", contents)
      .set("parse_strategy", parsed.strategy())
      .set("page_count", parsed.pages());
  return ResultVo.ok(List.of(fileSplitResult));
}

public List<TextSegment> split(String markdown) {
  Document document = new Document(markdown);
  // 2000 token 分块，400 token 重叠
  DocumentSplitter splitter = DocumentSplitters.recursive(2000, 400, new OpenAiTokenizer());
  return splitter.split(document);
}

import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.font.PDType1Font;

public class DocumentParsingServiceTest {
  /** 覆写 OCR 相关方法，测试期间不发起远程调用 */
  private static class Parser extends DocumentParsingService {
    int ocrCalls;
    @Override
    protected String ocr(byte[] data, String filename) {
      ocrCalls++;
      return "Scanned content " + ocrCalls;
    }
    @Override
    protected String ocrPdfPage(String sourceHash, int page, byte[] data) {
      return ocr(data, "page.pdf");
    }
  }
}

import java.io.ByteArrayOutputStream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

@Test
public void pdfUsesTextAndOcrPerPageInOriginalOrder() throws Exception {
  try (PDDocument pdf = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) {
    PDPage first = new PDPage();
    pdf.addPage(first);
    try (PDPageContentStream canvas = new PDPageContentStream(pdf, first)) {
      canvas.beginText();
      canvas.setFont(PDType1Font.HELVETICA, 12);
      canvas.newLineAtOffset(40, 700);
      canvas.showText("Native administrative review document with sufficient readable text for extraction.");
      canvas.endText();
    }
    pdf.addPage(new PDPage());
    pdf.save(out);

    Parser parser = new Parser();
    DocumentParsingService.Parsed result = parser.parse(out.toByteArray(), "misleading.txt");
    assertEquals("pdf-mixed", result.strategy());
    assertEquals(2, result.pages());
    assertEquals(1, parser.ocrCalls);
    assertTrue(result.text().indexOf("Native administrative") < result.text().indexOf("Scanned content"));
  }
}