缓存网页数据

package nexus.io.perplexica.services;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.locks.Lock;

import com.google.common.util.concurrent.Striped;
import nexus.io.db.activerecord.Db;
import nexus.io.db.activerecord.Row;
import nexus.io.perplexica.instance.PlaywrightBrowser;
import nexus.io.searxng.WebPageConteont;
import nexus.io.tio.utils.hutool.FilenameUtils;
import nexus.io.tio.utils.hutool.StrUtil;
import nexus.io.tio.utils.snowflake.SnowflakeIdUtils;
import nexus.io.tio.utils.thread.TioThreadUtils;
import com.microsoft.playwright.BrowserContext;
import com.microsoft.playwright.Page;

import lombok.extern.slf4j.Slf4j;

/**
 * PlaywrightService 负责爬取网页并将结果缓存至数据库。
 */
@Slf4j
public class PlaywrightService {

  // 数据表名称
  public static final String cache_table_name = "web_page_cache";

  /**
   * 批量异步抓取网页内容
   *
   * @param pages 包含url的WebPageConteont列表
   * @return 返回同一个列表，但其中的content属性已被填充或保持为空
   */
  public List<WebPageConteont> spiderAsync(List<WebPageConteont> pages) {
    List<Future<String>> futures = new ArrayList<>();

    // 为每个页面启动一个异步任务
    for (int i = 0; i < pages.size(); i++) {
      String link = pages.get(i).getUrl();

      Future<String> future = TioThreadUtils.submit(() -> {
        // 若后缀为pdf等其他非网页格式，直接跳过
        String suffix = FilenameUtils.getSuffix(link);
        if ("pdf".equalsIgnoreCase(suffix)) {
          log.info("skip: {}", link);
          return null;
        } else {
          // 爬取并返回文本内容
          return getPageContent(link);
        }
      });
      futures.add(i, future);
    }

    // 等待所有任务执行完成，并将结果填充回pages
    for (int i = 0; i < pages.size(); i++) {
      Future<String> future = futures.get(i);
      try {
        String result = future.get();
        if (StrUtil.isNotBlank(result)) {
          pages.get(i).setContent(result);
        }
      } catch (InterruptedException | ExecutionException e) {
        log.error("Error retrieving task result: {}", e.getMessage(), e);
      }
    }
    return pages;
  }

  // 使用Guava的Striped锁，设置64个锁段
  private static final Striped<Lock> stripedLocks = Striped.lock(1024);

  /**
   * 通过URL获取页面内容；若数据库有缓存则直接返回，否则利用Playwright实际爬取并写入缓存。
   *
   * @param link 要抓取的URL
   * @return 页面文本内容
   */
  private String getPageContent(String link) {
    // 先检查数据库缓存
    if (Db.exists(cache_table_name, "url", link)) {
      // 此处可以读取 text 或 html 等字段
      return Db.queryStr("SELECT text FROM " + cache_table_name + " WHERE url = ?", link);
    }

    // 使用Striped锁，为每个URL生成一把独立的锁，避免并发重复爬取
    Lock lock = stripedLocks.get(link);
    lock.lock();
    try {
      // 双重检查，防止其他线程已在获取锁后写入
      if (Db.exists(cache_table_name, "url", link)) {
        return Db.queryStr("SELECT text FROM " + cache_table_name + " WHERE url = ?", link);
      }

      // 使用 PlaywrightBrowser 获取context对象，执行真实的网页爬取
      BrowserContext context = PlaywrightBrowser.acquire();
      String html = null;
      String bodyText = null;
      try (Page page = context.newPage()) {
        page.navigate(link);
        // 获取文本内容
        bodyText = page.innerText("body");
        // 获取完整HTML
        html = page.content();
      } catch (Exception e) {
        log.error("Error getting content from {}: {}", link, e.getMessage(), e);
      } finally {
        // 归还context
        PlaywrightBrowser.release(context);
      }

      // 成功获取到的内容写入数据库缓存
      if (StrUtil.isNotBlank(bodyText)) {
        Row newRow = new Row();
        newRow.set("id", SnowflakeIdUtils.id())
              .set("url", link)
              .set("text", bodyText)
              .set("html", html)
              .set("markdown", "") // 如果有需要，实际业务中可在此生成并保存Markdown
              ;

        Db.save(cache_table_name, newRow);
      }

      return bodyText;
    } finally {
      lock.unlock();
    }
  }
}

package nexus.io.maxkb.task;

import org.quartz.JobExecutionContext;

import nexus.io.db.activerecord.Db;
import nexus.io.tio.utils.quartz.AbstractJobWithLog;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class SchduleTaskPerHour extends AbstractJobWithLog {
  @Override
  public void run(JobExecutionContext context) throws Exception {
    log.info("任务执行上下文: {}", context);
    // 清理1天前的数据
    Db.delete("DELETE FROM web_page_cache WHERE create_time < NOW() - INTERVAL '1 day'");
  }
}

package nexus.io.perplexica.services;

import java.util.Arrays;
import java.util.List;

import org.junit.Assert;
import org.junit.Test;

import nexus.io.searxng.WebPageConteont;

public class PlaywrightServiceTest {

  @Test
  public void testSpiderAsync() {
    PlaywrightService service = new PlaywrightService();

    // 构造测试数据
    WebPageConteont page1 = new WebPageConteont();
    page1.setUrl("https://www.example.com");

    WebPageConteont page2 = new WebPageConteont();
    page2.setUrl("https://www.wikipedia.org");

    List<WebPageConteont> pages = Arrays.asList(page1, page2);

    // 调用异步爬取
    List<WebPageConteont> resultPages = service.spiderAsync(pages);

    // 验证结果
    for (WebPageConteont webPageConteont : resultPages) {
      // 只要不是空，说明已成功抓取或者数据库已有缓存
      Assert.assertTrue("Content should not be null or empty",
          webPageConteont.getContent() != null && webPageConteont.getContent().length() > 0);

      System.out.println("URL: " + webPageConteont.getUrl());
      System.out.println("Content (partial): " + webPageConteont.getContent().substring(0, 100) + "...");
    }
  }
}