DOM 原理：Java 模型与跨 Frame 索引

package nexus.io.ai.browser.dom.model;

public abstract class DOMBaseNode {
  protected boolean isVisible;
  protected DOMElementNode parent;

  public DOMBaseNode(boolean isVisible) {
    this.isVisible = isVisible;
  }

  public boolean isVisible() {
    return isVisible;
  }

  public DOMElementNode getParent() {
    return parent;
  }

  public void setParent(DOMElementNode parent) {
    this.parent = parent;
  }
}

package nexus.io.ai.browser.dom.model;

public class DOMTextNode extends DOMBaseNode {
  private final String text;
  private final String type = "TEXT_NODE";

  public DOMTextNode(String text, boolean isVisible) {
    super(isVisible);
    this.text = text;
  }

  public String getText() {
    return text;
  }

  public boolean hasParentWithHighlightIndex() {
    DOMElementNode cur = parent;
    while (cur != null) {
      if (cur.getHighlightIndex() != null)
        return true;
      cur = cur.getParent();
    }
    return false;
  }

  public boolean isParentInViewport() {
    return parent != null && parent.isInViewport();
  }

  public boolean isParentTopElement() {
    return parent != null && parent.isTopElement();
  }
}

package nexus.io.ai.browser.dom.model;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import nexus.io.tio.utils.collect.Lists;

public class DOMElementNode extends DOMBaseNode {
  private final String tagName;
  private final String xpath;
  private final Map<String, String> attributes;
  private final List<DOMBaseNode> children = new ArrayList<>();

  private final boolean isInteractive;
  private final boolean isTopElement;
  private final boolean isInViewport;
  private final boolean shadowRoot;
  private final Integer highlightIndex;
  private final Boolean isNew;

  // （viewportInfo / pageCoordinates 按需补）
  public static final List<String> DEFAULT_INCLUDE_ATTRIBUTES = Lists.of("title", "type", "checked", "name", "role", "value",
      //
      "placeholder", "data-date-format", "alt", "aria-label", "aria-expanded", "data-state", "aria-checked",
      "readonly", "disabled", "editable", "selected", "selected-text");

  public DOMElementNode(String tagName, String xpath, Map<String, String> attributes, boolean isVisible, boolean
  //
  isInteractive, boolean isTopElement, boolean isInViewport, boolean shadowRoot, Integer highlightIndex, Boolean isNew) {
    super(isVisible);
    this.tagName = tagName;
    this.xpath = xpath;
    this.attributes = attributes;
    this.isInteractive = isInteractive;
    this.isTopElement = isTopElement;
    this.isInViewport = isInViewport;
    this.shadowRoot = shadowRoot;
    this.highlightIndex = highlightIndex;
    this.isNew = isNew;
  }

  public String getTagName() {
    return tagName;
  }

  /** buildDomTree 生成的相对 xpath,如 html/body/div[2]/form/input[1] */
  public String getXpath() {
    return xpath;
  }

  public Map<String, String> getAttributes() {
    return attributes;
  }

  public Integer getHighlightIndex() {
    return highlightIndex;
  }

  public boolean isInteractive() {
    return isInteractive;
  }

  public boolean isTopElement() {
    return isTopElement;
  }

  public boolean isInViewport() {
    return isInViewport;
  }

  public Boolean isNew() {
    return isNew;
  }

  public List<DOMBaseNode> getChildren() {
    return children;
  }

  public void addChild(DOMBaseNode child) {
    children.add(child);
    if (child instanceof DOMElementNode) {
      ((DOMElementNode) child).setParent(this);
    } else if (child instanceof DOMTextNode) {
      ((DOMTextNode) child).setParent(this);
    }
  }

  /**
   * 收集从 this 节点开始，直到下一个可点击元素前的所有文本
   */
  public String getAllTextTillNextClickableElement(int maxDepth) {
    StringBuilder sb = new StringBuilder();
    collectText(this, 0, maxDepth, sb);
    return sb.toString().trim();
  }

  private void collectText(DOMBaseNode node, int depth, int maxDepth, StringBuilder sb) {
    if (maxDepth != -1 && depth > maxDepth) {
      return;
    }
    if (node instanceof DOMElementNode) {
      DOMElementNode el = (DOMElementNode) node;
      if (el != this && el.getHighlightIndex() != null) {
        return;
      }
      for (DOMBaseNode c : el.getChildren()) {
        collectText(c, depth + 1, maxDepth, sb);
      }
    } else if (node instanceof DOMTextNode) {
      //sb.append(((DOMTextNode) node).getText()).append("\n");
      String text = ((DOMTextNode) node).getText();
      sb.append(text.trim()).append(" ");
    }
  }

  /**
   * 核心：把可点击元素按照 [index]<tag 属性拼接>文本/> 的格式输出
   */
  public String clickableElementsToString(List<String> includeAttributes) {
    if (includeAttributes == null) {
      includeAttributes = DEFAULT_INCLUDE_ATTRIBUTES;
    }

    List<String> out = new ArrayList<>();
    processNode(this, "", includeAttributes, out);
    return out.stream().collect(Collectors.joining("\n"));
  }

  private void processNode(DOMBaseNode node, String indent, List<String> includeAttributes, List<String> out) {
    if (node instanceof DOMElementNode) {
      DOMElementNode el = (DOMElementNode) node;
      if (el.getHighlightIndex() != null || (el.isVisible() && el.isTopElement &&
          List.of("input", "textarea", "select", "option", "button").contains(el.tagName))) {
        String text = el.getAllTextTillNextClickableElement(-1);
        String attrStr = buildAttributesHtml(el.attributes, includeAttributes, text);
        String indicator = el.highlightIndex == null ? "" : Boolean.TRUE.equals(el.isNew()) ? "*[" + el.getHighlightIndex() + "]" : "[" + el.getHighlightIndex() + "]";
        String line = indent + indicator + "<" + el.tagName + (attrStr.isEmpty() ? "" : " " + attrStr)
            + (text.isEmpty() ? (attrStr.isEmpty() ? " " : "") + "/>" : (attrStr.isEmpty() ? " " : "") + ">" + text + "/>");
        out.add(line);
      }
      // 继续递归
      boolean meaningful = el.highlightIndex != null || List.of("form", "fieldset", "table", "thead", "tbody",
          "tr", "td", "th", "ul", "ol", "li", "nav", "section", "dialog").contains(el.tagName);
      String childIndent = meaningful && indent.length() < 6 ? indent + "\t" : indent;
      for (DOMBaseNode c : el.getChildren()) {
        processNode(c, childIndent, includeAttributes, out);
      }
    } else if (node instanceof DOMTextNode) {
      DOMTextNode txt = (DOMTextNode) node;
      if (txt.hasParentWithHighlightIndex())
        return;
      DOMElementNode p = txt.getParent();
      if (p != null && p.isVisible() && p.isTopElement()) {
        out.add(indent + txt.getText());
      }
    }
  }

  private String buildAttributesHtml(Map<String, String> attrs, List<String> includeAttributes, String text) {
    Map<String, String> keep = new LinkedHashMap<>();
    for (String k : includeAttributes) {
      if (attrs.containsKey(k) && attrs.get(k) != null) {
        String value = attrs.get(k);
        if ("value".equals(k) && "password".equalsIgnoreCase(attrs.get("type"))) {
          value = "[redacted]";
        }
        if (List.of("title", "placeholder", "alt", "aria-label", "selected-text").contains(k)) {
          value = cap(value, 160);
        }
        keep.put(k, value);
      }
    }
    // “去重”“剔除与文本相同的属性” 等…可按需补
    return keep.entrySet().stream().map(e -> e.getKey() + "='" + escapeAttribute(e.getValue()) + "'")
        .collect(Collectors.joining(" "));
  }

  private static String escapeAttribute(String value) {
    return value.replace("&", "&amp;").replace("'", "&#39;").replace("<", "&lt;")
        .replace(">", "&gt;").replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t");
  }

  private static String cap(String s, int max) {
    return s.length() > max ? s.substring(0, max) + "..." : s;
  }
}

package nexus.io.ai.browser.dom.model;

import java.util.Collections;
import java.util.List;
import java.util.Map;

public class DOMState {
  private final DOMElementNode elementTree;
  private final Map<Integer, DOMElementNode> selectorMap;
  public final int pixelsAbove;
  public final int pixelsBelow;
  public final int viewportHeight;
  public final int pageHeight;

  /**
   * 全局索引 -&gt; frame 序号(0 是主 frame)
   *
   * <p>
   * 单 frame 快照时是空表(所有索引都在主 frame);带 frame 的快照(见 {@code get_browser_state} 的
   * {@code includeFrames})里每个索引都有一条。按索引命令靠它把动作路由到正确的 frame —— 索引里已经带了
   * frame 信息,所以调用方不必再传 frame 参数。
   */
  private final Map<Integer, Integer> indexToFrame;

  /**
   * 这次快照涉及的 frame,下标即 frame 序号(0 是主 frame)
   *
   * <p>单 frame 快照时是空表,调用方按「所有索引都在主 frame」处理。
   */
  private final List<FrameSnapshot> frames;

  public DOMState(DOMElementNode tree, Map<Integer, DOMElementNode> sel,
      //
      int pixelsAbove, int pixelsBelow, int viewportHeight, int pageHeight) {
    this(tree, sel, pixelsAbove, pixelsBelow, viewportHeight, pageHeight, Collections.emptyMap(),
        Collections.emptyList());
  }

  public DOMState(DOMElementNode tree, Map<Integer, DOMElementNode> sel, int pixelsAbove, int pixelsBelow,
      int viewportHeight, int pageHeight, Map<Integer, Integer> indexToFrame, List<FrameSnapshot> frames) {
    this.elementTree = tree;
    this.selectorMap = sel;
    this.pixelsAbove = pixelsAbove;
    this.pixelsBelow = pixelsBelow;
    this.viewportHeight = viewportHeight;
    this.pageHeight = pageHeight;
    this.indexToFrame = indexToFrame == null ? Collections.emptyMap() : indexToFrame;
    this.frames = frames == null ? Collections.emptyList() : frames;
  }

  public DOMElementNode getElementTree() {
    return elementTree;
  }

  public Map<Integer, DOMElementNode> getSelectorMap() {
    return selectorMap;
  }

  public int getPixelsAbove() {
    return pixelsAbove;
  }

  public int getPixelsBelow() {
    return pixelsBelow;
  }

  public int getViewportHeight() {
    return viewportHeight;
  }

  public int getPageHeight() {
    return pageHeight;
  }

  public Map<Integer, Integer> getIndexToFrame() {
    return indexToFrame;
  }

  public List<FrameSnapshot> getFrames() {
    return frames;
  }

  /** 这次快照是不是带 frame 的(带了才有 {@link FrameSnapshot} 与索引路由) */
  public boolean hasFrames() {
    return !frames.isEmpty();
  }

  /** 取某个索引所属的 frame;没有 frame 信息时返回 null(表示主 frame) */
  public FrameSnapshot frameOf(int index) {
    Integer ordinal = indexToFrame.get(index);
    if (ordinal == null || ordinal < 0 || ordinal >= frames.size()) {
      return null;
    }
    return frames.get(ordinal);
  }
}

package nexus.io.ai.browser.dom.model;

import com.microsoft.playwright.Frame;

/**
 * 一次快照里的一个 frame
 *
 * <p>
 * <b>为什么需要它</b>:企业微信后台把「邮件」应用套在 {@code exmail.qq.com} 的**跨域 iframe** 里,微盘 /
 * 文档 / 会议同理。这种形态下顶层 {@code document} 里一个元素都看不见,而 Playwright 本身是能跨 frame
 * 的(它走浏览器协议取内容,不依赖往页面里注入 JS)—— 缺的只是把 {@code page.frames()} 暴露出来。这个类
 * 就是那份暴露:每个 frame 一行元信息 + 它自己的可交互元素树 + 它占用的**全局索引区间**。
 *
 * <p>
 * <b>索引怎么保证全局唯一</b>:每个 frame 单独求值时索引会各自从 0 开始,必然撞车。所以按 frame 顺序依次
 * 求值,把「前面几个 frame 已经用掉多少」当起点传给下一个 frame(见 {@code DomService} 的
 * {@code highlightIndexStart}),于是 {@code firstIndex..lastIndex} 就是该 frame 在快照里的索引区间,
 * 而按索引命令(click_element_by_index / input_text …)靠 {@code indexToFrame} 自动路由到正确的 frame,
 * 调用方**不需要**再传 frame 参数。
 */
public class FrameSnapshot {

  /**
   * frame 序号:0 固定是主 frame,其余按 Playwright 的 frame 树顺序深度优先编号
   *
   * <p>与命令里的 {@code frame} 参数、{@code list_frames} 里的 {@code index} 都是同一个编号 ——
   * 从快照文本里读到 {@code --- frame[2] … ---} 就直接写 {@code {"frame":2}}。
   */
  public final int index;
  /** 该 frame 的 URL(跨域 frame 也能读到,这是识别它的主要依据) */
  public final String url;
  /** {@code <iframe name="...">} 的 name(没有则为空串) */
  public final String name;
  /** 是不是主 frame */
  public final boolean main;
  /** 父 frame 的序号;-1 表示没有父(主 frame) */
  public final int parentIndex;
  /** 嵌套深度,主 frame 是 0 */
  public final int depth;
  /** 这个 frame 占用的全局索引下界;没有可交互元素时与 lastIndex 一起为 -1 */
  public final int firstIndex;
  /** 这个 frame 占用的全局索引上界(含);没有可交互元素时为 -1 */
  public final int lastIndex;
  /** 可交互元素个数 */
  public final int elementCount;
  /** 这个 frame 自己的可交互结构化文本(与 data.text 里那一段完全一致) */
  public final String text;
  /** 该 frame 的元素树根(用于按 xpath 找回元素) */
  public final DOMElementNode tree;
  public final int pixelsAbove;
  public final int pixelsBelow;
  public final int viewportHeight;
  public final int pageHeight;
  /**
   * 快照时用的 frame 句柄
   *
   * <p>
   * frame 是运行期对象,页面一导航(或 iframe 被重建)它就 detach 了。所以动作执行时**不能**无条件拿它去定位:
   * 先按 {@link #url} 在当前 {@code page.frames()} 里重新认一次,认不出来才退回这个句柄(见
   * {@code PlaywrightService.frameForIndex})。
   */
  public final Frame frame;
  /** 这个 frame 读取失败的原因(跨域被拒 / frame 正在销毁 / 还没有文档);成功时为 null */
  public final String readFailure;
  /**
   * 这个 frame 是不是**没被纳入**本次快照
   *
   * <p>默认快照(不带 {@code includeFrames})只纳入同源 frame —— 与以前「顺着同源 iframe 递归」的行为一致;
   * 跨域 frame 被跳过的原因记在 {@link #skipReason}。要纳入全部 frame 就传 {@code includeFrames:true}。
   */
  public final boolean skipped;
  /** 被跳过的原因 */
  public final String skipReason;

  public FrameSnapshot(int index, String url, String name, boolean main, int parentIndex, int depth,
      int firstIndex, int lastIndex, int elementCount, String text, DOMElementNode tree, int pixelsAbove,
      int pixelsBelow, int viewportHeight, int pageHeight, Frame frame, String readFailure) {
    this(index, url, name, main, parentIndex, depth, firstIndex, lastIndex, elementCount, text, tree, pixelsAbove,
        pixelsBelow, viewportHeight, pageHeight, frame, readFailure, false, null);
  }

  public FrameSnapshot(int index, String url, String name, boolean main, int parentIndex, int depth,
      int firstIndex, int lastIndex, int elementCount, String text, DOMElementNode tree, int pixelsAbove,
      int pixelsBelow, int viewportHeight, int pageHeight, Frame frame, String readFailure, boolean skipped,
      String skipReason) {
    this.index = index;
    this.url = url;
    this.name = name;
    this.main = main;
    this.parentIndex = parentIndex;
    this.depth = depth;
    this.firstIndex = firstIndex;
    this.lastIndex = lastIndex;
    this.elementCount = elementCount;
    this.text = text;
    this.tree = tree;
    this.pixelsAbove = pixelsAbove;
    this.pixelsBelow = pixelsBelow;
    this.viewportHeight = viewportHeight;
    this.pageHeight = pageHeight;
    this.frame = frame;
    this.readFailure = readFailure;
    this.skipped = skipped;
    this.skipReason = skipReason;
  }

  /** 这个 frame 有没有可交互元素 */
  public boolean hasElements() {
    return elementCount > 0;
  }

  /** 给模型读的一行元信息(快照文本里的 frame 分隔行用它) */
  public String header() {
    StringBuilder sb = new StringBuilder("--- frame[").append(index).append(']');
    if (main) {
      sb.append(" (main)");
    }
    sb.append(' ').append(url == null ? "" : url);
    if (name != null && !name.isEmpty()) {
      sb.append(" name=\"").append(name).append('"');
    }
    sb.append(" elements=").append(elementCount);
    if (hasElements()) {
      sb.append(" index=").append(firstIndex).append("..").append(lastIndex);
    }
    if (readFailure != null) {
      sb.append(" [读取失败: ").append(readFailure).append(']');
    }
    sb.append(" ---");
    return sb.toString();
  }
}