源码教程：原生对话框、DOM 弹窗与人机协作

    put("get_dialog", (svc, id, a) -> svc.getDialog(id, a.getBoolean("consume")));

  public RespBodyVo getDialog(Long browserId, Boolean consume) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv dialog = inst.lastDialog;
    if (consume != null && consume) {
      inst.lastDialog = null;
    }
    if (dialog == null) {
      return RespBodyVo.ok(Kv.by("dialog", null));
    }
    return RespBodyVo.ok(Kv.by("dialog", dialog));
  }

    put("clear_dialog", (svc, id, a) -> svc.clearDialog(id));

  public RespBodyVo clearDialog(Long browserId) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    boolean cleared = inst.lastDialog != null;
    inst.lastDialog = null;
    return RespBodyVo.ok(Kv.by("cleared", cleared));
  }

    put("get_js_dialog", (svc, id, a) -> svc.getDialog(id, a.getBoolean("consume")));

  public RespBodyVo getDialog(Long browserId, Boolean consume) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv dialog = inst.lastDialog;
    if (consume != null && consume) {
      inst.lastDialog = null;
    }
    if (dialog == null) {
      return RespBodyVo.ok(Kv.by("dialog", null));
    }
    return RespBodyVo.ok(Kv.by("dialog", dialog));
  }

    put("clear_js_dialog", (svc, id, a) -> svc.clearDialog(id));

  public RespBodyVo clearDialog(Long browserId) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    boolean cleared = inst.lastDialog != null;
    inst.lastDialog = null;
    return RespBodyVo.ok(Kv.by("cleared", cleared));
  }

    put("set_dialog_behavior", (svc, id, a) -> svc.setDialogBehavior(id, optBool(a, "dismiss")));

  public RespBodyVo setDialogBehavior(Long browserId, boolean dismiss) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    inst.dismissDialogs = dismiss;
    return RespBodyVo.ok(Kv.by("dismiss", dismiss));
  }

    put("get_modals", (svc, id, a) -> svc.getModals(id));

  public RespBodyVo getModals(Long browserId) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv probe = modalProbe(inst);
    if (probe == null) {
      return RespBodyVo.fail("get_modals 失败：读取页面弹窗失败");
    }
    return RespBodyVo.ok(probe);
  }

    put("close_modal", (svc, id, a) -> svc.closeModal(id, optStr(a, "which"), optStr(a, "title"),
        optStr(a, "button")));

  public RespBodyVo closeModal(Long browserId, String which, String title, String button) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv before = modalProbe(inst);
    if (before == null) {
      return RespBodyVo.fail("close_modal 失败：读取页面弹窗失败");
    }
    int countBefore = asInt(before.get("count"));
    if (countBefore == 0) {
      return RespBodyVo.ok(Kv.by("closed", false).set("countBefore", 0).set("countAfter", 0)
          .set("note", "当前没有可见的 DOM 弹窗,无需关闭"));
    }
    String mode = which == null || which.isBlank() ? "top" : which.trim().toLowerCase(java.util.Locale.ROOT);
    List<Kv> targets = pickModals(before, mode, title);
    if (targets.isEmpty()) {
      return RespBodyVo.fail("close_modal 失败：没有匹配的弹窗（" + (mode.startsWith("class:")
          ? "className 含「" + mode.substring("class:".length()) + "」的弹窗不存在"
          : "标题/文本含「" + title + "」的弹窗不存在")
          + ",当前共 " + countBefore + " 个,用 get_modals 看清单（每项都有 className 与 matchedBy））");
    }
    List<Kv> clicked = new ArrayList<>();
    for (Kv target : targets) {
      Kv hit = clickModalButton(inst, target, button);
      if (hit != null) {
        clicked.add(hit);
      }
      // 关掉一个就重新探一次:数量真的减少了才算成功
      if (!"all".equals(mode)) {
        break;
      }
      Kv now = modalProbe(inst);
      if (now == null || asInt(now.get("count")) == 0) {
        break;
      }
    }
    Kv after = modalProbe(inst);
    int countAfter = after == null ? countBefore : asInt(after.get("count"));
    boolean closed = countAfter < countBefore;
    Kv data = Kv.by("closed", closed).set("countBefore", countBefore).set("countAfter", countAfter)
        .set("clicked", clicked).set("closedAll", countAfter == 0).set("which", mode);
    if (title != null && !title.isBlank()) {
      data.set("title", title);
    }
    if (!closed) {
      data.set("hint", "点了按钮但弹窗数量没减少:该弹窗可能只认真实鼠标事件、或点中的不是它的关闭按钮。"
          + "用 get_modals 拿 closePoint / buttonPoints,再直接 mouse_click 那个坐标");
    }
    return RespBodyVo.ok(data);
  }

    put("request_human_input", (svc, id, a) -> {
      // prompt 与 steps 二选一:两个都没给时在**碰浏览器对象之前**就说清楚是缺哪个参数
      if (a.getJSONArray("steps") == null && optStrRaw(a, "prompt") == null) {
        throw new IllegalArgumentException("缺少参数 prompt");
      }
      // expiresInSeconds 与 timeoutSeconds 是同一件事的两个名字:回执里输出的字段就叫
      // expiresInSeconds,照着回执写回来却完全没有效果(被静默忽略、按默认 300 秒过期),
      // 这个不对称实测确实坑过人,所以两个名字都收。
      return svc.requestHumanInput(id, optStrRaw(a, "prompt"), a.getInteger("index"), optStr(a, "selector"),
          a.getInteger("timeoutSeconds"), stepListOf(a.getJSONArray("steps")), a.getLong("expiresAt"),
          a.getInteger("expiresInSeconds"), a.getBoolean("ocr"), optStr(a, "ocrLanguage"),
          a.getBoolean("inline"), optStr(a, "frame"));
    });

  static long humanDeadline(long now, Integer timeoutSeconds, Long expiresAt, Integer expiresInSeconds) {
    if (expiresAt != null && expiresAt > 0) {
      return expiresAt;
    }
    int ttl;
    if (expiresInSeconds != null && expiresInSeconds > 0) {
      ttl = expiresInSeconds;
    } else {
      ttl = timeoutSeconds == null || timeoutSeconds <= 0 ? DEFAULT_HUMAN_TIMEOUT_SECONDS : timeoutSeconds;
    }
    return now + ttl * 1_000L;
  }

  public RespBodyVo requestHumanInput(Long browserId, String prompt, Integer index, String selector,
      Integer timeoutSeconds, List<Kv> steps, Long expiresAt, Integer expiresInSeconds, Boolean ocr,
      String ocrLanguage, Boolean inline, String frame) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    if ((prompt == null || prompt.isBlank()) && (steps == null || steps.isEmpty())) {
      return RespBodyVo.fail("request_human_input 需要 prompt,或用 steps 给出待办清单");
    }
    long now = System.currentTimeMillis();
    long deadline = humanDeadline(now, timeoutSeconds, expiresAt, expiresInSeconds);
    String requestId = "hr-" + inst.humanSeq.incrementAndGet() + "-" + SnowflakeIdUtils.id();
    Kv request = Kv.by("requestId", requestId).set("prompt", prompt).set("status", "pending").set("answer", null)
        .set("createdAt", now).set("expiresAt", deadline);
    inst.humanRequests.put(requestId, request);
    // 需要人工介入时把页面带到最前,人才能直接看到验证码/表单
    activate(inst.page);
    Kv data = Kv.by("requestId", requestId).set("prompt", prompt).set("expiresAt", deadline)
        .set("expiresInSeconds", Math.max(0, (deadline - now) / 1000)).set("url", inst.page.url());
    if (deadline <= now) {
      // 过期时刻已经过去了:直接说清楚,别让人等一个永远不会来的答复
      request.set("status", "expired");
      data.set("status", "expired").set("note", "expiresAt 已经过去了,这个请求没有生效;请重新发起");
      return RespBodyVo.ok(data);
    }

    boolean wantOcr = Boolean.TRUE.equals(ocr);
    boolean wantInline = inline == null || inline;
    // 第一个要看图的目标:老写法(index/selector)或 steps 里第一个带目标的步骤
    Integer shotIndex = index;
    String shotSelector = selector;
    String shotPrompt = prompt;
    // 验证码/二维码经常在跨域 iframe 里(实测企业微信登录页的二维码就在 iframe 中),
    // 不给 frame 时 selector 只在顶层文档找,回执里 imageUrl 会是空的 —— 所以这里一路透传下去
    String shotFrame = frame;
    if (steps != null && !steps.isEmpty()) {
      List<Kv> items = new ArrayList<>();
      int seq = 0;
      for (Kv step : steps) {
        String stepId = "s" + (++seq);
        Kv item = Kv.by("stepId", stepId).set("prompt", step.getStr("prompt")).set("status", "pending")
            .set("answer", null);
        if (step.get("index") != null) {
          item.set("index", step.get("index"));
        }
        if (step.getStr("selector") != null) {
          item.set("selector", step.getStr("selector"));
        }
        if (step.getStr("frame") != null) {
          item.set("frame", step.getStr("frame"));
        }
        items.add(item);
      }
      request.set("steps", items).set("status", "pending");
      data.set("steps", items).set("stepCount", items.size());
      if (shotPrompt == null || shotPrompt.isBlank()) {
        shotPrompt = steps.get(0).getStr("prompt");
      }
      if (shotIndex == null && (shotSelector == null || shotSelector.isBlank())) {
        Object stepIndex = steps.get(0).get("index");
        String stepSelector = steps.get(0).getStr("selector");
        shotIndex = stepIndex instanceof Number ? ((Number) stepIndex).intValue() : null;
        shotSelector = stepSelector;
        if (shotFrame == null || shotFrame.isBlank()) {
          shotFrame = steps.get(0).getStr("frame");
        }
      }
      data.set("note", "这是一次带多步待办的请求:让人一次做完,再用 submit_human_input 按 stepId 逐个回填;"
          + "全部回填后 get_human_input 的 status 会变成 answered");
    }

    if (shotIndex != null || (shotSelector != null && !shotSelector.isEmpty())) {
      // 请人看验证码/二维码是「确实必须看图」的场景:落盘 + 内联 + 可 GET 的 URL 一起给,
      // 读不了图的模型至少还能把 imageUrl 贴给用户,或走 OCR
      String path = defaultShotPath(inst);
      RespBodyVo shot = elementScreenshot(inst, shotIndex, shotSelector, path, wantInline, shotFrame);
      if (shot.isOk() && shot.getData() instanceof Kv) {
        Kv shotData = (Kv) shot.getData();
        if (wantInline) {
          data.set("imageBase64", shotData.getStr("base64"));
        }
        data.set("imageSize", shotData.get("size"))
            .set("imagePath", shotData.getStr("path"))
            .set("imageUrl", shotData.getStr("url"))
            // 截图取自哪个 frame/选择器:跨域 iframe 里截图时,这是判断「到底截到没有」的唯一线索
            .set("imageTarget", shotData.getStr("target"));
      } else {
        data.set("imageError", shot.getMsg());
      }
    }

    if (wantOcr) {
      String imagePath = data.getStr("imagePath");
      if (imagePath == null || imagePath.isEmpty()) {
        data.set("ocrError", "ocr:true 需要 index / selector(或 steps 里的第一个带目标的步骤)来指定要读的图");
      } else {
        Kv read = WindowsOcr.read(Paths.get(imagePath), ocrLanguage);
        data.set("ocr", read);
        if (Boolean.TRUE.equals(read.get("ok"))) {
          data.set("ocrText", read.getStr("text"));
        } else {
          data.set("ocrError", read.getStr("error"));
        }
      }
    }
    return RespBodyVo.ok(data);
  }

    // 向用户「要一个值」,而不是「请人去页面上操作」:银行卡号、身份证号、验证码、二选一。
    // 与 request_human_input 共用同一套待办存储,所以 submit_human_input / get_human_input 原样可用。
    put("ask_user", (svc, id, a) -> {
      if (a.getJSONArray("questions") == null) {
        throw new IllegalArgumentException("缺少参数 questions");
      }
      return svc.askUser(id, questionListOf(a.getJSONArray("questions")), a.getInteger("timeoutSeconds"),
          a.getLong("expiresAt"), a.getInteger("expiresInSeconds"));
    });

  private static java.util.List<Kv> questionListOf(com.alibaba.fastjson2.JSONArray questions) {
    return stepListOf(questions);
  }

  public RespBodyVo askUser(Long browserId, List<Kv> questions, Integer timeoutSeconds, Long expiresAt,
      Integer expiresInSeconds) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    if (questions == null || questions.isEmpty()) {
      return RespBodyVo.fail("ask_user 需要 questions(至少一个问题,每个问题要有 id 与 question)");
    }
    for (int i = 0; i < questions.size(); i++) {
      Kv question = questions.get(i);
      if (question == null || question.getStr("id") == null || question.getStr("id").isBlank()) {
        return RespBodyVo.fail("ask_user 第 " + (i + 1) + " 个问题缺少 id:答复要按 id 回填,没有 id 就对不上");
      }
      if (question.getStr("question") == null || question.getStr("question").isBlank()) {
        return RespBodyVo.fail("ask_user 第 " + (i + 1) + " 个问题缺少 question");
      }
    }
    long now = System.currentTimeMillis();
    long deadline = humanDeadline(now, timeoutSeconds, expiresAt, expiresInSeconds);
    String requestId = "aq-" + inst.humanSeq.incrementAndGet() + "-" + SnowflakeIdUtils.id();

    List<Kv> steps = new ArrayList<>();
    List<String> ids = new ArrayList<>();
    for (Kv question : questions) {
      String id = question.getStr("id");
      Kv step = Kv.by("stepId", id).set("prompt", question.getStr("question")).set("status", "pending")
          .set("answer", null);
      if (question.get("header") != null) {
        step.set("header", question.get("header"));
      }
      if (question.get("options") != null) {
        step.set("options", question.get("options"));
      }
      if (question.get("multiSelect") != null) {
        step.set("multiSelect", question.get("multiSelect"));
      }
      steps.add(step);
      ids.add(id);
    }

    String prompt = "请回答 " + steps.size() + " 个问题:" + String.join(" / ", ids);
    Kv request = Kv.by("requestId", requestId).set("prompt", prompt).set("status", "pending").set("answer", null)
        .set("createdAt", now).set("expiresAt", deadline).set("kind", "ask_user").set("steps", steps);
    inst.humanRequests.put(requestId, request);

    Kv data = Kv.by("requestId", requestId).set("prompt", prompt).set("expiresAt", deadline)
        .set("expiresInSeconds", Math.max(0, (deadline - now) / 1000)).set("questions", questions)
        .set("questionIds", ids);
    if (deadline <= now) {
      request.set("status", "expired");
      data.set("status", "expired").set("note", "expiresAt 已经过去了,这个提问没有生效;请重新发起");
      return RespBodyVo.ok(data);
    }
    // 与 request_human_input 不同:纯问答**不抢焦点**。人可能正看着别处,而这个问题不要求他去看页面
    return RespBodyVo.ok(data);
  }

  static JSONArray askUserAnswers(List<Kv> steps) {
    JSONArray out = new JSONArray();
    for (Kv step : steps) {
      JSONObject one = new JSONObject();
      one.put("id", step.getStr("stepId"));
      List<String> selected = new ArrayList<>();
      String custom = null;
      Object raw = step.get("answer");
      if (raw instanceof List) {
        for (Object item : (List<?>) raw) {
          selected.add(String.valueOf(item));
        }
      } else if (raw != null) {
        String text = String.valueOf(raw);
        for (String label : optionLabels(step)) {
          if (label.equals(text)) {
            selected.add(label);
            break;
          }
        }
        if (selected.isEmpty()) {
          custom = text;
        }
      }
      one.put("selected", selected);
      if (custom != null) {
        one.put("custom", custom);
      }
      out.add(one);
    }
    return out;
  }

  /** 某个问题的选项标签:选项允许写成字符串或 {@code {label, description}} 两种形态 */
  private static List<String> optionLabels(Kv step) {
    List<String> labels = new ArrayList<>();
    Object raw = step.get("options");
    if (!(raw instanceof List)) {
      return labels;
    }
    for (Object item : (List<?>) raw) {
      if (item instanceof Map) {
        Object label = ((Map<?, ?>) item).get("label");
        if (label != null) {
          labels.add(String.valueOf(label));
        }
      } else if (item != null) {
        labels.add(String.valueOf(item));
      }
    }
    return labels;
  }

    put("get_human_input",
        (svc, id, a) -> svc.getHumanInput(id, reqStr(a, "requestId"), a.getInteger("timeoutSeconds")));

  public RespBodyVo getHumanInput(Long browserId, String requestId, Integer timeoutSeconds) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv request = inst.humanRequests.get(requestId);
    if (request == null) {
      return RespBodyVo.fail("get_human_input 没有这个请求: " + requestId);
    }
    long deadline = timeoutSeconds == null || timeoutSeconds <= 0 ? 0
        : System.currentTimeMillis() + timeoutSeconds * 1_000L;
    while ("pending".equals(humanStatus(request)) && System.currentTimeMillis() < deadline) {
      sleepQuietly(500);
    }
    String status = humanStatus(request);
    Kv data = Kv.by("requestId", requestId).set("status", status).set("answer", request.get("answer"))
        .set("prompt", request.get("prompt")).set("expiresAt", request.get("expiresAt"));
    List<Kv> steps = stepList(request);
    if (!steps.isEmpty()) {
      data.set("steps", steps);
      boolean anyAnswered = false;
      for (Kv step : steps) {
        if ("answered".equals(step.getStr("status"))) {
          anyAnswered = true;
          break;
        }
      }
      if (anyAnswered && "pending".equals(status)) {
        data.set("status", "partial");
        status = "partial";
      }
    }
    if ("ask_user".equals(request.getStr("kind"))) {
      // 除了逐条的 steps,再给一份与模型侧提问工具同形的 answers:
      // 调用方拿到的就是它熟悉的那个结构,不必自己把 steps 再拼一遍
      data.set("answers", askUserAnswers(steps));
    }
    if ("expired".equals(status)) {
      data.set("expired", true)
          .set("hint", "这个人工请求已经过期:二维码/短信码这类有短时效的凭证通常也已经失效,"
              + "请重新发起 request_human_input(可以用 expiresAt 明确告诉它什么时候过期)");
    }
    return RespBodyVo.ok(data);
  }

    put("submit_human_input", (svc, id, a) -> svc.submitHumanInput(id, reqStr(a, "requestId"),
        optStrRaw(a, "answer"), optStr(a, "stepId"), a.getJSONObject("answers")));

  public RespBodyVo submitHumanInput(Long browserId, String requestId, String answer, String stepId,
      JSONObject answers) {
    BrowserInstance inst = INSTANCES.get(browserId);
    if (inst == null) {
      return notFound(browserId);
    }
    Kv request = inst.humanRequests.get(requestId);
    if (request == null) {
      return RespBodyVo.fail("submit_human_input 没有这个请求: " + requestId);
    }
    long now = System.currentTimeMillis();
    List<Kv> steps = stepList(request);
    // 多步待办:必须按 stepId / answers 逐条回填 —— 直接给一个 answer 说不清它对应哪一步,静默接受
    // 只会让剩下的步骤一直挂在 pending 上,所以这里明确报错并给出待办清单
    if (!steps.isEmpty()) {
      Map<String, Object> filled = new LinkedHashMap<>();
      if (answers != null) {
        for (String key : answers.keySet()) {
          // 不把值强转成字符串:多选题的答复是一个数组(选了哪几项),强转会把选项信息压扁
          filled.put(key, answers.get(key));
        }
      }
      if (stepId != null) {
        filled.put(stepId, answer);
      }
      if (filled.isEmpty()) {
        List<String> pendingIds = new ArrayList<>();
        for (Kv step : steps) {
          pendingIds.add(step.getStr("stepId") + "(" + step.getStr("prompt") + ")");
        }
        return RespBodyVo.fail("submit_human_input 这条请求有 " + steps.size() + " 个待办步骤,"
            + "请用 stepId 指定回填哪一步,或用 answers 一次回填多步。待办:" + String.join(" / ", pendingIds));
      }
      List<String> unknown = new ArrayList<>();
      for (Map.Entry<String, Object> entry : filled.entrySet()) {
        Kv target = null;
        for (Kv step : steps) {
          if (entry.getKey().equals(step.getStr("stepId"))) {
            target = step;
            break;
          }
        }
        if (target == null) {
          unknown.add(entry.getKey());
          continue;
        }
        target.set("answer", entry.getValue()).set("status", "answered").set("answeredAt", now);
      }
      if (!unknown.isEmpty()) {
        return RespBodyVo.fail("submit_human_input 不认识的 stepId: " + String.join(" / ", unknown));
      }
      boolean allDone = true;
      List<Kv> pendingSteps = new ArrayList<>();
      for (Kv step : steps) {
        if (!"answered".equals(step.getStr("status"))) {
          allDone = false;
          pendingSteps.add(Kv.by("stepId", step.getStr("stepId")).set("prompt", step.getStr("prompt")));
        }
      }
      if (allDone) {
        request.set("status", "answered").set("answeredAt", now);
      }
      Kv data = Kv.by("requestId", requestId)
          .set("status", allDone ? "answered" : "partial").set("steps", steps);
      if (!allDone) {
        data.set("pendingSteps", pendingSteps)
            .set("note", "还有 " + pendingSteps.size() + " 步没回填;全部回填后 status 才是 answered");
      }
      return RespBodyVo.ok(data);
    }
    if (answer == null) {
      return RespBodyVo.fail("submit_human_input 需要 answer(或用 stepId / answers 逐条回填)");
    }
    request.set("answer", answer).set("status", "answered").set("answeredAt", now);
    return RespBodyVo.ok(Kv.by("requestId", requestId).set("status", "answered").set("answer", answer));
  }