+ }
+ const lines = snapshot.outline.split('\n')
+ const emittedRefs = Array.from(snapshot.outline.matchAll(/\[ref=(\d+)\]/g), (match) =>
+ Number(match[1])
+ )
+ const indexedRefs = Object.keys(snapshot.refLineIndexes).map(Number)
+
+ expect(snapshot.truncated).toBe(true)
+ expect(lines).toHaveLength(600)
+ expect(snapshot.outline).toContain('button "Emitted"')
+ expect(snapshot.outline).not.toContain('button "Truncated"')
+ expect(snapshot.refIds).toEqual(emittedRefs)
+ expect(indexedRefs).toEqual(emittedRefs)
+ for (const ref of indexedRefs) {
+ expect(lines[snapshot.refLineIndexes[ref]]).toContain(`[ref=${ref}]`)
+ }
+ })
+
+ it('marks file inputs unsupported and refuses to open a native chooser', () => {
+ document.body.innerHTML = ''
+ visible(document.querySelector('input') as HTMLInputElement)
+ const outline = outlineOf(collectSnapshot())
+ const ref = refFor(outline, 'Upload receipt')
+
+ expect(outline).toContain('file-input "Upload receipt"')
+ expect(outline).toContain('upload-unsupported')
+ expect(clickElement(ref)).toEqual({ error: 'file-input' })
+ })
+
+ it('keeps plain visible leaf text available as an actionable ref', () => {
+ document.body.innerHTML = 'announce
'
+ visible(document.querySelector('span') as HTMLSpanElement)
+
+ expect(outlineOf(collectSnapshot())).toContain('text "announce" [ref=')
+ })
+
+ it('retains sender and timestamp text omitted from a row accessibility label', () => {
+ document.body.innerHTML = `
+
+ Sid Studio
+ Quarterly plan
+ 11:42 AM
+
+
+ `
+ visible(document.querySelector('[role="link"]') as HTMLDivElement)
+ for (const child of document.querySelectorAll('span')) visible(child)
+
+ const outline = outlineOf(collectSnapshot())
+
+ expect(outline).toContain('link "Quarterly plan Updated forecast"')
+ expect(outline).toContain('text "Sid Studio"')
+ expect(outline).toContain('text "11:42 AM"')
+ expect(outline).toContain('text "Has attachment"')
+ expect(outline).not.toContain('text "Quarterly plan"')
+ })
+
+ it('recovers a ref when React uniquely replaces the same logical element', () => {
+ document.body.innerHTML = ''
+ const original = visible(document.querySelector('button') as HTMLButtonElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'Messages')
+ const replacement = visible(original.cloneNode(true) as HTMLButtonElement)
+ let clicked = false
+ replacement.addEventListener('click', () => {
+ clicked = true
+ })
+ original.replaceWith(replacement)
+
+ expect(clickElement(ref)).toMatchObject({
+ dispatched: true,
+ refRecovered: true,
+ })
+ expect(clicked).toBe(true)
+ })
+
+ it('recovers from a connected but collapsed node to its unique visible replacement', () => {
+ document.body.innerHTML =
+ ''
+ const original = visible(document.querySelector('[role="combobox"]') as HTMLDivElement)
+ visible(document.querySelector('input') as HTMLInputElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'To:')
+ original.getBoundingClientRect = () =>
+ ({ width: 0, height: 0, top: 0, left: 0, right: 0, bottom: 0 }) as DOMRect
+ const replacement = visible(original.cloneNode(true) as HTMLDivElement)
+ visible(replacement.querySelector('input') as HTMLInputElement)
+ document.body.append(replacement)
+
+ expect(focusElementForTyping(ref)).toMatchObject({
+ focused: true,
+ refRecovered: true,
+ })
+ })
+
+ it('does not guess between visible replacements for a collapsed connected ref', () => {
+ document.body.innerHTML =
+ ''
+ const original = visible(document.querySelector('[role="combobox"]') as HTMLDivElement)
+ visible(document.querySelector('input') as HTMLInputElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'To:')
+ original.getBoundingClientRect = () =>
+ ({ width: 0, height: 0, top: 0, left: 0, right: 0, bottom: 0 }) as DOMRect
+ for (let index = 0; index < 2; index++) {
+ const replacement = visible(original.cloneNode(true) as HTMLDivElement)
+ visible(replacement.querySelector('input') as HTMLInputElement)
+ document.body.append(replacement)
+ }
+
+ expect(focusElementForTyping(ref)).toEqual({ error: 'stale' })
+ })
+
+ it('refuses to recover a ref when replacement is ambiguous', () => {
+ document.body.innerHTML = ''
+ const original = visible(document.querySelector('button') as HTMLButtonElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'Close')
+ const first = visible(original.cloneNode(true) as HTMLButtonElement)
+ const second = visible(original.cloneNode(true) as HTMLButtonElement)
+ original.replaceWith(first, second)
+
+ expect(clickElement(ref)).toEqual({ error: 'stale' })
+ })
+
+ it('invalidates a connected virtual row when its identity is recycled in place', () => {
+ document.body.innerHTML = 'eng-bugs
'
+ const row = visible(document.querySelector('[role="listitem"]') as HTMLDivElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'eng-bugs')
+
+ row.textContent = 'random'
+ row.dataset.key = 'channel-2'
+
+ expect(clickElement(ref)).toEqual({ error: 'stale' })
+ })
+
+ it('invalidates a generic connected row action when its surrounding item is recycled', () => {
+ document.body.innerHTML = `
+ eng-bugs
+ `
+ for (const element of document.querySelectorAll('*')) visible(element as HTMLElement)
+ const button = document.querySelector('button') as HTMLButtonElement
+ const ref = refFor(outlineOf(collectSnapshot()), 'More actions')
+
+ ;(document.querySelector('span') as HTMLSpanElement).textContent = 'random'
+
+ expect(button.isConnected).toBe(true)
+ expect(clickElement(ref)).toEqual({ error: 'stale' })
+ })
+
+ it('never recycles numeric refs across snapshots', () => {
+ document.body.innerHTML = ''
+ visible(document.querySelector('button') as HTMLButtonElement)
+ const firstRef = refFor(outlineOf(collectSnapshot()), 'Pins')
+ const secondRef = refFor(outlineOf(collectSnapshot()), 'Pins')
+
+ expect(secondRef).toBeGreaterThan(firstRef)
+ expect(clickElement(firstRef)).toEqual({ error: 'stale' })
+ expect(clickElement(secondRef)).toMatchObject({ dispatched: true })
+ })
+
+ it('reports a targeted control semantic disappearance after its panel closes', () => {
+ document.body.innerHTML = `
+
+ `
+ const panel = document.querySelector('aside') as HTMLElement
+ visible(panel)
+ visible(document.querySelector('button') as HTMLButtonElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'Close thread')
+
+ const before = readPageActionState(true, ref) as {
+ targetState: { present: boolean; rendered: boolean }
+ }
+ panel.remove()
+ const composer = visible(document.createElement('textarea'))
+ composer.setAttribute('aria-label', 'Message')
+ document.body.append(composer)
+ const after = readPageActionState(false, ref) as {
+ targetState: { present: boolean; rendered: boolean }
+ }
+
+ expect(before.targetState).toMatchObject({ present: true, rendered: true })
+ expect(after.targetState).toEqual({ present: false, rendered: false })
+ })
+
+ it('keeps semantic target presence through a unique React replacement', () => {
+ document.body.innerHTML =
+ ''
+ const original = visible(document.querySelector('button') as HTMLButtonElement)
+ const ref = refFor(outlineOf(collectSnapshot()), 'Close thread')
+ const before = readPageActionState(true, ref) as { targetState: unknown }
+ const replacement = visible(original.cloneNode(true) as HTMLButtonElement)
+ original.replaceWith(replacement)
+ const after = readPageActionState(false, ref) as { targetState: unknown }
+
+ expect(after.targetState).toEqual(before.targetState)
+ })
+})
+
+describe('scrollPage', () => {
+ function makeScroller(scrollTop: number): {
+ scroller: HTMLDivElement
+ child: HTMLDivElement
+ } {
+ document.body.innerHTML =
+ ''
+ const scroller = visible(document.querySelector('#messages') as HTMLDivElement)
+ const child = visible(scroller.firstElementChild as HTMLDivElement)
+ Object.defineProperties(scroller, {
+ clientHeight: { configurable: true, value: 200 },
+ scrollHeight: { configurable: true, value: 1_000 },
+ scrollTop: { configurable: true, writable: true, value: scrollTop },
+ })
+ Object.defineProperty(scroller, 'scrollBy', {
+ configurable: true,
+ value: ({ top }: ScrollToOptions) => {
+ const next = scroller.scrollTop + (top || 0)
+ scroller.scrollTop = Math.max(0, Math.min(800, next))
+ },
+ })
+ return { scroller, child }
+ }
+
+ it('scrolls the movable internal container under the viewport center', () => {
+ const { scroller, child } = makeScroller(600)
+ Object.defineProperty(document, 'elementsFromPoint', {
+ configurable: true,
+ value: () => [child, scroller],
+ })
+
+ expect(scrollPage('up', 100)).toMatchObject({
+ target: 'Message history',
+ targetSource: 'viewport-center',
+ scrollTop: 500,
+ movedBy: -100,
+ atTop: false,
+ atBottom: false,
+ })
+ })
+
+ it('targets the nearest scrollable ancestor of an explicit ref', () => {
+ const { scroller, child } = makeScroller(0)
+ const ref = refFor(outlineOf(collectSnapshot()), 'message')
+
+ expect(scrollPage('down', 125, ref)).toMatchObject({
+ target: 'Message history',
+ targetSource: 'element',
+ scrollTop: 125,
+ movedBy: 125,
+ })
+ expect(child.textContent).toBe('message')
+ expect(scroller.scrollTop).toBe(125)
+ })
+
+ it('walks past an immovable nearest scroller to a movable ancestor for an explicit ref', () => {
+ document.body.innerHTML = `
+
+ `
+ const outer = visible(document.querySelector('#outer') as HTMLDivElement)
+ const inner = visible(document.querySelector('#inner') as HTMLDivElement)
+ const message = visible(inner.firstElementChild as HTMLDivElement)
+ for (const [element, scrollTop] of [
+ [outer, 500],
+ [inner, 0],
+ ] as const) {
+ Object.defineProperties(element, {
+ clientHeight: { configurable: true, value: 200 },
+ scrollHeight: { configurable: true, value: 1_000 },
+ scrollTop: { configurable: true, writable: true, value: scrollTop },
+ })
+ Object.defineProperty(element, 'scrollBy', {
+ configurable: true,
+ value: ({ top }: ScrollToOptions) => {
+ element.scrollTop = Math.max(0, Math.min(800, element.scrollTop + (top || 0)))
+ },
+ })
+ }
+ const ref = refFor(outlineOf(collectSnapshot()), 'message')
+
+ expect(scrollPage('up', 100, ref)).toMatchObject({
+ target: 'Workspace',
+ targetSource: 'element',
+ movedBy: -100,
+ scrollTop: 400,
+ })
+ expect(message.textContent).toBe('message')
+ expect(inner.scrollTop).toBe(0)
+ expect(outer.scrollTop).toBe(400)
+ })
+
+ it('skips an immovable focused sidebar for the movable centered history pane', () => {
+ document.body.innerHTML = `
+
+
+ `
+ const sidebar = visible(document.querySelector('#sidebar') as HTMLDivElement)
+ const history = visible(document.querySelector('#history') as HTMLDivElement)
+ const message = visible(history.firstElementChild as HTMLDivElement)
+ for (const [element, scrollTop] of [
+ [sidebar, 0],
+ [history, 600],
+ ] as const) {
+ Object.defineProperties(element, {
+ clientHeight: { configurable: true, value: 200 },
+ scrollHeight: { configurable: true, value: 1_000 },
+ scrollTop: { configurable: true, writable: true, value: scrollTop },
+ })
+ Object.defineProperty(element, 'scrollBy', {
+ configurable: true,
+ value: ({ top }: ScrollToOptions) => {
+ element.scrollTop = Math.max(0, Math.min(800, element.scrollTop + (top || 0)))
+ },
+ })
+ }
+ setActiveElement(document, sidebar)
+ Object.defineProperty(document, 'elementsFromPoint', {
+ configurable: true,
+ value: () => [message, history],
+ })
+
+ expect(scrollPage('up', 100)).toMatchObject({
+ target: 'Message history',
+ targetSource: 'viewport-center',
+ movedBy: -100,
+ })
+ expect(sidebar.scrollTop).toBe(0)
+ expect(history.scrollTop).toBe(500)
+ })
+
+ it('keeps a centered pane at its boundary instead of scrolling another pane', () => {
+ const { scroller: history, child: message } = makeScroller(800)
+ const sidebar = visible(document.createElement('div'))
+ sidebar.setAttribute('aria-label', 'Channels')
+ sidebar.style.overflowY = 'auto'
+ document.body.prepend(sidebar)
+ Object.defineProperties(sidebar, {
+ clientHeight: { configurable: true, value: 200 },
+ scrollHeight: { configurable: true, value: 1_000 },
+ scrollTop: { configurable: true, writable: true, value: 0 },
+ })
+ Object.defineProperty(sidebar, 'scrollBy', {
+ configurable: true,
+ value: ({ top }: ScrollToOptions) => {
+ sidebar.scrollTop += top || 0
+ },
+ })
+ Object.defineProperty(document, 'elementsFromPoint', {
+ configurable: true,
+ value: () => [message, history],
+ })
+ setActiveElement(document, document.body)
+
+ expect(scrollPage('down', 100)).toMatchObject({
+ target: 'Message history',
+ targetSource: 'viewport-center-boundary',
+ movedBy: 0,
+ atBottom: true,
+ })
+ expect(sidebar.scrollTop).toBe(0)
+ })
+})
+
+describe('readChildFrameElementState', () => {
+ it('rejects a frame hidden by an embedding ancestor', () => {
+ document.body.innerHTML = `
+
+
+
+ `
+ visible(document.querySelector('iframe') as HTMLIFrameElement)
+
+ expect(readChildFrameElementState('apps', '', '', 0)).toMatchObject({
+ known: true,
+ visible: false,
+ frameName: 'apps',
+ })
+ })
+
+ it('rejects a covered frame and reports the blocking surface', () => {
+ document.body.innerHTML = `
+
+
+ `
+ const frame = visible(document.querySelector('iframe') as HTMLIFrameElement)
+ const overlay = visible(document.querySelector('div') as HTMLDivElement)
+ Object.defineProperty(document, 'elementFromPoint', {
+ configurable: true,
+ value: () => overlay,
+ })
+
+ expect(frame.isConnected).toBe(true)
+ expect(readChildFrameElementState('apps', '', '', 0)).toMatchObject({
+ known: true,
+ visible: false,
+ blocker: 'Consent overlay',
+ frameName: 'apps',
+ })
+ })
+
+ it('hit-tests a frame against its shadow root instead of the outer document', () => {
+ const host = document.createElement('div')
+ document.body.append(host)
+ const shadow = host.attachShadow({ mode: 'open' })
+ const frame = document.createElement('iframe')
+ frame.name = 'apps'
+ shadow.append(frame)
+ visible(frame)
+ Object.defineProperty(shadow, 'elementFromPoint', {
+ configurable: true,
+ value: () => frame,
+ })
+ Object.defineProperty(document, 'elementFromPoint', {
+ configurable: true,
+ value: () => host,
+ })
+
+ expect(readChildFrameElementState('apps', '', '', 0)).toMatchObject({
+ known: true,
+ visible: true,
+ frameName: 'apps',
+ })
+ })
+
+ it('uses WindowProxy identity to distinguish duplicate frame metadata', () => {
+ document.body.innerHTML = `
+
+
+ `
+ const frames = Array.from(document.querySelectorAll('iframe')) as HTMLIFrameElement[]
+ frames.forEach(visible)
+ Object.defineProperty(document, 'elementFromPoint', {
+ configurable: true,
+ value: () => frames[1],
+ })
+
+ expect(
+ readChildFrameElementState('apps', 'https://example.com/widget', 'https://example.com', 1)
+ ).toMatchObject({ known: true, visible: true, frameName: 'apps' })
+ })
})
describe('readActiveElementState', () => {
@@ -346,7 +1119,10 @@ describe('readActiveElementState', () => {
''
setActiveElement(document, document.querySelector('input'))
- expect(readActiveElementState()).toMatchObject({ redacted: true, valuePreview: '' })
+ expect(readActiveElementState()).toMatchObject({
+ redacted: true,
+ valuePreview: '',
+ })
})
it.each([
@@ -399,7 +1175,10 @@ describe('XHTML lower-case tagName', () => {
function lowerCaseTagInput(html: string): HTMLInputElement {
document.body.innerHTML = html
const input = document.querySelector('input') as HTMLInputElement
- Object.defineProperty(input, 'tagName', { configurable: true, get: () => 'input' })
+ Object.defineProperty(input, 'tagName', {
+ configurable: true,
+ get: () => 'input',
+ })
return input
}
@@ -431,6 +1210,24 @@ describe('activeElementSecrecy', () => {
expect(activeElementSecrecy()).toBe('safe')
})
+ it('distinguishes a different focused element from an invalid target ref', () => {
+ document.body.innerHTML = `
+
+
+ `
+ const expected = visible(document.querySelectorAll('input')[0])
+ const other = visible(document.querySelectorAll('input')[1])
+ const snapshot = collectSnapshot() as { refIds: number[] }
+ const expectedRef = snapshot.refIds[0]
+ setActiveElement(document, other)
+
+ expect(activeElementSecrecy(expectedRef)).toBe('different')
+ expect(activeElementSecrecy(Number.MAX_SAFE_INTEGER)).toBe('stale')
+
+ setActiveElement(document, expected)
+ expect(activeElementSecrecy(expectedRef)).toBe('safe')
+ })
+
it('reports safe when nothing is focused', () => {
setActiveElement(document, document.body)
@@ -460,7 +1257,10 @@ describe('activeElementSecrecy', () => {
document.body.append(frame)
// A cross-origin frame yields null here; jsdom cannot host one, so the
// boundary is reproduced directly.
- Object.defineProperty(frame, 'contentDocument', { configurable: true, get: () => null })
+ Object.defineProperty(frame, 'contentDocument', {
+ configurable: true,
+ get: () => null,
+ })
setActiveElement(document, frame)
expect(activeElementSecrecy()).toBe('opaque')
diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts
index a0d99506286..f5f4cb2f9a3 100644
--- a/apps/desktop/src/main/browser-agent/page-functions.ts
+++ b/apps/desktop/src/main/browser-agent/page-functions.ts
@@ -6,8 +6,10 @@
* arguments and page globals. Helpers live INSIDE the function that uses them.
*
* The element registry (`window.__simAgentElements`) is rebuilt by every
- * snapshot and naturally cleared by navigation; interaction functions treat a
- * missing or disconnected entry as a stale id.
+ * snapshot and naturally cleared by navigation. A snapshot also installs a
+ * semantic resolver that can recover a ref when React replaces the same
+ * logical control with a new DOM node; navigation and ambiguous matches still
+ * make the ref stale.
*
* Several functions repeat an identical `isSecretField` helper. That
* duplication is required, not accidental: self-containment means a shared
@@ -27,6 +29,13 @@
declare global {
interface Window {
__simAgentElements?: Element[]
+ __simAgentResolveElement?: (id: number) => { element: Element; recovered: boolean } | null
+ __simAgentMutationStates?: Array<{
+ root: Node
+ observer: MutationObserver
+ revision: number
+ }>
+ __simAgentNextElementId?: number
}
}
@@ -35,17 +44,42 @@ declare global {
* interactive elements carrying numeric ids, walking open shadow roots and
* same-origin iframes. Rebuilds the element registry as a side effect.
*/
-export function collectSnapshot(): unknown {
+export function collectSnapshot(startingElementId = 0): unknown {
const refCap = 300
const lineCap = 600
+ const nodeCap = 12_000
+ const depthCap = 100
// Surrogate-safe truncation: plain slice() cuts by UTF-16 code units and
// can split an astral character (emoji, 𝐛𝐨𝐥𝐝 text), leaving a lone high
// surrogate that is invalid JSON downstream (Postgres jsonb rejects it).
const cut = (s: string, n: number): string => {
- const out = s.slice(0, n)
- const last = out.charCodeAt(out.length - 1)
- return last >= 0xd800 && last <= 0xdbff ? out.slice(0, -1) : out
+ let out = ''
+ for (let index = 0; index < s.length && out.length < n; index++) {
+ const code = s.charCodeAt(index)
+ if (code === 0) {
+ out += '\uFFFD'
+ continue
+ }
+ if (code >= 0xd800 && code <= 0xdbff) {
+ const next = s.charCodeAt(index + 1)
+ if (next >= 0xdc00 && next <= 0xdfff) {
+ if (out.length + 2 > n) break
+ out += s[index] + s[index + 1]
+ index++
+ } else {
+ out += '\uFFFD'
+ }
+ continue
+ }
+ out += code >= 0xdc00 && code <= 0xdfff ? '\uFFFD' : s[index]
+ }
+ return out
}
+ // Keep page-controlled text from becoming indistinguishable from the
+ // structural ref token consumed by the native driver. The zero-width break
+ // is model-readable but prevents a literal label such as "[ref=4]" from
+ // invalidating or impersonating the line's real element marker.
+ const quote = (value: string): string => JSON.stringify(value.replace(/\[ref=/g, '[ref\u200B='))
const interactiveSelector = [
'a[href]',
'button',
@@ -67,6 +101,11 @@ export function collectSnapshot(): unknown {
'[role="switch"]',
'[role="option"]',
'[role="slider"]',
+ '[role="treeitem"]',
+ '[role="gridcell"]',
+ '[role="row"]',
+ '[role="listitem"]',
+ '[tabindex]',
'[onclick]',
'[contenteditable="true"]',
'[contenteditable=""]',
@@ -89,18 +128,80 @@ export function collectSnapshot(): unknown {
].join(', ')
const registry: Element[] = []
+ const refLineIndexes: Record = {}
+ const locators: Array<{
+ url: string
+ tag: string
+ role: string
+ name: string
+ attributes: Record
+ ancestor: string
+ context: string
+ }> = []
window.__simAgentElements = registry
const lines: string[] = []
let truncated = false
+ let refCount = 0
+ let textRefCount = 0
+ const textRefCap = 120
+ let visitedNodes = 0
+ const previousElementId = window.__simAgentNextElementId
+ const safePreviousElementId =
+ typeof previousElementId === 'number' &&
+ Number.isSafeInteger(previousElementId) &&
+ previousElementId >= 0
+ ? previousElementId
+ : 0
+ const safeStartingElementId =
+ Number.isSafeInteger(startingElementId) && startingElementId >= 0 ? startingElementId : 0
+ let nextElementId = Math.max(safePreviousElementId, safeStartingElementId)
+
+ const visibility = new WeakMap()
const isVisible = (el: Element): boolean => {
+ const cached = visibility.get(el)
+ if (cached !== undefined) return cached
const rect = el.getBoundingClientRect()
- if (rect.width <= 0 || rect.height <= 0) return false
+ if (rect.width <= 0 || rect.height <= 0) {
+ visibility.set(el, false)
+ return false
+ }
const doc = el.ownerDocument
const win = doc.defaultView
- if (!win) return false
- const style = win.getComputedStyle(el)
- return style.visibility !== 'hidden' && style.display !== 'none'
+ if (!win) {
+ visibility.set(el, false)
+ return false
+ }
+ let visible = true
+ for (let current: Element | null = el; current && visible; ) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ visible = Boolean(
+ style &&
+ style.visibility !== 'hidden' &&
+ style.display !== 'none' &&
+ style.contentVisibility !== 'hidden' &&
+ (!Number.isFinite(opacity) || opacity > 0.01) &&
+ !current.hasAttribute('hidden') &&
+ current.getAttribute('aria-hidden') !== 'true'
+ )
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ }
+ visibility.set(el, visible)
+ return visible
+ }
+
+ const pageUrlFor = (el: Element): string => {
+ try {
+ return el.ownerDocument.defaultView?.location.href || ''
+ } catch {
+ return ''
+ }
}
const isSecretField = (el: Element | null): boolean => {
@@ -150,14 +251,17 @@ export function collectSnapshot(): unknown {
const roleFor = (el: Element): string => {
const explicit = el.getAttribute('role')
- if (explicit) return explicit
- const tag = el.tagName
+ if (explicit) {
+ return cut(explicit.replace(/[^a-zA-Z0-9_-]+/g, '-'), 40) || 'clickable'
+ }
+ const tag = String(el.tagName || '').toUpperCase()
if (tag === 'A') return 'link'
if (tag === 'BUTTON' || tag === 'SUMMARY') return 'button'
if (tag === 'SELECT') return 'combobox'
if (tag === 'TEXTAREA') return 'textbox'
if (tag === 'INPUT') {
const type = (el as HTMLInputElement).type
+ if (type === 'file') return 'file-input'
if (type === 'checkbox') return 'checkbox'
if (type === 'radio') return 'radio'
if (type === 'submit' || type === 'button' || type === 'reset') return 'button'
@@ -169,11 +273,19 @@ export function collectSnapshot(): unknown {
const nameFor = (el: Element): string => {
let name = el.getAttribute('aria-label') || ''
+ if (!name) {
+ const labelledBy = (el.getAttribute('aria-labelledby') || '').trim().split(/\s+/)
+ name = labelledBy
+ .filter(Boolean)
+ .map((id) => el.ownerDocument.getElementById(id)?.textContent || '')
+ .join(' ')
+ }
if (!name) {
const labels = (el as HTMLInputElement).labels
if (labels && labels.length > 0) name = labels[0].innerText || ''
}
if (!name) name = (el as HTMLElement).innerText || ''
+ if (!name) name = el.textContent || ''
if (!name) {
name =
el.getAttribute('placeholder') ||
@@ -182,9 +294,127 @@ export function collectSnapshot(): unknown {
el.getAttribute('name') ||
''
}
+ if (!name) {
+ const labelledDescendant = el.querySelector(
+ '[aria-label], img[alt], [title], svg title, [data-title], [data-emoji-name], [data-short-name], [data-name]'
+ )
+ name =
+ labelledDescendant?.getAttribute('aria-label') ||
+ labelledDescendant?.getAttribute('alt') ||
+ labelledDescendant?.getAttribute('title') ||
+ labelledDescendant?.getAttribute('data-title') ||
+ labelledDescendant?.getAttribute('data-emoji-name') ||
+ labelledDescendant?.getAttribute('data-short-name') ||
+ labelledDescendant?.getAttribute('data-name') ||
+ labelledDescendant?.textContent ||
+ ''
+ }
+ if (!name) {
+ name =
+ el.getAttribute('data-emoji-name') ||
+ el.getAttribute('data-short-name') ||
+ el.getAttribute('data-name') ||
+ el.getAttribute('data-title') ||
+ el.getAttribute('data-qa') ||
+ el.getAttribute('data-testid') ||
+ el.getAttribute('data-test-id') ||
+ ''
+ }
return cut(name.replace(/\s+/g, ' ').trim(), 120)
}
+ const locatorAttributes = (el: Element): Record => {
+ const result: Record = {}
+ for (const attribute of [
+ 'id',
+ 'name',
+ 'type',
+ 'href',
+ 'aria-label',
+ 'aria-labelledby',
+ 'placeholder',
+ 'title',
+ 'data-testid',
+ 'data-test-id',
+ 'data-qa',
+ 'data-key',
+ 'data-index',
+ 'data-emoji-name',
+ 'data-short-name',
+ 'data-name',
+ 'data-title',
+ ]) {
+ const value = el.getAttribute(attribute)
+ if (value) result[attribute] = cut(value.replace(/\s+/g, ' ').trim(), 200)
+ }
+ return result
+ }
+
+ const ancestorSignature = (el: Element): string => {
+ const composedParent = (element: Element): Element | null => {
+ if (element.parentElement) return element.parentElement
+ const root = element.getRootNode()
+ return 'host' in root ? (root.host as Element) : null
+ }
+ let parent = composedParent(el)
+ for (let depth = 0; parent && depth < 5; depth++, parent = composedParent(parent)) {
+ for (const attribute of [
+ 'id',
+ 'aria-label',
+ 'data-testid',
+ 'data-test-id',
+ 'data-qa',
+ 'data-key',
+ ]) {
+ const marker = parent.getAttribute(attribute)
+ if (marker) {
+ return `${parent.tagName.toUpperCase()}:${attribute}=${cut(marker, 120)}`
+ }
+ }
+ }
+ return ''
+ }
+
+ const contextSignature = (el: Element): string => {
+ const ownName = nameFor(el)
+ let current: Element | null = el
+ for (let depth = 0; depth < 4; depth++) {
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ if (!current || ['BODY', 'HTML'].includes(current.tagName.toUpperCase())) break
+ const raw = ((current as HTMLElement).innerText || current.textContent || '')
+ .replace(/\s+/g, ' ')
+ .trim()
+ const rowLike = current.matches(
+ 'li, tr, [role="row"], [role="listitem"], [role="treeitem"], [role="gridcell"]'
+ )
+ if (raw && raw !== ownName && (rowLike || raw.length <= 400)) {
+ return `${current.tagName.toUpperCase()}:${cut(raw, 240)}`
+ }
+ }
+ return ''
+ }
+
+ const registerElement = (el: Element, role: string, name: string): number => {
+ const id = nextElementId++
+ registry[id] = el
+ locators[id] = {
+ url: pageUrlFor(el),
+ tag: el.tagName.toUpperCase(),
+ role,
+ name,
+ attributes: locatorAttributes(el),
+ ancestor: ancestorSignature(el),
+ context: contextSignature(el),
+ }
+ refCount++
+ window.__simAgentNextElementId = nextElementId
+ return id
+ }
+
const push = (line: string): boolean => {
if (lines.length >= lineCap) {
truncated = true
@@ -195,36 +425,61 @@ export function collectSnapshot(): unknown {
}
const emitInteractive = (el: Element, indent: string): void => {
- if (registry.length >= refCap) {
+ if (refCount >= refCap || lines.length >= lineCap) {
truncated = true
return
}
- const id = registry.length
- registry.push(el)
let role = roleFor(el)
+ const tag = String(el.tagName || '').toUpperCase()
+ const name = nameFor(el)
+ const id = registerElement(el, role, name)
const parts: string[] = []
if (isSecretField(el)) {
role = 'password-field'
- } else if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT') {
+ } else if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') {
// Tag comparison so fields inside same-origin iframes report their value
// like any other. Redaction above is realm-safe and runs first, so
// widening this cannot expose a credential field.
const value = (el as HTMLInputElement).value
- if (value && isSensitiveValueField(el)) parts.push('value-withheld')
- else if (value) parts.push(`value="${cut(String(value), 120)}"`)
+ if (tag === 'INPUT' && (el as HTMLInputElement).type === 'file') {
+ parts.push('upload-unsupported')
+ } else if (value && isSensitiveValueField(el)) parts.push('value-withheld')
+ else if (value) parts.push(`value=${quote(cut(String(value), 120))}`)
}
- if (el.tagName === 'A') {
+ if (tag === 'A') {
const href = el.getAttribute('href')
- if (href) parts.push(`href="${cut(href, 200)}"`)
+ if (href) parts.push(`href=${quote(cut(href, 200))}`)
}
if ((el as HTMLInputElement).disabled === true) parts.push('disabled')
+ if (el.getAttribute('aria-disabled') === 'true') parts.push('aria-disabled')
if ((el as HTMLInputElement).checked === true) parts.push('checked')
const suffix = parts.length > 0 ? ` ${parts.join(' ')}` : ''
- push(`${indent}- ${role} "${nameFor(el)}" [ref=${id}]${suffix}`)
+ const lineIndex = lines.length
+ if (push(`${indent}- ${role} ${quote(name)} [ref=${id}]${suffix}`)) {
+ refLineIndexes[id] = lineIndex
+ }
+ }
+
+ const emitTextLeaf = (el: Element, indent: string, renderedLabel?: string): void => {
+ if (refCount >= refCap || textRefCount >= textRefCap || lines.length >= lineCap) {
+ truncated = true
+ return
+ }
+ const text = cut(
+ (renderedLabel || (el as HTMLElement).innerText || el.textContent || nameFor(el) || '')
+ .replace(/\s+/g, ' ')
+ .trim(),
+ 160
+ )
+ if (!text) return
+ const id = registerElement(el, roleFor(el), text)
+ textRefCount++
+ const lineIndex = lines.length
+ if (push(`${indent}- text ${quote(text)} [ref=${id}]`)) refLineIndexes[id] = lineIndex
}
const headingLevel = (el: Element): number | null => {
- const match = /^H([1-6])$/.exec(el.tagName)
+ const match = /^H([1-6])$/.exec(String(el.tagName || '').toUpperCase())
if (match) return Number(match[1])
if (el.getAttribute('role') === 'heading') {
const level = Number(el.getAttribute('aria-level') || '2')
@@ -234,7 +489,8 @@ export function collectSnapshot(): unknown {
}
const landmarkLabel = (el: Element): string => {
- const role = el.getAttribute('role')
+ const rawRole = el.getAttribute('role')
+ const role = rawRole ? cut(rawRole.replace(/[^a-zA-Z0-9_-]+/g, '-'), 40) : ''
const tag = el.tagName.toLowerCase()
const kind =
role ||
@@ -248,40 +504,78 @@ export function collectSnapshot(): unknown {
? 'complementary'
: tag)
const label = cut((el.getAttribute('aria-label') || '').replace(/\s+/g, ' ').trim(), 80)
- return label ? `${kind} "${label}"` : kind
+ return label ? `${kind} ${quote(label)}` : kind
}
- const walk = (root: ParentNode, depth: number): void => {
- if (truncated && registry.length >= refCap) return
+ const pointerBoundary = (el: Element): boolean => {
+ const view = el.ownerDocument.defaultView
+ if (!view || view.getComputedStyle(el).cursor !== 'pointer') return false
+ const root = el.getRootNode()
+ const parent = el.parentElement ?? ('host' in root ? (root.host as Element) : null)
+ return (
+ !parent || parent.ownerDocument.defaultView?.getComputedStyle(parent).cursor !== 'pointer'
+ )
+ }
+
+ const walk = (root: ParentNode, depth: number, suppressTextCoveredBy = ''): void => {
+ if (refCount >= refCap || depth > depthCap) {
+ truncated = true
+ return
+ }
for (const el of Array.from(root.children)) {
- if (registry.length >= refCap && lines.length >= lineCap) return
- const tag = el.tagName
+ visitedNodes++
+ if (refCount >= refCap || visitedNodes > nodeCap) {
+ truncated = true
+ return
+ }
+ const tag = String(el.tagName || '').toUpperCase()
if (tag === 'SCRIPT' || tag === 'STYLE' || tag === 'NOSCRIPT' || tag === 'TEMPLATE') continue
const indent = ' '.repeat(depth)
let childDepth = depth
+ let emittedInteractive = false
+ let interactiveName = ''
+ const visible = isVisible(el)
- if (el.matches(landmarkSelector) && isVisible(el)) {
+ if (el.matches(landmarkSelector) && visible) {
if (!push(`${indent}- ${landmarkLabel(el)}:`)) return
childDepth = depth + 1
} else {
const level = headingLevel(el)
- if (level !== null && isVisible(el)) {
+ if (level !== null && visible) {
const text = cut(((el as HTMLElement).innerText || '').replace(/\s+/g, ' ').trim(), 160)
- if (text) push(`${indent}- heading "${text}" (h${level})`)
- } else if (el.matches(interactiveSelector) && isVisible(el)) {
+ if (text) push(`${indent}- heading ${quote(text)} (h${level})`)
+ } else if (visible && (el.matches(interactiveSelector) || pointerBoundary(el))) {
emitInteractive(el, indent)
+ emittedInteractive = true
+ interactiveName = nameFor(el)
// Interactive containers rarely nest other interactives; still
// recurse so e.g. a clickable card exposes its inner links.
+ } else if (visible) {
+ const visibleElementChild = Array.from(el.children).some(isVisible)
+ const leafLabel = visibleElementChild
+ ? ''
+ : (((el as HTMLElement).innerText || el.textContent || nameFor(el) || '') as string)
+ .replace(/\s+/g, ' ')
+ .trim()
+ if (
+ !visibleElementChild &&
+ leafLabel &&
+ (!suppressTextCoveredBy || !suppressTextCoveredBy.includes(leafLabel))
+ ) {
+ emitTextLeaf(el, indent, leafLabel)
+ }
}
}
+ const coveredText = emittedInteractive ? interactiveName : suppressTextCoveredBy
+
if (tag === 'IFRAME' || tag === 'FRAME') {
try {
const innerDoc = (el as HTMLIFrameElement).contentDocument
if (innerDoc?.body && isVisible(el)) {
if (!push(`${indent}- iframe:`)) return
- walk(innerDoc.body, childDepth + 1)
+ walk(innerDoc.body, childDepth + 1, coveredText)
}
} catch {
// Cross-origin iframe — not readable.
@@ -290,25 +584,238 @@ export function collectSnapshot(): unknown {
}
const shadow = (el as HTMLElement).shadowRoot
- if (shadow) walk(shadow, childDepth)
- walk(el, childDepth)
+ if (shadow) walk(shadow, childDepth, coveredText)
+ walk(el, childDepth, coveredText)
}
}
if (document.body) walk(document.body, 0)
+ /**
+ * React commonly replaces a control's DOM node while preserving its
+ * semantics. Recover only when the old page URL and a strong semantic
+ * fingerprint still identify one candidate; a weak or ambiguous match is a
+ * real stale ref, never permission to click something nearby.
+ */
+ window.__simAgentResolveElement = (id: number) => {
+ const locator = locators[id]
+ if (!locator) return null
+
+ const stableAttributes = [
+ 'id',
+ 'href',
+ 'data-testid',
+ 'data-test-id',
+ 'data-key',
+ 'data-emoji-name',
+ 'data-short-name',
+ ]
+ const connectedIdentityAttributes = [
+ ...stableAttributes,
+ 'data-qa',
+ 'data-index',
+ 'data-name',
+ 'data-title',
+ ]
+ const identityMatches = (candidate: Element, connected = false): boolean => {
+ if (
+ candidate.tagName.toUpperCase() !== locator.tag ||
+ pageUrlFor(candidate) !== locator.url ||
+ roleFor(candidate) !== locator.role
+ ) {
+ return false
+ }
+ if (nameFor(candidate) !== locator.name) return false
+ const candidateAttributes = locatorAttributes(candidate)
+ const attributes = connected ? connectedIdentityAttributes : stableAttributes
+ const genericName =
+ /^(?:more(?: actions?)?|open|menu|options?|edit|delete|view|button|link)$/i.test(
+ locator.name
+ )
+ const stableAttributePresent = attributes.some((attribute) =>
+ Boolean(locator.attributes[attribute])
+ )
+ const hasIdentitySignal =
+ Boolean(locator.name) ||
+ Boolean(locator.ancestor) ||
+ Boolean(locator.context) ||
+ stableAttributePresent
+ if (!hasIdentitySignal) return false
+ if (locator.ancestor && ancestorSignature(candidate) !== locator.ancestor) return false
+ // Full row text is useful to disambiguate a detached replacement and a
+ // generic recycled action button. It is intentionally not a hard check
+ // for every connected control: timestamps and unread badges can update
+ // without changing the control itself, which would recreate Slack's
+ // chronic one-action-old ref behavior.
+ if (
+ locator.context &&
+ (!connected || genericName) &&
+ contextSignature(candidate) !== locator.context
+ ) {
+ return false
+ }
+ if (
+ connected &&
+ genericName &&
+ !locator.ancestor &&
+ !locator.context &&
+ !stableAttributePresent
+ ) {
+ return false
+ }
+ return attributes.every(
+ (attribute) =>
+ !locator.attributes[attribute] ||
+ candidateAttributes[attribute] === locator.attributes[attribute]
+ )
+ }
+
+ // The snapshot-time visibility WeakMap is intentionally not used here.
+ // React apps often keep the old combobox/control connected but collapse it
+ // to zero size while mounting a replacement. Re-check live so a parked
+ // node can fall through to the same strict, unique recovery used for a
+ // detached node.
+ const isCurrentlyVisible = (candidate: Element): boolean => {
+ const rect = candidate.getBoundingClientRect()
+ if (rect.width <= 0 || rect.height <= 0) return false
+ for (let current: Element | null = candidate; current; ) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ if (
+ !style ||
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.contentVisibility === 'hidden' ||
+ (Number.isFinite(opacity) && opacity <= 0.01) ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return false
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ }
+ return true
+ }
+
+ const current = registry[id]
+ if (current?.isConnected) {
+ if (!identityMatches(current, true)) return null
+ if (isCurrentlyVisible(current)) return { element: current, recovered: false }
+ }
+
+ const reachable: Element[] = []
+ let candidateCount = 0
+ const collect = (root: ParentNode, depth = 0): void => {
+ if (depth > depthCap || candidateCount >= nodeCap) return
+ for (const element of Array.from(root.children)) {
+ candidateCount++
+ if (candidateCount > nodeCap) return
+ reachable.push(element)
+ const shadow = (element as HTMLElement).shadowRoot
+ if (shadow) collect(shadow, depth + 1)
+ const tag = String(element.tagName || '').toUpperCase()
+ if (tag === 'IFRAME' || tag === 'FRAME') {
+ try {
+ const inner = (element as HTMLIFrameElement).contentDocument
+ if (inner?.body) collect(inner.body, depth + 1)
+ } catch {
+ // Cross-origin frame — not searchable.
+ }
+ }
+ collect(element, depth + 1)
+ }
+ }
+ if (document.body) collect(document.body)
+
+ const scored = reachable
+ .filter((candidate) => identityMatches(candidate) && isCurrentlyVisible(candidate))
+ .map((candidate) => {
+ const candidateAttributes = locatorAttributes(candidate)
+ const logicalAttributeMatch = [
+ 'href',
+ 'data-key',
+ 'data-emoji-name',
+ 'data-short-name',
+ ].some(
+ (attribute) =>
+ Boolean(locator.attributes[attribute]) &&
+ candidateAttributes[attribute] === locator.attributes[attribute]
+ )
+ const ancestorMatch = Boolean(
+ locator.ancestor && ancestorSignature(candidate) === locator.ancestor
+ )
+ const textContextMatch = Boolean(
+ locator.context && contextSignature(candidate) === locator.context
+ )
+ const genericAttributeMatch = ['id', 'data-testid', 'data-test-id'].some(
+ (attribute) =>
+ Boolean(locator.attributes[attribute]) &&
+ candidateAttributes[attribute] === locator.attributes[attribute]
+ )
+ // Exact role+label alone is unsafe for generic controls such as Close:
+ // after one panel disappears, another panel's Close can be the sole
+ // candidate. Require a stable key or the same structural context.
+ if (locator.ancestor && !ancestorMatch) return { candidate, score: -1 }
+ if (locator.context && !textContextMatch) return { candidate, score: -1 }
+ if (
+ !logicalAttributeMatch &&
+ !genericAttributeMatch &&
+ !ancestorMatch &&
+ !textContextMatch
+ ) {
+ return { candidate, score: -1 }
+ }
+ let score = 65
+ for (const [attribute, expected] of Object.entries(locator.attributes)) {
+ if (candidateAttributes[attribute] !== expected) continue
+ if (attribute === 'id') score += 100
+ else if (attribute.startsWith('data-')) score += 45
+ else if (attribute === 'name' || attribute === 'href' || attribute === 'aria-label') {
+ score += 25
+ } else score += 8
+ }
+ if (ancestorMatch) score += 20
+ if (textContextMatch) score += 20
+ return { candidate, score }
+ })
+ .filter((entry) => entry.score >= 45)
+ .sort((a, b) => b.score - a.score)
+
+ if (scored.length === 0) return null
+ const bestScore = scored[0].score
+ const best = scored.filter((entry) => entry.score === bestScore)
+ const chosen = best.length === 1 ? best[0] : undefined
+ if (!chosen) return null
+ registry[id] = chosen.candidate
+ return { element: chosen.candidate, recovered: true }
+ }
+
return {
- url: window.location.href,
- title: document.title,
+ url: cut(window.location.href, 4096),
+ title: cut(document.title, 500),
outline: lines.join('\n'),
truncated,
scrollY: Math.round(window.scrollY),
pageHeight: Math.round(document.documentElement.scrollHeight),
+ viewportWidth: window.innerWidth,
viewportHeight: window.innerHeight,
+ refIds: Object.keys(locators).map(Number),
+ refLineIndexes,
+ nextElementId,
}
}
-export function clickElement(id: number): unknown {
+export function clickElement(
+ id: number,
+ dispatchSynthetic = true,
+ focusForKeyboard = false,
+ allowDisabled = false
+): unknown {
const isSecretField = (node: Element | null): boolean => {
if (!node || String(node.tagName || '').toUpperCase() !== 'INPUT') return false
if (String((node as HTMLInputElement).type || '').toLowerCase() === 'password') return true
@@ -318,40 +825,372 @@ export function clickElement(id: number): unknown {
.some((token) => token === 'current-password' || token === 'new-password')
}
- const el = (window.__simAgentElements || [])[id]
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
if (!el || !el.isConnected) return { error: 'stale' }
+ const isDisabled = (node: Element | null): boolean =>
+ Boolean(
+ node &&
+ ((node as Element & { disabled?: boolean }).disabled === true ||
+ node.getAttribute('aria-disabled') === 'true')
+ )
// Clicking focuses, and a focused credential field is the one state in
// which subsequent keystrokes would land in a password. Refusing the click
// keeps that state unreachable rather than relying on every later keyboard
// path to re-check.
if (isSecretField(el)) return { error: 'password' }
- el.scrollIntoView({ block: 'center', inline: 'center' })
- const rect = el.getBoundingClientRect()
+ if (!allowDisabled && isDisabled(el)) return { error: 'disabled' }
+ if (
+ String(el.tagName || '').toUpperCase() === 'INPUT' &&
+ String((el as HTMLInputElement).type || '').toLowerCase() === 'file'
+ ) {
+ return { error: 'file-input' }
+ }
+ if (String(el.tagName || '').toUpperCase() === 'LABEL') {
+ const control = (el as HTMLLabelElement).control
+ if (isSecretField(control)) return { error: 'password' }
+ if (!allowDisabled && isDisabled(control)) return { error: 'disabled' }
+ if (
+ control &&
+ String(control.tagName || '').toUpperCase() === 'INPUT' &&
+ String((control as HTMLInputElement).type || '').toLowerCase() === 'file'
+ ) {
+ return { error: 'file-input' }
+ }
+ }
+ el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' })
+
+ const view = el.ownerDocument.defaultView
+ if (!view) return { error: 'stale' }
+ for (let current: Element | null = el; current; ) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ if (
+ !style ||
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.contentVisibility === 'hidden' ||
+ (Number.isFinite(opacity) && opacity <= 0.01) ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return { error: 'not-visible' }
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ if ('host' in root) current = root.host as Element
+ else {
+ const frame: Element | null = current.ownerDocument.defaultView?.frameElement ?? null
+ current = frame ? (frame as Element) : null
+ }
+ }
+ }
+ const rawRects = Array.from(el.getClientRects())
+ if (rawRects.length === 0) rawRects.push(el.getBoundingClientRect())
+ const rects = rawRects
+ .map((rect) => ({
+ left: Math.max(0, rect.left),
+ top: Math.max(0, rect.top),
+ right: Math.min(view.innerWidth, rect.right),
+ bottom: Math.min(view.innerHeight, rect.bottom),
+ }))
+ .filter(
+ (rect) =>
+ Number.isFinite(rect.left) &&
+ Number.isFinite(rect.top) &&
+ Number.isFinite(rect.right) &&
+ Number.isFinite(rect.bottom) &&
+ rect.right - rect.left > 1 &&
+ rect.bottom - rect.top > 1
+ )
+ if (rects.length === 0) return { error: 'not-visible' }
+
+ const composedParent = (node: Element): Element | null => {
+ if (node.parentElement) return node.parentElement
+ const root = node.getRootNode()
+ return 'host' in root ? (root.host as Element) : null
+ }
+ const isIndependentInteractive = (node: Element): boolean => {
+ const role = node.getAttribute('role') || ''
+ const tag = String(node.tagName || '').toUpperCase()
+ return (
+ tag === 'A' ||
+ tag === 'BUTTON' ||
+ tag === 'INPUT' ||
+ tag === 'SELECT' ||
+ tag === 'TEXTAREA' ||
+ tag === 'SUMMARY' ||
+ node.hasAttribute('onclick') ||
+ (node.hasAttribute('tabindex') && Number(node.getAttribute('tabindex')) >= 0) ||
+ (node as HTMLElement).isContentEditable ||
+ [
+ 'button',
+ 'link',
+ 'textbox',
+ 'searchbox',
+ 'checkbox',
+ 'radio',
+ 'combobox',
+ 'menuitem',
+ 'tab',
+ 'switch',
+ 'option',
+ ].includes(role)
+ )
+ }
+ const hitBelongsToTarget = (hit: Element | null): boolean => {
+ if (
+ hit?.contains(el) &&
+ el.ownerDocument.defaultView?.getComputedStyle(el).pointerEvents === 'none'
+ ) {
+ return true
+ }
+ for (let current = hit; current; current = composedParent(current)) {
+ if (current === el) return true
+ // A card/row may contain its own link or button. Clicking that nested
+ // control is a different action even though it is technically a
+ // descendant of the requested ref.
+ if (isIndependentInteractive(current)) return false
+ }
+ return false
+ }
+ const elementAt = (x: number, y: number): Element | null => {
+ const root = el.getRootNode() as ParentNode & {
+ elementFromPoint?: (clientX: number, clientY: number) => Element | null
+ }
+ if (typeof root.elementFromPoint === 'function') return root.elementFromPoint(x, y)
+ if (typeof el.ownerDocument.elementFromPoint === 'function') {
+ return el.ownerDocument.elementFromPoint(x, y)
+ }
+ // DOM-only test environments do not implement hit testing; real Chromium
+ // always takes one of the branches above.
+ return el
+ }
+
+ let clientX = 0
+ let clientY = 0
+ let blocker: Element | null = null
+ const blockedHits: Array = []
+ let foundPoint = false
+ const blockerLabel = (element: Element | null): string =>
+ (
+ element?.getAttribute('aria-label') ||
+ (element as HTMLElement | null)?.innerText ||
+ element?.textContent ||
+ element?.tagName ||
+ 'another element'
+ )
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 120)
+ .replace(/[\uD800-\uDBFF]$/, '')
+ const fractions = [0.5, 0.2, 0.8]
+ for (const rect of rects) {
+ for (const xFraction of fractions) {
+ for (const yFraction of fractions) {
+ const x = rect.left + (rect.right - rect.left) * xFraction
+ const y = rect.top + (rect.bottom - rect.top) * yFraction
+ const hit = elementAt(x, y)
+ if (hitBelongsToTarget(hit)) {
+ clientX = x
+ clientY = y
+ foundPoint = true
+ break
+ }
+ blocker ??= hit
+ blockedHits.push(hit)
+ }
+ if (foundPoint) break
+ }
+ if (foundPoint) break
+ }
+ if (!foundPoint) {
+ const suggestionsCoverFocusedEditable = (): boolean => {
+ const candidates: HTMLElement[] = []
+ const addCandidate = (candidate: Element): void => {
+ const candidateTag = String(candidate.tagName || '').toUpperCase()
+ const inputType =
+ candidateTag === 'INPUT'
+ ? String((candidate as HTMLInputElement).type || 'text').toLowerCase()
+ : ''
+ if (
+ candidateTag === 'TEXTAREA' ||
+ (candidateTag === 'INPUT' &&
+ ['text', 'search', 'email', 'url', 'tel', 'number'].includes(inputType)) ||
+ (candidate as HTMLElement).isContentEditable
+ ) {
+ candidates.push(candidate as HTMLElement)
+ }
+ }
+ addCandidate(el)
+ for (const candidate of Array.from(
+ el.querySelectorAll(
+ 'input, textarea, [contenteditable="true"], [contenteditable=""]'
+ )
+ )) {
+ addCandidate(candidate)
+ }
+ const editables = Array.from(new Set(candidates))
+ if (editables.length !== 1 || !blocker || blockedHits.length === 0) return false
+ const editable = editables[0]
+ if (editable.ownerDocument.activeElement !== editable) return false
+ let owner: Element | null = editable
+ for (let depth = 0; owner && depth < 10; depth++) {
+ if (owner.getAttribute('role') === 'combobox') break
+ owner = composedParent(owner)
+ }
+ if (!owner || owner.getAttribute('aria-expanded') !== 'true') return false
+ const ids = new Set()
+ for (let current: Element | null = editable; current; current = composedParent(current)) {
+ for (const attribute of ['aria-controls', 'aria-owns']) {
+ for (const token of (current.getAttribute(attribute) || '').trim().split(/\s+/)) {
+ if (token) ids.add(token)
+ }
+ }
+ if (current === owner) break
+ }
+ const scopes = Array.from(
+ new Set([owner.getRootNode() as ParentNode, editable.ownerDocument])
+ )
+ const controlledPopups: Element[] = []
+ for (const idRef of ids) {
+ const matches = new Set()
+ for (const scope of scopes) {
+ let visited = 0
+ for (const candidate of Array.from(scope.querySelectorAll('[id]'))) {
+ if (++visited > 12_000) break
+ if (candidate.id === idRef) matches.add(candidate)
+ }
+ }
+ if (matches.size !== 1) continue
+ const popup = Array.from(matches)[0]
+ if (
+ ['listbox', 'tree', 'grid'].includes(popup.getAttribute('role') || '') &&
+ popup.getAttribute('aria-modal') !== 'true'
+ ) {
+ controlledPopups.push(popup)
+ }
+ }
+ if (controlledPopups.length === 0) return false
+ const coveringPopups = new Set()
+ for (const hit of blockedHits) {
+ if (!hit) return false
+ const popup = controlledPopups.find((candidate) => candidate.contains(hit))
+ if (!popup) return false
+ coveringPopups.add(popup)
+ }
+ return coveringPopups.size === 1
+ }
+ if (suggestionsCoverFocusedEditable()) {
+ return { error: 'suggestions-open', blocker: blockerLabel(blocker) }
+ }
+ return { error: 'obstructed', blocker: blockerLabel(blocker) }
+ }
+
+ let pageX = clientX
+ let pageY = clientY
+ let ownerView: Window | null = el.ownerDocument.defaultView
+ let frameDepth = 0
+ while (ownerView && ownerView !== window) {
+ const frame: Element | null = ownerView.frameElement
+ if (!frame) break
+ const frameRect = frame.getBoundingClientRect()
+ const frameElement = frame as HTMLElement
+ const scaleX = frameElement.offsetWidth > 0 ? frameRect.width / frameElement.offsetWidth : 1
+ const scaleY = frameElement.offsetHeight > 0 ? frameRect.height / frameElement.offsetHeight : 1
+ pageX = frameRect.left + (pageX + frameElement.clientLeft) * scaleX
+ pageY = frameRect.top + (pageY + frameElement.clientTop) * scaleY
+ const parentRoot = frame.getRootNode() as ParentNode & {
+ elementFromPoint?: (clientX: number, clientY: number) => Element | null
+ }
+ const parentDocument: Document = frame.ownerDocument
+ const parentElementAt: ((clientX: number, clientY: number) => Element | null) | null =
+ typeof parentRoot.elementFromPoint === 'function'
+ ? parentRoot.elementFromPoint.bind(parentRoot)
+ : typeof parentDocument.elementFromPoint === 'function'
+ ? parentDocument.elementFromPoint.bind(parentDocument)
+ : null
+ if (parentElementAt) {
+ const parentHit: Element | null = parentElementAt(pageX, pageY)
+ if (parentHit !== frame) {
+ return { error: 'obstructed', blocker: blockerLabel(parentHit) }
+ }
+ }
+ ownerView = frame.ownerDocument.defaultView
+ frameDepth++
+ }
const opts = {
bubbles: true,
cancelable: true,
composed: true,
- clientX: rect.x + rect.width / 2,
- clientY: rect.y + rect.height / 2,
+ clientX,
+ clientY,
button: 0,
}
// Duck-typed rather than `instanceof HTMLElement`: an element reached
// through a same-origin iframe belongs to that frame's realm, so the check
// is false there and the click would skip focus entirely.
const html = el as HTMLElement
- el.dispatchEvent(new PointerEvent('pointerdown', opts))
- el.dispatchEvent(new MouseEvent('mousedown', opts))
- if (typeof html.focus === 'function') html.focus()
- el.dispatchEvent(new PointerEvent('pointerup', opts))
- el.dispatchEvent(new MouseEvent('mouseup', opts))
- if (typeof html.click === 'function') html.click()
- else el.dispatchEvent(new MouseEvent('click', opts))
+ const role = el.getAttribute('role') || ''
+ const tag = String(el.tagName || '').toUpperCase()
+ const inputType =
+ tag === 'INPUT' ? String((el as HTMLInputElement).type || 'text').toLowerCase() : ''
+ const activationKey =
+ tag === 'A' ||
+ tag === 'BUTTON' ||
+ tag === 'SUMMARY' ||
+ role === 'button' ||
+ role === 'link' ||
+ role === 'menuitem' ||
+ role === 'tab'
+ ? 'Enter'
+ : (tag === 'INPUT' &&
+ ['button', 'submit', 'reset', 'image', 'checkbox', 'radio'].includes(inputType)) ||
+ role === 'checkbox' ||
+ role === 'radio' ||
+ role === 'switch' ||
+ role === 'option'
+ ? 'Space'
+ : undefined
+ const editable =
+ tag === 'INPUT' ||
+ tag === 'TEXTAREA' ||
+ tag === 'SELECT' ||
+ html.isContentEditable ||
+ role === 'textbox' ||
+ role === 'searchbox' ||
+ role === 'combobox'
+ if (focusForKeyboard && typeof html.focus === 'function') html.focus()
+ if (dispatchSynthetic) {
+ el.dispatchEvent(new PointerEvent('pointerdown', opts))
+ el.dispatchEvent(new MouseEvent('mousedown', opts))
+ if (typeof html.focus === 'function') html.focus()
+ el.dispatchEvent(new PointerEvent('pointerup', opts))
+ el.dispatchEvent(new MouseEvent('mouseup', opts))
+ if (typeof html.click === 'function') html.click()
+ else el.dispatchEvent(new MouseEvent('click', opts))
+ }
const label = (el.getAttribute('aria-label') || (el as HTMLElement).innerText || '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 80)
// Drop a trailing lone high surrogate the slice may have created.
- return { clicked: true, element: label.replace(/[\uD800-\uDBFF]$/, '') }
+ return {
+ dispatched: dispatchSynthetic,
+ element: label.replace(/[\uD800-\uDBFF]$/, ''),
+ x: pageX,
+ y: pageY,
+ clientX,
+ clientY,
+ activationKey,
+ editable,
+ frameDepth,
+ focusSucceeded: focusForKeyboard && el.ownerDocument.activeElement === el,
+ refRecovered: resolved?.recovered === true,
+ }
}
/**
@@ -360,7 +1199,7 @@ export function clickElement(id: number): unknown {
* what's there — including inside code editors (CodeMirror/Monaco), whose
* models sync from the DOM selection / native input pipeline.
*/
-export function focusElementForTyping(id: number): unknown {
+export function focusElementForTyping(id: number, moveFocus = true): unknown {
const isSecretField = (node: Element | null): boolean => {
if (!node || String(node.tagName || '').toUpperCase() !== 'INPUT') return false
if (String((node as HTMLInputElement).type || '').toLowerCase() === 'password') return true
@@ -370,43 +1209,271 @@ export function focusElementForTyping(id: number): unknown {
.some((token) => token === 'current-password' || token === 'new-password')
}
- const el = (window.__simAgentElements || [])[id]
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
if (!el || !el.isConnected) return { error: 'stale' }
- el.scrollIntoView({ block: 'center' })
- if (isSecretField(el)) {
- return { error: 'password' }
+ const isWritableTextField = (
+ field: HTMLInputElement | HTMLTextAreaElement
+ ): 'writable' | 'disabled' | 'readonly' | 'not-editable' => {
+ if (field.disabled || field.getAttribute('aria-disabled') === 'true') return 'disabled'
+ if (field.readOnly || field.getAttribute('aria-readonly') === 'true') return 'readonly'
+ if (String(field.tagName || '').toUpperCase() === 'TEXTAREA') return 'writable'
+ const type = String((field as HTMLInputElement).type || 'text').toLowerCase()
+ return ['text', 'search', 'email', 'url', 'tel', 'number'].includes(type)
+ ? 'writable'
+ : 'not-editable'
}
- // Tag comparisons, not `instanceof`: element wrappers are realm-bound, so an
- // input inside a same-origin iframe — a framed login form, a TinyMCE body —
- // fails every `instanceof` against the top frame's constructors and would be
- // reported back as "not a text input".
- const tag = el.tagName
- if (tag === 'INPUT' || tag === 'TEXTAREA') {
- const field = el as HTMLInputElement | HTMLTextAreaElement
- field.focus()
- field.select()
- return { focused: true, kind: tag === 'INPUT' ? 'input' : 'textarea' }
+ const tagFor = (node: Element): string => String(node.tagName || '').toUpperCase()
+ const potentialEditables: HTMLElement[] = []
+ const addEditable = (node: Element): void => {
+ const tag = tagFor(node)
+ const inputType =
+ tag === 'INPUT' ? String((node as HTMLInputElement).type || 'text').toLowerCase() : ''
+ if (
+ tag === 'TEXTAREA' ||
+ (tag === 'INPUT' &&
+ ['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType)) ||
+ (node as HTMLElement).isContentEditable
+ ) {
+ potentialEditables.push(node as HTMLElement)
+ }
}
+ addEditable(el)
+ for (const candidate of Array.from(
+ el.querySelectorAll(
+ 'input, textarea, [contenteditable="true"], [contenteditable=""]'
+ )
+ )) {
+ addEditable(candidate)
+ }
+ const editables = Array.from(new Set(potentialEditables))
+ if (editables.length === 0) return { error: 'not-editable' }
+ if (editables.length > 1) return { error: 'ambiguous-editable' }
+ const editable = editables[0]
+ const editableTag = tagFor(editable)
+
+ if (isSecretField(editable)) return { error: 'password' }
+ if (editableTag === 'INPUT' || editableTag === 'TEXTAREA') {
+ const writable = isWritableTextField(editable as HTMLInputElement | HTMLTextAreaElement)
+ if (writable !== 'writable') return { error: writable }
+ } else {
+ if (editable.getAttribute('aria-disabled') === 'true') return { error: 'disabled' }
+ if (editable.getAttribute('aria-readonly') === 'true') return { error: 'readonly' }
+ }
+
+ editable.scrollIntoView({ block: 'nearest', inline: 'nearest', behavior: 'instant' })
- // Editors often register a wrapper as the interactive element while the
- // actual editable surface is a descendant.
- const editable = (el as HTMLElement).isContentEditable
- ? (el as HTMLElement)
- : el.querySelector('[contenteditable="true"], [contenteditable=""]')
- if (editable) {
+ const composedParent = (node: Element): Element | null => {
+ if (node.parentElement) return node.parentElement
+ const root = node.getRootNode()
+ return 'host' in root ? (root.host as Element) : null
+ }
+ const rawRects = Array.from(editable.getClientRects())
+ if (rawRects.length === 0) rawRects.push(editable.getBoundingClientRect())
+ const view = editable.ownerDocument.defaultView
+ if (!view) return { error: 'stale' }
+ const rects = rawRects
+ .map((rect) => ({
+ left: Math.max(0, rect.left),
+ top: Math.max(0, rect.top),
+ right: Math.min(view.innerWidth, rect.right),
+ bottom: Math.min(view.innerHeight, rect.bottom),
+ }))
+ .filter(
+ (rect) =>
+ Number.isFinite(rect.left) &&
+ Number.isFinite(rect.top) &&
+ Number.isFinite(rect.right) &&
+ Number.isFinite(rect.bottom) &&
+ rect.right - rect.left > 1 &&
+ rect.bottom - rect.top > 1
+ )
+ if (rects.length === 0) return { error: 'not-visible' }
+ for (let current: Element | null = editable; current; current = composedParent(current)) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ if (
+ !style ||
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.contentVisibility === 'hidden' ||
+ (Number.isFinite(opacity) && opacity <= 0.01) ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return { error: 'not-visible' }
+ }
+ }
+
+ if (moveFocus) {
editable.focus()
- const selection = editable.ownerDocument.defaultView?.getSelection()
- if (selection) {
- const range = editable.ownerDocument.createRange()
- range.selectNodeContents(editable)
- selection.removeAllRanges()
- selection.addRange(range)
+ if (editableTag === 'INPUT' || editableTag === 'TEXTAREA') {
+ try {
+ ;(editable as HTMLInputElement | HTMLTextAreaElement).select()
+ } catch {
+ // Trusted Mod+A in the driver still covers field types that reject select().
+ }
+ } else {
+ const selection = editable.ownerDocument.defaultView?.getSelection()
+ if (selection) {
+ const range = editable.ownerDocument.createRange()
+ range.selectNodeContents(editable)
+ selection.removeAllRanges()
+ selection.addRange(range)
+ }
+ }
+ }
+
+ const deepestActiveElement = (): HTMLElement | null => {
+ let active = editable.ownerDocument.activeElement as HTMLElement | null
+ for (let depth = 0; active && depth < 10; depth++) {
+ if (active.shadowRoot?.activeElement) {
+ active = active.shadowRoot.activeElement as HTMLElement
+ continue
+ }
+ const activeTag = tagFor(active)
+ if (activeTag === 'IFRAME' || activeTag === 'FRAME') {
+ try {
+ const inner = (active as HTMLIFrameElement).contentDocument
+ if (inner?.activeElement && inner.activeElement !== inner.body) {
+ active = inner.activeElement as HTMLElement
+ continue
+ }
+ } catch {
+ return active
+ }
+ }
+ break
}
- return { focused: true, kind: 'contenteditable' }
+ return active
+ }
+ const active = deepestActiveElement()
+ if (isSecretField(active)) return { error: 'password' }
+ if (!active || (active !== editable && !editable.contains(active))) return { error: 'different' }
+
+ const root = editable.getRootNode() as ParentNode & {
+ elementFromPoint?: (x: number, y: number) => Element | null
+ }
+ const elementAt =
+ typeof root.elementFromPoint === 'function'
+ ? root.elementFromPoint.bind(root)
+ : typeof editable.ownerDocument.elementFromPoint === 'function'
+ ? editable.ownerDocument.elementFromPoint.bind(editable.ownerDocument)
+ : null
+
+ let comboboxOwner: Element | null = editable
+ for (let depth = 0; comboboxOwner && depth < 10; depth++) {
+ if (comboboxOwner.getAttribute('role') === 'combobox') break
+ comboboxOwner = composedParent(comboboxOwner)
+ }
+ const controlledPopups: Element[] = []
+ if (comboboxOwner?.getAttribute('aria-expanded') === 'true') {
+ const idRefs = new Set()
+ for (let current: Element | null = editable; current; current = composedParent(current)) {
+ for (const attribute of ['aria-controls', 'aria-owns']) {
+ for (const token of (current.getAttribute(attribute) || '').trim().split(/\s+/)) {
+ if (token) idRefs.add(token)
+ }
+ }
+ if (current === comboboxOwner) break
+ }
+ const scopes = Array.from(
+ new Set([comboboxOwner.getRootNode() as ParentNode, editable.ownerDocument])
+ )
+ for (const idRef of idRefs) {
+ const matches = new Set()
+ for (const scope of scopes) {
+ let visited = 0
+ for (const candidate of Array.from(scope.querySelectorAll('[id]'))) {
+ if (++visited > 12_000) break
+ if (candidate.id === idRef) matches.add(candidate)
+ }
+ }
+ if (matches.size !== 1) continue
+ const popup = Array.from(matches)[0]
+ if (
+ ['listbox', 'tree', 'grid'].includes(popup.getAttribute('role') || '') &&
+ popup.getAttribute('aria-modal') !== 'true'
+ ) {
+ controlledPopups.push(popup)
+ }
+ }
+ }
+
+ const blockerLabel = (element: Element | null): string =>
+ (
+ element?.getAttribute('aria-label') ||
+ (element as HTMLElement | null)?.innerText ||
+ element?.textContent ||
+ element?.tagName ||
+ 'another element'
+ )
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 120)
+ .replace(/[\uD800-\uDBFF]$/, '')
+ const fractions = [0.5, 0.2, 0.8]
+ let chosenPoint: { x: number; y: number } | null = null
+ let firstBlocker: Element | null = null
+ const blockedPoints: Array<{ x: number; y: number; hit: Element | null }> = []
+ for (const rect of rects) {
+ for (const xFraction of fractions) {
+ for (const yFraction of fractions) {
+ const x = rect.left + (rect.right - rect.left) * xFraction
+ const y = rect.top + (rect.bottom - rect.top) * yFraction
+ const hit = elementAt ? elementAt(x, y) : editable
+ if (hit && (hit === editable || editable.contains(hit))) {
+ chosenPoint = { x, y }
+ break
+ }
+ firstBlocker ??= hit
+ blockedPoints.push({ x, y, hit })
+ }
+ if (chosenPoint) break
+ }
+ if (chosenPoint) break
+ }
+
+ let coveredByRelatedPopup = false
+ if (!chosenPoint && blockedPoints.length > 0) {
+ const coveringPopups = new Set()
+ let allRelated = controlledPopups.length > 0
+ for (const point of blockedPoints) {
+ const popup = point.hit
+ ? controlledPopups.find((candidate) => candidate.contains(point.hit))
+ : undefined
+ if (!popup) {
+ allRelated = false
+ break
+ }
+ coveringPopups.add(popup)
+ }
+ if (allRelated && coveringPopups.size === 1) {
+ chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y }
+ coveredByRelatedPopup = true
+ }
+ }
+ if (!chosenPoint) {
+ return { error: 'obstructed', blocker: blockerLabel(firstBlocker) }
+ }
+
+ return {
+ focused: true,
+ kind:
+ editableTag === 'INPUT'
+ ? 'input'
+ : editableTag === 'TEXTAREA'
+ ? 'textarea'
+ : 'contenteditable',
+ x: chosenPoint.x,
+ y: chosenPoint.y,
+ coveredByRelatedPopup,
+ refRecovered: resolved?.recovered === true,
}
- return { error: 'not-editable' }
}
/**
@@ -533,7 +1600,7 @@ export function readActiveElementState(): unknown {
* Escape are not.
* - `safe` — anything we can see and that is not a credential field.
*/
-export function activeElementSecrecy(): string {
+export function activeElementSecrecy(elementId?: number): string {
const isSecretField = (node: Element | null): boolean => {
if (!node || String(node.tagName || '').toUpperCase() !== 'INPUT') return false
if (String((node as HTMLInputElement).type || '').toLowerCase() === 'password') return true
@@ -543,6 +1610,11 @@ export function activeElementSecrecy(): string {
.some((token) => token === 'current-password' || token === 'new-password')
}
+ const resolver = window.__simAgentResolveElement
+ const resolved = typeof elementId === 'number' ? resolver?.(elementId) : undefined
+ if (typeof elementId === 'number' && (!resolver || !resolved?.element?.isConnected))
+ return 'stale'
+
let active = document.activeElement as HTMLElement | null
for (let depth = 0; active && depth < 10; depth++) {
if (isSecretField(active)) return 'secret'
@@ -617,7 +1689,39 @@ export function activeElementSecrecy(): string {
}
break
}
- return isSecretField(active) ? 'secret' : 'safe'
+ if (isSecretField(active)) return 'secret'
+ if (typeof elementId === 'number') {
+ const expected = resolved?.element ?? null
+ const ownsVisibleSurface = (focused: HTMLElement): boolean => {
+ const rect = focused.getBoundingClientRect()
+ if (rect.width <= 0 || rect.height <= 0) return false
+ const root = focused.getRootNode() as ParentNode & {
+ elementFromPoint?: (x: number, y: number) => Element | null
+ }
+ const elementAt =
+ typeof root.elementFromPoint === 'function'
+ ? root.elementFromPoint.bind(root)
+ : typeof focused.ownerDocument.elementFromPoint === 'function'
+ ? focused.ownerDocument.elementFromPoint.bind(focused.ownerDocument)
+ : null
+ if (!elementAt) return true
+ const hit = elementAt(rect.left + rect.width / 2, rect.top + rect.height / 2)
+ return Boolean(hit && (hit === focused || focused.contains(hit)))
+ }
+ let current: Element | null = active
+ while (current) {
+ if (current === expected) {
+ return active && ownsVisibleSurface(active) ? 'safe' : 'different'
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ }
+ return 'different'
+ }
+ return 'safe'
}
export function typeIntoElement(id: number, text: string, submit: boolean): unknown {
@@ -630,22 +1734,65 @@ export function typeIntoElement(id: number, text: string, submit: boolean): unkn
.some((token) => token === 'current-password' || token === 'new-password')
}
- const el = (window.__simAgentElements || [])[id]
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
if (!el || !el.isConnected) return { error: 'stale' }
- el.scrollIntoView({ block: 'center' })
- if (isSecretField(el)) {
- return { error: 'password' }
+ const isWritableTextField = (
+ field: HTMLInputElement | HTMLTextAreaElement
+ ): 'writable' | 'disabled' | 'readonly' | 'not-editable' => {
+ if (field.disabled || field.getAttribute('aria-disabled') === 'true') return 'disabled'
+ if (field.readOnly || field.getAttribute('aria-readonly') === 'true') return 'readonly'
+ if (String(field.tagName || '').toUpperCase() === 'TEXTAREA') return 'writable'
+ const type = String((field as HTMLInputElement).type || 'text').toLowerCase()
+ return ['text', 'search', 'email', 'url', 'tel', 'number'].includes(type)
+ ? 'writable'
+ : 'not-editable'
+ }
+
+ const potentialEditables: HTMLElement[] = []
+ const addEditable = (node: Element): void => {
+ const candidateTag = String(node.tagName || '').toUpperCase()
+ const inputType =
+ candidateTag === 'INPUT'
+ ? String((node as HTMLInputElement).type || 'text').toLowerCase()
+ : ''
+ if (
+ candidateTag === 'TEXTAREA' ||
+ (candidateTag === 'INPUT' &&
+ ['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType)) ||
+ (node as HTMLElement).isContentEditable
+ ) {
+ potentialEditables.push(node as HTMLElement)
+ }
}
+ addEditable(el)
+ for (const candidate of Array.from(
+ el.querySelectorAll(
+ 'input, textarea, [contenteditable="true"], [contenteditable=""]'
+ )
+ )) {
+ addEditable(candidate)
+ }
+ const editables = Array.from(new Set(potentialEditables))
+ if (editables.length === 0) return { error: 'not-editable' }
+ if (editables.length > 1) return { error: 'ambiguous-editable' }
+ const editable = editables[0]
+ const tag = String(editable.tagName || '').toUpperCase()
+ editable.scrollIntoView({ block: 'nearest', inline: 'nearest', behavior: 'instant' })
+ if (isSecretField(editable)) return { error: 'password' }
- const tag = el.tagName
+ let submissionTarget: HTMLElement = editable
if (tag === 'INPUT' || tag === 'TEXTAREA') {
- const field = el as HTMLInputElement | HTMLTextAreaElement
+ const field = editable as HTMLInputElement | HTMLTextAreaElement
+ const writable = isWritableTextField(field)
+ if (writable !== 'writable') return { error: writable }
field.focus()
// The native setter must come from the element's OWN realm. A same-origin
// iframe has its own constructors, and calling the top frame's setter on
// one of its nodes throws "Illegal invocation".
- const view = el.ownerDocument.defaultView ?? window
+ const view = editable.ownerDocument.defaultView ?? window
const proto =
tag === 'INPUT' ? view.HTMLInputElement.prototype : view.HTMLTextAreaElement.prototype
const descriptor = Object.getOwnPropertyDescriptor(proto, 'value')
@@ -653,15 +1800,15 @@ export function typeIntoElement(id: number, text: string, submit: boolean): unkn
else field.value = text
field.dispatchEvent(new Event('input', { bubbles: true }))
field.dispatchEvent(new Event('change', { bubbles: true }))
- } else if ((el as HTMLElement).isContentEditable) {
- const editable = el as HTMLElement
+ } else {
+ if (editable.getAttribute('aria-disabled') === 'true') return { error: 'disabled' }
+ if (editable.getAttribute('aria-readonly') === 'true') return { error: 'readonly' }
+ submissionTarget = editable
editable.focus()
editable.textContent = text
editable.dispatchEvent(
new InputEvent('input', { bubbles: true, data: text, inputType: 'insertText' })
)
- } else {
- return { error: 'not-editable' }
}
if (submit) {
@@ -673,15 +1820,25 @@ export function typeIntoElement(id: number, text: string, submit: boolean): unkn
keyCode: 13,
which: 13,
}
- const notCancelled = el.dispatchEvent(new KeyboardEvent('keydown', key))
- el.dispatchEvent(new KeyboardEvent('keyup', key))
- const form = (el as HTMLInputElement).form ?? (el as HTMLElement).closest?.('form') ?? null
+ const notCancelled = submissionTarget.dispatchEvent(new KeyboardEvent('keydown', key))
+ submissionTarget.dispatchEvent(new KeyboardEvent('keyup', key))
+ const form =
+ (submissionTarget as HTMLInputElement).form ?? submissionTarget.closest?.('form') ?? null
if (notCancelled && form) {
if (typeof form.requestSubmit === 'function') form.requestSubmit()
else form.submit()
}
}
- return { typed: true, submitted: submit === true }
+ return {
+ dispatched: true,
+ replacedExisting: true,
+ submitRequested: submit === true,
+ submitDispatched: submit === true,
+ submitted: false,
+ submitUncertain: false,
+ submissionEffectObserved: false,
+ refRecovered: resolved?.recovered === true,
+ }
}
export function pressKeyOnPage(
@@ -723,24 +1880,493 @@ export function pressKeyOnPage(
return { pressed: key, target: target.tagName.toLowerCase() }
}
-export function scrollPage(direction: string, amount?: number): unknown {
+/**
+ * Captures non-sensitive page state around a trusted input event. The driver
+ * compares two readings so “the event was dispatched” is never confused with
+ * “the page visibly reacted.”
+ */
+export function readPageActionState(resetMutationRevision = false, elementId?: number): unknown {
+ const registeredElement =
+ typeof elementId === 'number' ? (window.__simAgentElements || [])[elementId] : undefined
+ const resolver = window.__simAgentResolveElement
+ const resolved =
+ typeof elementId === 'number'
+ ? resolver
+ ? resolver(elementId)
+ : registeredElement?.isConnected
+ ? { element: registeredElement, recovered: false }
+ : null
+ : undefined
+ const observedElement = resolved?.element ?? null
+ const observedDocument =
+ observedElement?.ownerDocument ?? registeredElement?.ownerDocument ?? document
+ const observedWindow = observedDocument.defaultView ?? window
+ const observationRoot = observedDocument.body
+
+ const roots: ParentNode[] = observationRoot ? [observationRoot] : []
+ const allElements: Element[] = []
+ const stateNodeCap = 12_000
+ for (let index = 0; index < roots.length; index++) {
+ for (const element of Array.from(roots[index].querySelectorAll('*'))) {
+ if (allElements.length >= stateNodeCap) break
+ allElements.push(element)
+ const shadow = (element as HTMLElement).shadowRoot
+ if (shadow) roots.push(shadow)
+ }
+ if (allElements.length >= stateNodeCap) break
+ }
+
+ const mutationStates = (window.__simAgentMutationStates ??= [])
+ let mutationState = observationRoot
+ ? mutationStates.find((state) => state.root === observationRoot)
+ : undefined
+ if (!mutationState && observationRoot) {
+ mutationState = {
+ root: observationRoot,
+ observer: null as unknown as MutationObserver,
+ revision: 0,
+ }
+ const state = mutationState
+ state.observer = new MutationObserver((records) => {
+ state.revision += records.length
+ })
+ for (const root of roots) {
+ state.observer.observe(root, {
+ subtree: true,
+ childList: true,
+ characterData: true,
+ attributes: true,
+ attributeFilter: [
+ 'aria-activedescendant',
+ 'aria-expanded',
+ 'aria-hidden',
+ 'aria-selected',
+ 'checked',
+ 'disabled',
+ 'hidden',
+ 'open',
+ 'selected',
+ ],
+ })
+ }
+ mutationStates.push(state)
+ if (mutationStates.length > 10) mutationStates.shift()?.observer.disconnect()
+ }
+ if (resetMutationRevision) {
+ mutationState?.observer.takeRecords()
+ if (mutationState) mutationState.revision = 0
+ }
+
+ let active = observedDocument.activeElement as HTMLElement | null
+ for (let depth = 0; active && depth < 10; depth++) {
+ if (active.shadowRoot?.activeElement) {
+ active = active.shadowRoot.activeElement as HTMLElement
+ continue
+ }
+ const tag = String(active.tagName || '').toUpperCase()
+ if (tag === 'IFRAME' || tag === 'FRAME') {
+ try {
+ const inner = (active as HTMLIFrameElement).contentDocument
+ if (inner?.activeElement && inner.activeElement !== inner.body) {
+ active = inner.activeElement as HTMLElement
+ continue
+ }
+ } catch {
+ // Cross-origin frame — report the frame itself.
+ }
+ }
+ break
+ }
+
+ const dialogs = allElements.filter((element) =>
+ element.matches('dialog[open], [role="dialog"], [aria-modal="true"]')
+ )
+ const visibleDialogLabels = dialogs
+ .filter((element) => {
+ const rect = element.getBoundingClientRect()
+ const view = element.ownerDocument.defaultView
+ if (!view || rect.width <= 0 || rect.height <= 0) return false
+ for (let current: Element | null = element; current; ) {
+ const style = view.getComputedStyle(current)
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ Number.parseFloat(style.opacity || '1') <= 0.01 ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return false
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ }
+ return (
+ rect.right > 0 &&
+ rect.bottom > 0 &&
+ rect.left < view.innerWidth &&
+ rect.top < view.innerHeight
+ )
+ })
+ .slice(0, 10)
+ .map((element) =>
+ (
+ element.getAttribute('aria-label') ||
+ (element as HTMLElement).innerText ||
+ element.textContent ||
+ ''
+ )
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 120)
+ .replace(/[\uD800-\uDBFF]$/, '')
+ )
+
+ const visiblePopupLabels = allElements
+ .filter((element) => element.matches('[role="tooltip"], [role="menu"], [role="listbox"]'))
+ .filter((element) => {
+ const rect = element.getBoundingClientRect()
+ const view = element.ownerDocument.defaultView
+ if (!view || rect.width <= 0 || rect.height <= 0) return false
+ const style = view.getComputedStyle(element)
+ return (
+ style.display !== 'none' &&
+ style.visibility !== 'hidden' &&
+ Number.parseFloat(style.opacity || '1') > 0.01 &&
+ element.getAttribute('aria-hidden') !== 'true' &&
+ rect.right > 0 &&
+ rect.bottom > 0 &&
+ rect.left < view.innerWidth &&
+ rect.top < view.innerHeight
+ )
+ })
+ .slice(0, 10)
+ .map((element) =>
+ (
+ element.getAttribute('aria-label') ||
+ (element as HTMLElement).innerText ||
+ element.textContent ||
+ element.getAttribute('role') ||
+ ''
+ )
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 120)
+ .replace(/[\uD800-\uDBFF]$/, '')
+ )
+
+ const scrolledRegions = allElements
+ .filter((element) => (element as HTMLElement).scrollTop !== 0)
+ .slice(0, 30)
+ .map((element) => `${element.tagName}:${Math.round((element as HTMLElement).scrollTop)}`)
+
+ const isEffectivelyRendered = (element: Element): boolean => {
+ const rect = element.getBoundingClientRect()
+ const view = element.ownerDocument.defaultView
+ if (
+ !view ||
+ rect.width <= 1 ||
+ rect.height <= 1 ||
+ rect.right <= 0 ||
+ rect.bottom <= 0 ||
+ rect.left >= view.innerWidth ||
+ rect.top >= view.innerHeight
+ ) {
+ return false
+ }
+ for (let current: Element | null = element; current; ) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ if (
+ !style ||
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.contentVisibility === 'hidden' ||
+ (Number.isFinite(opacity) && opacity <= 0.01) ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return false
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ current = 'host' in root ? (root.host as Element) : null
+ }
+ }
+ return true
+ }
+
+ const targetState =
+ typeof elementId !== 'number'
+ ? undefined
+ : observedElement
+ ? {
+ present: true,
+ rendered: isEffectivelyRendered(observedElement),
+ ariaExpanded: observedElement.getAttribute('aria-expanded'),
+ ariaSelected: observedElement.getAttribute('aria-selected'),
+ ariaPressed: observedElement.getAttribute('aria-pressed'),
+ ariaChecked: observedElement.getAttribute('aria-checked'),
+ checked:
+ 'checked' in observedElement
+ ? Boolean((observedElement as HTMLInputElement).checked)
+ : undefined,
+ selected:
+ 'selected' in observedElement
+ ? Boolean((observedElement as HTMLOptionElement).selected)
+ : undefined,
+ open: observedElement.hasAttribute('open'),
+ hidden:
+ observedElement.hasAttribute('hidden') ||
+ observedElement.getAttribute('aria-hidden') === 'true',
+ }
+ : { present: false, rendered: false }
+
+ return {
+ url: observedWindow.location.href.slice(0, 4096),
+ title: observedDocument.title.slice(0, 500),
+ focus:
+ !active || active === active.ownerDocument.body
+ ? 'body'
+ : [
+ active.tagName.toLowerCase(),
+ active.getAttribute('role') || '',
+ active.getAttribute('id') || '',
+ active.getAttribute('name') || '',
+ active.getAttribute('aria-label') || '',
+ ].join(':'),
+ mutationRevision: mutationState?.revision || 0,
+ dialogs: visibleDialogLabels,
+ popups: visiblePopupLabels,
+ scroll: [Math.round(observedWindow.scrollY), ...scrolledRegions],
+ ...(targetState ? { targetState } : {}),
+ observationTruncated: allElements.length >= stateNodeCap,
+ }
+}
+
+export function scrollPage(direction: string, amount?: number, elementId?: number): unknown {
const distance = typeof amount === 'number' && amount > 0 ? amount : window.innerHeight * 0.85
- window.scrollBy({ top: direction === 'up' ? -distance : distance, behavior: 'instant' })
- const scrollY = Math.round(window.scrollY)
- const pageHeight = Math.round(document.documentElement.scrollHeight)
+ const delta = direction === 'up' ? -distance : distance
+ const scrollingElement = (document.scrollingElement || document.documentElement) as HTMLElement
+
+ const isVisible = (element: Element): boolean => {
+ const rect = element.getBoundingClientRect()
+ const view = element.ownerDocument.defaultView
+ if (!view || rect.width <= 0 || rect.height <= 0) return false
+ if (
+ rect.right <= 0 ||
+ rect.bottom <= 0 ||
+ rect.left >= view.innerWidth ||
+ rect.top >= view.innerHeight
+ ) {
+ return false
+ }
+ for (let current: Element | null = element; current; ) {
+ const currentView: Window | null = current.ownerDocument.defaultView
+ const style = currentView?.getComputedStyle(current)
+ const opacity = Number.parseFloat(style?.opacity || '1')
+ if (
+ !style ||
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.contentVisibility === 'hidden' ||
+ (Number.isFinite(opacity) && opacity <= 0.01) ||
+ current.hasAttribute('hidden') ||
+ current.getAttribute('aria-hidden') === 'true'
+ ) {
+ return false
+ }
+ if (current.parentElement) current = current.parentElement
+ else {
+ const root = current.getRootNode()
+ if ('host' in root) current = root.host as Element
+ else {
+ const frame: Element | null = current.ownerDocument.defaultView?.frameElement ?? null
+ current = frame
+ }
+ }
+ }
+ return true
+ }
+ const isScrollable = (element: Element): element is HTMLElement => {
+ const html = element as HTMLElement
+ if (html.scrollHeight <= html.clientHeight + 1) return false
+ const ownerScroller =
+ element.ownerDocument.scrollingElement || element.ownerDocument.documentElement
+ if (element === ownerScroller) return true
+ const view = element.ownerDocument.defaultView
+ if (!view) return false
+ const overflow = view.getComputedStyle(element).overflowY
+ return overflow === 'auto' || overflow === 'scroll' || overflow === 'overlay'
+ }
+ const canMove = (element: HTMLElement): boolean => {
+ const max = Math.max(0, element.scrollHeight - element.clientHeight)
+ return direction === 'up' ? element.scrollTop > 1 : element.scrollTop < max - 1
+ }
+ const ancestors = (start: Element | null): HTMLElement[] => {
+ const result: HTMLElement[] = []
+ let current = start
+ while (current) {
+ if (isScrollable(current) && isVisible(current)) result.push(current)
+ const root = current.getRootNode()
+ if (current.parentElement) current = current.parentElement
+ else if ('host' in root && root.host) current = root.host as Element
+ else {
+ const frame = current.ownerDocument.defaultView?.frameElement
+ current = frame ? (frame as Element) : null
+ }
+ }
+ return result
+ }
+ const deepActiveElement = (): Element | null => {
+ let active = document.activeElement
+ for (let depth = 0; active && depth < 10; depth++) {
+ if ((active as HTMLElement).shadowRoot?.activeElement) {
+ active = (active as HTMLElement).shadowRoot?.activeElement ?? active
+ continue
+ }
+ const tag = String(active.tagName || '').toUpperCase()
+ if (tag === 'IFRAME' || tag === 'FRAME') {
+ try {
+ const inner = (active as HTMLIFrameElement).contentDocument
+ if (inner?.activeElement && inner.activeElement !== inner.body) {
+ active = inner.activeElement
+ continue
+ }
+ } catch {
+ // Cross-origin frame — use the frame as the focus anchor.
+ }
+ }
+ break
+ }
+ return active
+ }
+
+ let target: HTMLElement | undefined
+ let boundaryFallback: HTMLElement | undefined
+ let boundarySource = 'page'
+ let source = 'page'
+ if (typeof elementId === 'number') {
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(elementId)
+ const element = resolver ? resolved?.element : (window.__simAgentElements || [])[elementId]
+ if (!element || !element.isConnected) return { error: 'stale' }
+ const candidates = ancestors(element)
+ target = candidates.find(canMove) ?? candidates[0]
+ source = target && canMove(target) ? 'element' : 'element-boundary'
+ }
+ if (!target) {
+ const focused = ancestors(deepActiveElement())
+ const movable = focused.find(canMove)
+ if (movable) {
+ target = movable
+ source = 'focus'
+ } else if (focused[0]) {
+ boundaryFallback = focused[0]
+ boundarySource = 'focus-boundary'
+ }
+ }
+ if (!target && typeof document.elementsFromPoint === 'function') {
+ const centered = document.elementsFromPoint(window.innerWidth / 2, window.innerHeight / 2)
+ for (const element of centered) {
+ const candidates = ancestors(element)
+ const movable = candidates.find(canMove)
+ if (movable) {
+ target = movable
+ source = 'viewport-center'
+ break
+ }
+ if (candidates[0] && boundarySource !== 'viewport-center-boundary') {
+ boundaryFallback = candidates[0]
+ boundarySource = 'viewport-center-boundary'
+ }
+ }
+ }
+ // A focused or center-hit pane is an explicit affinity signal. If it is at
+ // the requested boundary, report a zero move there instead of wandering to
+ // an unrelated movable sidebar and calling that success.
+ if (!target && boundaryFallback) {
+ target = boundaryFallback
+ source = boundarySource
+ }
+ if (!target) {
+ const scanCap = 12_000
+ const roots: ParentNode[] = [document]
+ const scanned: Element[] = []
+ for (let rootIndex = 0; rootIndex < roots.length && scanned.length < scanCap; rootIndex++) {
+ for (const element of Array.from(roots[rootIndex].querySelectorAll('*'))) {
+ if (scanned.length >= scanCap) break
+ scanned.push(element)
+ const shadow = (element as HTMLElement).shadowRoot
+ if (shadow) roots.push(shadow)
+ }
+ }
+ const candidates = scanned
+ .filter((element): element is HTMLElement => isScrollable(element) && isVisible(element))
+ .filter(canMove)
+ .sort((a, b) => {
+ const aRect = a.getBoundingClientRect()
+ const bRect = b.getBoundingClientRect()
+ return bRect.width * bRect.height - aRect.width * aRect.height
+ })
+ target = candidates[0]
+ if (target) source = 'largest-visible'
+ }
+ target ??= scrollingElement
+
+ const targetDocument = target.ownerDocument
+ const targetWindow = targetDocument.defaultView
+ const targetDocumentScroller = targetDocument.scrollingElement || targetDocument.documentElement
+ const isDocumentScroller = target === targetDocumentScroller
+ const before = isDocumentScroller ? (targetWindow?.scrollY ?? target.scrollTop) : target.scrollTop
+ if (isDocumentScroller && targetWindow) {
+ targetWindow.scrollBy({ top: delta, behavior: 'instant' })
+ } else if (typeof target.scrollBy === 'function') {
+ target.scrollBy({ top: delta, behavior: 'instant' })
+ } else {
+ target.scrollTop += delta
+ }
+ const scrollTop = isDocumentScroller
+ ? (targetWindow?.scrollY ?? target.scrollTop)
+ : target.scrollTop
+ const scrollHeight = isDocumentScroller
+ ? targetDocument.documentElement.scrollHeight
+ : target.scrollHeight
+ const clientHeight = isDocumentScroller
+ ? (targetWindow?.innerHeight ?? target.clientHeight)
+ : target.clientHeight
+ const label =
+ target.getAttribute('aria-label') ||
+ target.getAttribute('role') ||
+ target.getAttribute('id') ||
+ target.tagName.toLowerCase()
return {
- scrollY,
- pageHeight,
- atTop: scrollY <= 0,
- atBottom: scrollY + window.innerHeight >= pageHeight - 2,
+ target: label,
+ targetSource: source,
+ scrollTop: Math.round(scrollTop),
+ scrollHeight: Math.round(scrollHeight),
+ clientHeight: Math.round(clientHeight),
+ movedBy: Math.round(scrollTop - before),
+ atTop: scrollTop <= 1,
+ atBottom: scrollTop + clientHeight >= scrollHeight - 2,
+ windowScrollY: Math.round(window.scrollY),
}
}
export function selectOptionInElement(id: number, value: string): unknown {
- const el = (window.__simAgentElements || [])[id]
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
if (!el || !el.isConnected) return { error: 'stale' }
- if (el.tagName !== 'SELECT') return { error: 'not-select' }
+ if (String(el.tagName || '').toUpperCase() !== 'SELECT') return { error: 'not-select' }
const select = el as HTMLSelectElement
+ if (select.disabled || select.getAttribute('aria-disabled') === 'true') {
+ return { error: 'disabled' }
+ }
const wanted = value.trim().toLowerCase()
const option = Array.from(select.options).find(
(o) => o.value.trim().toLowerCase() === wanted || o.label.trim().toLowerCase() === wanted
@@ -750,19 +2376,46 @@ export function selectOptionInElement(id: number, value: string): unknown {
error: 'no-option',
options: Array.from(select.options)
.slice(0, 50)
- .map((o) => o.label.trim()),
+ .map((o) =>
+ o.label
+ .trim()
+ .slice(0, 200)
+ .replace(/[\uD800-\uDBFF]$/, '')
+ ),
}
}
+ if (option.disabled || (option.parentElement as HTMLOptGroupElement | null)?.disabled === true) {
+ return { error: 'disabled' }
+ }
select.value = option.value
select.dispatchEvent(new Event('input', { bubbles: true }))
select.dispatchEvent(new Event('change', { bubbles: true }))
- return { selected: option.label.trim() }
+ return {
+ selected: option.label.trim(),
+ value: option.value,
+ refRecovered: resolved?.recovered === true,
+ }
+}
+
+export function readSelectElementState(id: number): unknown {
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
+ if (!el || !el.isConnected) return { error: 'stale' }
+ if (String(el.tagName || '').toUpperCase() !== 'SELECT') return { error: 'not-select' }
+ const select = el as HTMLSelectElement
+ return {
+ selected: select.selectedOptions[0]?.label.trim() || '',
+ value: select.value,
+ }
}
export function hoverElement(id: number): unknown {
- const el = (window.__simAgentElements || [])[id]
+ const resolver = window.__simAgentResolveElement
+ const resolved = resolver?.(id)
+ const el = resolver ? resolved?.element : (window.__simAgentElements || [])[id]
if (!el || !el.isConnected) return { error: 'stale' }
- el.scrollIntoView({ block: 'center' })
+ el.scrollIntoView({ block: 'center', behavior: 'instant' })
const rect = el.getBoundingClientRect()
const opts = {
bubbles: true,
@@ -777,14 +2430,211 @@ export function hoverElement(id: number): unknown {
el.dispatchEvent(new MouseEvent('mouseenter', opts))
el.dispatchEvent(new PointerEvent('pointermove', opts))
el.dispatchEvent(new MouseEvent('mousemove', opts))
- return { hovered: true }
+ return { hovered: true, refRecovered: resolved?.recovered === true }
+}
+
+/**
+ * Resolves one child frame's embedding element from its parent frame and
+ * verifies that the surface is rendered, onscreen, and not covered. This is
+ * evaluated in the parent because a cross-origin child cannot inspect its own
+ *