[ux] sync dom elements to live view

images/chromium-headful/client/src/app.vue

@@ -13,6 +13,8 @@
             ref="video"
             :hideControls="hideControls"
             :extraControls="isEmbedMode"
+            :showDomOverlay="showDomOverlay"
+            :domSyncTypes="domSyncTypes"
             @control-attempt="controlAttempt"
           />
         </div>
@@ -186,6 +188,7 @@
   import About from '~/components/about.vue'
   import Header from '~/components/header.vue'
   import Unsupported from '~/components/unsupported.vue'
+  import { DomSyncPayload, DomWebSocketMessage, DomElementType, DOM_ELEMENT_TYPES } from '~/neko/dom-types'
 
   @Component({
     name: 'neko',
@@ -208,6 +211,41 @@
 
     shakeKbd = false
     wasConnected = false
+    private domWebSocket: WebSocket | null = null
+    private domReconnectTimeout: number | null = null
+
+    // dom_sync: enables WebSocket connection for DOM element syncing
+    // Values: false (default), true (inputs only), or comma-separated types (e.g., "inputs,buttons,links")
+    get isDomSyncEnabled(): boolean {
+      const params = new URL(location.href).searchParams
+      const param = params.get('dom_sync') || params.get('domSync')
+      if (!param || param === 'false' || param === '0') return false
+      return true
+    }
+
+    // Get enabled DOM element types from query param
+    // ?dom_sync=true -> ['inputs'] (default, backwards compatible)
+    // ?dom_sync=inputs,buttons,links -> ['inputs', 'buttons', 'links']
+    get domSyncTypes(): DomElementType[] {
+      const params = new URL(location.href).searchParams
+      const param = params.get('dom_sync') || params.get('domSync')
+      if (!param || param === 'false' || param === '0') return []
+      // If true/1, default to inputs only (backwards compatible)
+      if (param === 'true' || param === '1') return ['inputs']
+      // Parse comma-separated list and filter to valid types
+      const types = param.split(',').map((t) => t.trim().toLowerCase()) as DomElementType[]
+      return types.filter((t) => DOM_ELEMENT_TYPES.includes(t))
+    }
+
+    // dom_overlay: shows purple overlay rectangles when dom_sync is enabled (default: true)
+    get showDomOverlay() {
+      if (!this.isDomSyncEnabled) return false
+      const params = new URL(location.href).searchParams
+      const param = params.get('dom_overlay') || params.get('domOverlay')
+      // Default to true if not specified, false only if explicitly set to 'false' or '0'
+      if (param === null) return true
+      return param !== 'false' && param !== '0'
+    }
 
     get volume() {
       const numberParam = parseFloat(new URL(location.href).searchParams.get('volume') || '1.0')
@@ -278,13 +316,20 @@
       if (value) {
         this.wasConnected = true
         this.applyQueryResolution()
+        // Connect to DOM sync if enabled
+        if (this.isDomSyncEnabled) {
+          this.connectDomSync()
+        }
         try {
           if (window.parent !== window) {
             window.parent.postMessage({ type: 'KERNEL_CONNECTED', connected: true }, this.parentOrigin)
           }
         } catch (e) {
           console.error('Failed to post message to parent', e)
         }
+      } else {
+        // Disconnect DOM sync when main connection is lost
+        this.disconnectDomSync()
       }
     }
 
@@ -331,6 +376,92 @@
 
     // KERNEL: end custom resolution, frame rate, and readOnly control via query params
 
+    // KERNEL: DOM Sync - connects to kernel-images API WebSocket for bounding box overlay
+    private domRetryCount = 0
+    private readonly domMaxRetries = 10
+
+    private connectDomSync() {
+      if (!this.isDomSyncEnabled) return
+      if (this.domWebSocket && this.domWebSocket.readyState === WebSocket.OPEN) return
+
+      const params = new URL(location.href).searchParams
+      const domPort = params.get('dom_port') || '444'
+      const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'
+      const domUrl = `${protocol}//${location.hostname}:${domPort}/dom-sync`
+
+      console.log(`[dom-sync] Connecting to ${domUrl} (attempt ${this.domRetryCount + 1})`)
+
+      try {
+        this.domWebSocket = new WebSocket(domUrl)
+
+        this.domWebSocket.onopen = () => {
+          console.log('[dom-sync] Connected')
+          this.domRetryCount = 0 // Reset retry count on success
+          this.$accessor.dom.setEnabled(true)
+          this.$accessor.dom.setConnected(true)
+        }
+
+        this.domWebSocket.onmessage = (event) => {
+          try {
+            const message: DomWebSocketMessage = JSON.parse(event.data)
+            if (message.event === 'dom/sync' && message.data) {
+              this.$accessor.dom.applySync(message.data)
+            }
+          } catch (e) {
+            console.error('[dom-sync] Failed to parse message:', e)
+          }
+        }
+
+        this.domWebSocket.onclose = () => {
+          console.log('[dom-sync] Disconnected')
+          this.$accessor.dom.setConnected(false)
+          this.scheduleDomReconnect()
+        }
+
+        this.domWebSocket.onerror = (error) => {
+          console.error('[dom-sync] WebSocket error:', error)
+          // Error will trigger onclose, which handles reconnect
+        }
+      } catch (e) {
+        console.error('[dom-sync] Failed to connect:', e)
+        this.scheduleDomReconnect()
+      }
+    }
+
+    private scheduleDomReconnect() {
+      if (!this.isDomSyncEnabled || !this.connected) return
+      if (this.domRetryCount >= this.domMaxRetries) {
+        console.log('[dom-sync] Max retries reached, giving up')
+        return
+      }
+      // Exponential backoff: 500ms, 1s, 2s, 4s... capped at 5s
+      const delay = Math.min(500 * Math.pow(2, this.domRetryCount), 5000)
+      this.domRetryCount++
+      console.log(`[dom-sync] Reconnecting in ${delay}ms`)
+      this.domReconnectTimeout = window.setTimeout(() => {
+        this.connectDomSync()
+      }, delay)
+    }
+
+    private disconnectDomSync() {
+      if (this.domReconnectTimeout) {
+        clearTimeout(this.domReconnectTimeout)
+        this.domReconnectTimeout = null
+      }
+      if (this.domWebSocket) {
+        this.domWebSocket.close()
+        this.domWebSocket = null
+      }
+      this.domRetryCount = 0
+      this.$accessor.dom.setEnabled(false)
+      this.$accessor.dom.setConnected(false)
+      this.$accessor.dom.reset()
+    }
+
+    beforeDestroy() {
+      this.disconnectDomSync()
+    }
+
     controlAttempt() {
       if (this.shakeKbd || this.$accessor.remote.hosted) return
 

images/chromium-headful/client/src/components/dom-overlay.vue

@@ -0,0 +1,176 @@
+<template>
+  <div v-if="enabled && showOverlay && hasFilteredElements" ref="overlay" class="dom-overlay" :class="{ disabled: tempDisabled }">
+    <div
+      v-for="el in filteredTransformedElements"
+      :key="el.id"
+      class="dom-element"
+      :style="getElementStyle(el)"
+      @touchstart="onInputTap"
+      @mousedown="onInputTap"
+    />
+  </div>
+</template>
+
+<style lang="scss" scoped>
+.dom-overlay {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  pointer-events: none;
+  z-index: 110;
+
+  &.disabled {
+    pointer-events: none !important;
+    .dom-element {
+      pointer-events: none !important;
+    }
+  }
+}
+
+.dom-element {
+  position: absolute;
+  pointer-events: auto;
+  cursor: pointer;
+  border-radius: 3px;
+  /* Default style - will be overridden by inline styles */
+  background: rgba(139, 92, 246, 0.1);
+  border: 1px solid rgba(139, 92, 246, 0.3);
+}
+</style>
+
+<script lang="ts">
+import { Component, Vue, Prop } from 'vue-property-decorator'
+import { DomElement, DomWindowBounds, DomElementType, DOM_TYPE_COLORS } from '~/neko/dom-types'
+
+interface TransformedElement extends DomElement {
+  screenX: number
+  screenY: number
+}
+
+@Component({
+  name: 'dom-overlay',
+})
+export default class extends Vue {
+  @Prop({ type: Number, default: 1920 }) screenWidth!: number
+  @Prop({ type: Number, default: 1080 }) screenHeight!: number
+  @Prop({ type: Boolean, default: true }) showOverlay!: boolean
+  @Prop({ type: Array, default: () => ['inputs'] }) enabledTypes!: DomElementType[]
+
+  tempDisabled = false
+
+  get enabled(): boolean {
+    return this.$accessor.dom.enabled
+  }
+
+  get elements(): DomElement[] {
+    return this.$accessor.dom.elements
+  }
+
+  get hasElements(): boolean {
+    return this.$accessor.dom.hasElements
+  }
+
+  get windowBounds(): DomWindowBounds {
+    return this.$accessor.dom.windowBounds
+  }
+
+  // Filter elements by enabled types
+  get filteredElements(): DomElement[] {
+    if (this.enabledTypes.length === 0) return []
+    return this.elements.filter((el) => this.enabledTypes.includes(el.type))
+  }
+
+  get hasFilteredElements(): boolean {
+    return this.filteredElements.length > 0
+  }
+
+  get filteredTransformedElements(): TransformedElement[] {
+    const bounds = this.windowBounds
+    const offsetX = bounds.x + bounds.chromeLeft
+    const offsetY = bounds.y + bounds.chromeTop
+
+    return this.filteredElements.map((el) => ({
+      ...el,
+      screenX: offsetX + el.rect.x,
+      screenY: offsetY + el.rect.y,
+    }))
+  }
+
+  getElementStyle(el: TransformedElement): Record<string, string> {
+    const xPercent = (el.screenX / this.screenWidth) * 100
+    const yPercent = (el.screenY / this.screenHeight) * 100
+    const wPercent = (el.rect.w / this.screenWidth) * 100
+    const hPercent = (el.rect.h / this.screenHeight) * 100
+
+    // Get colors for this element type
+    const colors = DOM_TYPE_COLORS[el.type] || DOM_TYPE_COLORS.inputs
+
+    return {
+      left: `${xPercent}%`,
+      top: `${yPercent}%`,
+      width: `${wPercent}%`,
+      height: `${hPercent}%`,
+      background: colors.bg,
+      borderColor: colors.border,
+    }
+  }
+
+  onInputTap(e: MouseEvent | TouchEvent) {
+    e.preventDefault()
+    e.stopPropagation()
+
+    // Get click coordinates
+    let clientX: number, clientY: number
+    if ('touches' in e && e.touches.length > 0) {
+      clientX = e.touches[0].clientX
+      clientY = e.touches[0].clientY
+    } else if ('clientX' in e) {
+      clientX = e.clientX
+      clientY = e.clientY
+    } else {
+      return
+    }
+
+    // Find the neko overlay textarea - this is what handles input events
+    const overlay = document.querySelector('.player-container .overlay') as HTMLTextAreaElement
+
+    // Focus the textarea to trigger mobile keyboard
+    if (overlay) {
+      overlay.focus()
+
+      // Create event options
+      const eventInit: MouseEventInit = {
+        bubbles: true,
+        cancelable: true,
+        clientX,
+        clientY,
+        screenX: clientX,
+        screenY: clientY,
+        view: window,
+        button: 0,
+        buttons: 1,
+      }
+
+      // Temporarily enable pointer events on overlay if needed
+      const oldPointerEvents = overlay.style.pointerEvents
+      overlay.style.pointerEvents = 'auto'
+
+      // Dispatch mouse events to simulate a click
+      overlay.dispatchEvent(new MouseEvent('mousedown', eventInit))
+      setTimeout(() => {
+        overlay.dispatchEvent(new MouseEvent('mouseup', { ...eventInit, buttons: 0 }))
+        // Restore pointer events
+        overlay.style.pointerEvents = oldPointerEvents
+      }, 20)
+    }
+
+    // Temporarily disable dom overlay for follow-up interactions
+    this.tempDisabled = true
+    setTimeout(() => {
+      this.tempDisabled = false
+    }, 500)
+  }
+}
+</script>

images/chromium-headful/client/src/components/video.vue

@@ -27,6 +27,7 @@
           @touchstart.stop.prevent="onTouchHandler"
           @touchend.stop.prevent="onTouchHandler"
         />
+        <dom-overlay :screenWidth="width" :screenHeight="height" :showOverlay="showDomOverlay" :enabledTypes="domSyncTypes" />
 <!-- KERNEL
         <div v-if="!playing && playable" class="player-overlay" @click.stop.prevent="playAndUnmute">
           <i class="fas fa-play-circle" />
@@ -205,6 +206,7 @@
           color: transparent;
           background: transparent;
           resize: none;
+          z-index: 100; // Above dom-overlay (z-index: 110)
         }
 
         .player-aspect {
@@ -224,6 +226,8 @@
   import Emote from './emote.vue'
   import Resolution from './resolution.vue'
   import Clipboard from './clipboard.vue'
+  import DomOverlay from './dom-overlay.vue'
+  import { DomElementType } from '~/neko/dom-types'
 
   // @ts-ignore
   import GuacamoleKeyboard from '~/utils/guacamole-keyboard.ts'
@@ -236,6 +240,7 @@
       'neko-emote': Emote,
       'neko-resolution': Resolution,
       'neko-clipboard': Clipboard,
+      'dom-overlay': DomOverlay,
     },
   })
   export default class extends Vue {
@@ -252,6 +257,10 @@
     @Prop(Boolean) readonly hideControls!: boolean
     // extra controls are shown (e.g. for embed mode)
     @Prop(Boolean) readonly extraControls!: boolean
+    // show the purple DOM overlay rectangles (defaults to true)
+    @Prop({ type: Boolean, default: true }) readonly showDomOverlay!: boolean
+    // enabled DOM element types for overlay (defaults to ['inputs'])
+    @Prop({ type: Array, default: () => ['inputs'] }) readonly domSyncTypes!: DomElementType[]
 
     private keyboard = GuacamoleKeyboard()
     private observer = new ResizeObserver(this.onResize.bind(this))