feat(app): frame a device capture as the device

.changeset/frame-a-device-capture.md

@@ -0,0 +1,11 @@
+---
+"@wdio/devtools-app": minor
+---
+
+Frame a capture that came off a device as the device, not as a desktop browser window. A mobile trace rendered inside the desktop chrome — traffic lights, and an address bar reading `unknown`, since a native app has no url — with the phone left as a narrow strip in the middle of a landscape frame. Measured on a 1170x2532 capture: the image used 162 px of a 388 px frame and the remaining ~60% was backdrop.
+
+The frame is now shaped from the capture's own decoded pixels, and its header states the device instead of drawing window furniture that describes nothing. The capture's pixels are the only workable source: the metadata viewport disagrees with the screenshot on both platforms, since Android reports the window without the navigation bar and iOS reports points, so an older binary on an iPhone 17 reports a 390x844 window for a 402x874 screen. The frame's own header and padding are taken off before fitting and added back after, or the capture area comes out short by them and the image letterboxes inside a frame that was supposed to be its shape.
+
+Only the screenshot branch is reframed. A mobile *browser* session — Appium driving Chrome on Android — reports a device and also carries a DOM, and that replay is an iframe laid out at its own captured viewport; shaping the frame to a screenshot as well would fight that sizing for the same box, and such a session is a real browser with a real url, so the browser chrome stays honest there. A desktop capture is untouched.
+
+Reads the `device` field added in #345, and the decoded-size helper added in #344.

packages/app/src/components/browser/device-frame.ts

@@ -0,0 +1,100 @@
+// The player's frame for a capture that came off a device. A native session has
+// no browser window and no url, so the desktop chrome — traffic lights and an
+// address bar reading `unknown` — describes nothing, and a landscape frame
+// leaves a portrait capture as a narrow strip in the middle of it.
+//
+// Kept out of snapshot.ts (already over the file cap) so the geometry is a pure
+// function the specs can measure without a browser.
+
+import type { DeviceInfo, ImageSize } from '@wdio/devtools-shared'
+import { deviceLabel } from '@wdio/devtools-shared'
+import { html, type nothing, type TemplateResult } from 'lit'
+
+/**
+ * Padding plus border across one axis. Both come out of the size set on a
+ * border-box element, so a frame sized without the border hands its capture an
+ * area smaller than the one it was fitted for.
+ */
+export function edgeInset(
+  style: CSSStyleDeclaration,
+  from: 'Left' | 'Top',
+  to: 'Right' | 'Bottom'
+): number {
+  const read = (value: string) => parseFloat(value || '0') || 0
+  return (
+    read(style[`padding${from}` as 'paddingLeft']) +
+    read(style[`padding${to}` as 'paddingRight']) +
+    read(style[`border${from}Width` as 'borderLeftWidth']) +
+    read(style[`border${to}Width` as 'borderRightWidth'])
+  )
+}
+
+/** What the frame spends on itself before the capture gets any room. */
+export interface FrameChrome {
+  /** Height of the frame's header — furniture above the capture. */
+  headerHeight: number
+  /** The frame's own horizontal padding and border. */
+  insetX: number
+  /** The frame's own vertical padding and border. */
+  insetY: number
+}
+
+/**
+ * Size for a frame holding `capture`, so the frame is the shape of the device
+ * rather than of the pane. Derived from the capture's own decoded pixels: the
+ * metadata viewport cannot serve, because it disagrees with the screenshot on
+ * both mobile platforms — Android reports the window without the navigation
+ * bar, and iOS reports points, so an older binary on an iPhone 17 says 390x844
+ * of a 402x874 screen.
+ *
+ * The frame's own furniture is taken off before fitting and added back after.
+ * Fitting the whole frame to the capture instead leaves the capture AREA short
+ * by the header and the padding, so the image letterboxes inside a frame that
+ * was supposed to be its shape.
+ */
+export function deviceFrameSize(
+  pane: { width: number; height: number },
+  capture: ImageSize,
+  chrome: FrameChrome
+): { width: number; height: number } {
+  const usableWidth = pane.width - chrome.insetX
+  const usableHeight = pane.height - chrome.insetY - chrome.headerHeight
+  if (
+    !capture.width ||
+    !capture.height ||
+    usableWidth <= 0 ||
+    usableHeight <= 0
+  ) {
+    return { width: pane.width, height: pane.height }
+  }
+  const scale = Math.min(
+    usableWidth / capture.width,
+    usableHeight / capture.height
+  )
+  return {
+    width: Math.round(capture.width * scale) + chrome.insetX,
+    height:
+      Math.round(capture.height * scale) + chrome.headerHeight + chrome.insetY
+  }
+}
+
+/**
+ * The frame's header for a device capture: what it was recorded on, in place of
+ * window furniture that does not apply. Keeps the view-toggle slot, which is
+ * how the Snapshot/Screencast switch stays reachable.
+ */
+export function renderDeviceChrome(
+  device: DeviceInfo,
+  viewToggle: TemplateResult | typeof nothing
+): TemplateResult {
+  return html`
+    <header
+      class="device-chrome flex items-center mx-2 bg-sideBarBackground rounded-t-[14px]"
+    >
+      <span class="device-label truncate" title=${deviceLabel(device)}
+        >${deviceLabel(device)}</span
+      >
+      ${viewToggle}
+    </header>
+  `
+}

packages/app/src/components/browser/snapshot-styles.ts

@@ -63,6 +63,24 @@ export const snapshotStyles = css`
     border-radius: 0 0 14px 14px;
   }
 
+  /* Device frame: the header states what the capture came off, in place of an
+     address bar a native session could only have filled with "unknown".
+     NB: no backticks in this file — the rules live in a tagged template. */
+  .device-chrome {
+    padding: 0.45rem 0.25rem;
+    gap: 0.5rem;
+  }
+
+  .device-label {
+    flex: 1;
+    min-width: 0;
+    padding-left: 0.5rem;
+    font-size: 11px;
+    font-weight: 600;
+    letter-spacing: 0.02em;
+    color: var(--vscode-descriptionForeground, #ccc);
+  }
+
   .screenshot-overlay {
     position: absolute;
     inset: 0;

packages/app/src/components/browser/snapshot.ts

@@ -4,6 +4,11 @@ import { html, nothing } from 'lit'
 import { consume } from '@lit/context'
 import { snapshotStyles } from './snapshot-styles.js'
 import { renderBrowserChrome } from './browser-chrome.js'
+import {
+  deviceFrameSize,
+  edgeInset,
+  renderDeviceChrome
+} from './device-frame.js'
 import {
   drawElementOverlay,
   clearElementOverlay,
@@ -21,15 +26,24 @@ import type { SimplifiedVNode } from '@wdio/devtools-script/types'
 // characterData wire shape (parent ref + child index), so the replay reads it
 // from the same declaration that produces it.
 import type { TextMutation } from '@wdio/devtools-script/mutations.js'
-import { imageMime, type CommandLog } from '@wdio/devtools-shared'
+import {
+  imageDimensions,
+  imageMime,
+  type CommandLog,
+  type ImageSize
+} from '@wdio/devtools-shared'
 
 import {
   mutationContext,
   metadataContext,
   metadataBySessionContext,
   commandContext
 } from '../../controller/context.js'
-import type { Metadata, MetadataBySession } from '@wdio/devtools-shared'
+import type {
+  DeviceInfo,
+  Metadata,
+  MetadataBySession
+} from '@wdio/devtools-shared'
 
 import '../placeholder.js'
 import './screencast-player.js'
@@ -157,6 +171,48 @@ export class DevtoolsBrowser extends Element {
     }
   }
 
+  #captureShape?: { screenshot: string; size: ImageSize | null }
+
+  /** Shape of the capture on screen, from its own decoded pixels. Memoized on
+   *  the frame it was read from, because this is read per render and per rAF. */
+  get #captureSize(): ImageSize | null {
+    const screenshot = this.#screenshotData ?? this.#latestAutoScreenshot
+    if (!screenshot) {
+      return null
+    }
+    if (this.#captureShape?.screenshot !== screenshot) {
+      this.#captureShape = { screenshot, size: imageDimensions(screenshot) }
+    }
+    return this.#captureShape.size
+  }
+
+  /**
+   * The device a capture was recorded on, once there is an image to shape the
+   * frame around — a mobile BROWSER session (Appium driving Chrome on Android)
+   * reports a device too, and must keep the browser frame. Two conditions
+   * separate them, because either alone is decidable too late:
+   *
+   * - no DOM. That replay is an iframe laid out at its own captured viewport,
+   *   so shaping the frame to a screenshot would fight
+   *   `#sizeSnapshotToViewport` for the same box. It is the same signal
+   *   `#renderViewport` branches on, so the frame can never disagree with what
+   *   is inside it — but mutations arrive in batches, so early in a live run a
+   *   browser session has none yet.
+   * - no url ON THE METADATA. A native session never issues a navigation, so
+   *   it never reports one, while a browser session reports one from its first
+   *   navigation — which lands before any DOM batch and closes that window.
+   *   Deliberately not `#displayUrl`: that is a display concern, resolved from
+   *   the selected command and empty until one is selected.
+   */
+  get #deviceCapture(): { device: DeviceInfo; size: ImageSize } | null {
+    if (this.mutations?.length || this.metadata?.url) {
+      return null
+    }
+    const device = this.metadata?.device
+    const size = device ? this.#captureSize : null
+    return device && size ? { device, size } : null
+  }
+
   #setIframeSize() {
     if (!this.section || !this.header) {
       return
@@ -170,9 +226,62 @@ export class DevtoolsBrowser extends Element {
       this.section.style.height = '100%'
       return
     }
+    if (this.#deviceCapture) {
+      this.#sizeSectionToDevice()
+      return
+    }
     this.#sizeSnapshotToViewport()
   }
 
+  /** Shape the frame to the device rather than to the pane. A native capture
+   *  reaches the screenshot branch, which fills whatever box it is given — so
+   *  in a landscape frame a portrait capture was a narrow strip with ~60% of
+   *  the frame backdrop, wrapped in window furniture describing nothing. */
+  #sizeSectionToDevice() {
+    requestAnimationFrame(() => {
+      const capture = this.#deviceCapture
+      // The mode can flip between scheduling and firing: `updated()` re-sizes
+      // on every view-mode change and the video branch is synchronous, so an
+      // unguarded callback lands AFTER it and pins the screencast inside a
+      // phone-shaped box until the next resize.
+      if (
+        !this.section ||
+        !this.header ||
+        !capture ||
+        this.#viewMode === 'video'
+      ) {
+        return
+      }
+      const hostStyle = getComputedStyle(this)
+      const rect = this.getBoundingClientRect()
+      const padX =
+        parseFloat(hostStyle.paddingLeft || '0') +
+        parseFloat(hostStyle.paddingRight || '0')
+      const padY =
+        parseFloat(hostStyle.paddingTop || '0') +
+        parseFloat(hostStyle.paddingBottom || '0')
+      const sectionStyle = getComputedStyle(this.section)
+      const frame = deviceFrameSize(
+        {
+          width: Math.max(0, rect.width - padX),
+          height: Math.max(0, rect.height - padY)
+        },
+        capture.size,
+        {
+          headerHeight: this.header.getBoundingClientRect().height,
+          // Padding AND border: the section is border-box, so both come out of
+          // the width and height set on it. Omitting the 2px border left the
+          // capture area 4px short per axis and letterboxed it inside a frame
+          // that was supposed to be its shape.
+          insetX: edgeInset(sectionStyle, 'Left', 'Right'),
+          insetY: edgeInset(sectionStyle, 'Top', 'Bottom')
+        }
+      )
+      this.section.style.width = `${frame.width}px`
+      this.section.style.height = `${frame.height}px`
+    })
+  }
+
   #sizeSnapshotToViewport() {
     const metadata = this.metadata
     if (!this.section || !this.header || !metadata) {
@@ -940,12 +1049,19 @@ export class DevtoolsBrowser extends Element {
       <section
         class="w-full h-full bg-sideBarBackground rounded-[14px] border-2 border-panelBorder"
       >
-        ${renderBrowserChrome(
-          this.#displayUrl,
-          html`${this.#renderOverlayToggle(
-            hasMutations
-          )}${this.#renderViewToggle()}`
-        )}
+        ${
+          this.#deviceCapture
+            ? renderDeviceChrome(
+                this.#deviceCapture.device,
+                this.#renderViewToggle()
+              )
+            : renderBrowserChrome(
+                this.#displayUrl,
+                html`${this.#renderOverlayToggle(
+                  hasMutations
+                )}${this.#renderViewToggle()}`
+              )
+        }
         ${this.#renderViewport(hasMutations)}
       </section>
     `

packages/app/test-ui/workbench/player/fixtures.ts

@@ -557,6 +557,17 @@ export const metadataForViewport = (
   viewport: { ...viewport, width, height }
 })
 
+/**
+ * A native mobile capture: it states its device, and its viewport is the one a
+ * device really reports — iOS in POINTS, deliberately disagreeing with the
+ * 120x260 frames, which is why the frame is shaped by the decoded image.
+ */
+export const deviceMetadata: Metadata = {
+  type: TraceType.Testrunner,
+  device: { platform: 'ios', name: 'iPhone 17', version: '18.1' },
+  viewport: { ...viewport, width: 402, height: 874 }
+}
+
 /** Metadata whose viewport never made it onto the wire — the race the player
  *  defaults for. */
 export const viewportlessMetadata: Metadata = { type: TraceType.Testrunner }