fix(app): fit a DOM-less capture by both axes

.changeset/player-fits-capture-by-both-axes.md

@@ -0,0 +1,7 @@
+---
+"@wdio/devtools-app": patch
+---
+
+Fit a capture with no DOM by both axes. The player's screenshot branch — reached by every trace that carries no mutation stream, so by every native mobile one — was bounded on the width alone inside a wrapper that hides its overflow, and the filmstrip drew each frame in a fixed 16:9 box with `object-cover`. A portrait capture was therefore scaled up to the pane width, overflowed its height, and had the remainder cut off, while its thumbnails were cropped to a horizontal band through the middle of the screen. Measured on a 1206x2622 iPhone 17 capture in a 1240x457 pane: the main pane showed 17% of the device screen at 5.9x magnification, and because that band is empty page on a phone app screen, the whole filmstrip rendered as blank white rectangles. The captured bytes were always correct — dragging the image out of the player showed the whole screen.
+
+This is not mobile-specific, it was only unmissable there: the same width-only fit cut the bottom 15% off a 1280x800 desktop capture in a 400px-tall pane. Both places now fit by the capture's own pixels, read from its PNG or JPEG header — the metadata viewport cannot serve, because it disagrees with the screenshot on both mobile platforms (Android reports the window without the navigation bar, iOS reports points rather than pixels) and a DOM-less trace carries no viewport at all. The main pane fills the pane and contains inside it, matching the screencast branch; a filmstrip thumbnail takes the capture's own aspect ratio and contains rather than covers, keeping the 16:9 box only for bytes that name no size.

packages/app/src/components/browser/snapshot-styles.ts

@@ -74,9 +74,15 @@ export const snapshotStyles = css`
     overflow: hidden;
   }
 
+  /* The capture fills the pane and contains inside it — the same fit as the
+     screencast branch. Bounding the width alone scaled a portrait capture up to
+     the pane width, overflowed its height, and the wrapper's overflow:hidden
+     clipped the remainder: a 1206x2622 phone screen showed 17% of itself at
+     5.9x in a 1240x457 pane. */
   .screenshot-overlay img {
-    max-width: 100%;
-    height: auto;
+    width: 100%;
+    height: 100%;
+    object-fit: contain;
     display: block;
   }
 

packages/app/src/components/browser/snapshot.ts

@@ -11,7 +11,6 @@ import {
 } from './element-overlay.js'
 import { commandPageUrl } from './url-at-timestamp.js'
 import { mutationForCommand } from './mutation-at-command.js'
-import { imageMime } from './trace-timeline-utils.js'
 import { booleanAttributeOn, isBooleanAttribute } from './boolean-attribute.js'
 
 import { type ComponentChildren, h, render, type VNode } from 'preact'
@@ -22,7 +21,7 @@ import type { SimplifiedVNode } from '@wdio/devtools-script/types'
 // characterData wire shape (parent ref + child index), so the replay reads it
 // from the same declaration that produces it.
 import type { TextMutation } from '@wdio/devtools-script/mutations.js'
-import type { CommandLog } from '@wdio/devtools-shared'
+import { imageMime, type CommandLog } from '@wdio/devtools-shared'
 
 import {
   mutationContext,

packages/app/src/components/browser/trace-timeline-utils.ts

@@ -3,12 +3,6 @@ import {
   TICK_TARGET_DIVISIONS
 } from './trace-timeline-constants.js'
 
-/** Detect image mime from a base64 string's magic bytes — trace screenshots
- *  may be PNG (polling capture) or JPEG (CDP), and the zip names both `.jpeg`. */
-export function imageMime(base64: string): string {
-  return base64.startsWith('/9j/') ? 'image/jpeg' : 'image/png'
-}
-
 export function tickStep(
   durationMs: number,
   targetTicks = TICK_TARGET_DIVISIONS

packages/app/src/components/browser/trace-timeline.ts

@@ -3,7 +3,11 @@ import { html, type TemplateResult } from 'lit'
 import { customElement, state, query } from 'lit/decorators.js'
 import { consume } from '@lit/context'
 import type { CommandLog, TracePlayerFrame } from '@wdio/devtools-shared'
-import { isKeyboardCommand } from '@wdio/devtools-shared'
+import {
+  imageDimensions,
+  imageMime,
+  isKeyboardCommand
+} from '@wdio/devtools-shared'
 
 import { commandContext, framesContext } from '../../controller/context.js'
 import { elapsedSince } from '../../utils/elapsed.js'
@@ -19,7 +23,6 @@ import {
 import {
   formatTickLabel,
   formatTimecode,
-  imageMime,
   tickStep
 } from './trace-timeline-utils.js'
 import { timelineStyles } from './trace-timeline-styles.js'
@@ -50,6 +53,7 @@ export class TraceTimeline extends Element {
   @query('[data-scrub]') scrubEl?: HTMLElement
 
   #dragging = false
+  #thumbAspectMemo?: { screenshot: string; aspect: string | null }
 
   static styles = [...Element.styles, timelineStyles]
 
@@ -338,6 +342,30 @@ export class TraceTimeline extends Element {
     `
   }
 
+  /** Shape of a filmstrip thumbnail, as a CSS `aspect-ratio`, or null when the
+   *  capture's bytes name no size. Read from the capture's own pixels rather
+   *  than the metadata viewport, which disagrees with the screenshot on both
+   *  mobile platforms. A fixed 16:9 box cropped (object-cover) a portrait
+   *  capture to a horizontal band through its middle — usually empty page, so
+   *  the whole strip rendered as blank rectangles. One ratio for the strip:
+   *  every frame of a run shares one capture surface, and a mid-run rotation
+   *  letterboxes inside the box instead of being cropped away. Memoized on the
+   *  frame it was read from, because this runs on every playback tick. */
+  get #thumbAspect(): string | null {
+    const screenshot = this.frames[0]?.screenshot
+    if (!screenshot) {
+      return null
+    }
+    if (this.#thumbAspectMemo?.screenshot !== screenshot) {
+      const size = imageDimensions(screenshot)
+      this.#thumbAspectMemo = {
+        screenshot,
+        aspect: size ? `${size.width} / ${size.height}` : null
+      }
+    }
+    return this.#thumbAspectMemo.aspect
+  }
+
   // Thumbnails sit at their wall-clock position along the axis.
   #renderThumbTrack(): TemplateResult {
     if (!this.frames.length) {
@@ -348,13 +376,16 @@ export class TraceTimeline extends Element {
       </div>`
     }
     const activeFrame = this.#activeFrameTimestamp
+    const aspect = this.#thumbAspect
     return html`
       <div class="relative flex-1 min-h-0">
         ${this.frames.map((frame) => {
           const fraction = this.#fraction(frame.timestamp)
           const active = frame.timestamp === activeFrame
           return html`<button
-            class="absolute top-0.5 bottom-0.5 aspect-video border rounded overflow-hidden hover:border-chartsBlue hover:z-10 ${
+            class="absolute top-0.5 bottom-0.5 border rounded overflow-hidden hover:border-chartsBlue hover:z-10 ${
+              aspect ? '' : 'aspect-video'
+            } ${
               active
                 ? `border-chartsBlue ring-1 ring-chartsBlue${
                     this.playing ? '' : ' z-10'
@@ -363,12 +394,12 @@ export class TraceTimeline extends Element {
             }"
             style="left:${fraction * 100}%; transform:translateX(-${
               fraction * 100
-            }%);"
+            }%);${aspect ? ` aspect-ratio:${aspect};` : ''}"
             title="${formatTimecode(frame.timestamp - this.#start)}"
             @click="${() => this.#seekToTimestamp(frame.timestamp)}"
           >
             <img
-              class="h-full w-full object-cover"
+              class="h-full w-full object-contain"
               src="data:${imageMime(
                 frame.screenshot
               )};base64,${frame.screenshot}"

packages/app/test-ui/workbench/player/fixtures.ts

@@ -42,6 +42,20 @@ export const SECURE_SHOT =
 export const FRAME_SHOT =
   'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP48OEDAAWkAtFkkTHCAAAAAElFTkSuQmCC'
 
+/** A capture taller than it is wide — the shape every phone screenshot has, and
+ *  the one a width-only fit blew up past the pane. Its own pixels are what the
+ *  player fits by, so the specs read this ratio back off the layout. Both of
+ *  these are larger than the panes the specs give them, or a fit that ignores
+ *  an axis still leaves them inside it and nothing is proved. */
+export const PORTRAIT_SHOT =
+  'iVBORw0KGgoAAAANSUhEUgAAAHgAAAEECAIAAABskWeLAAABX0lEQVR42u3QMQ0AAAgDsMlBE4qRhYtdTaqgmT0KokC0aESLFm1BtGhEixZtQbRoRIsWjWjRiBYtGtGiES1aNKJFI1q0aESLRrRo0YgWjWjRohEtGtGiRSNaNKJFi0a0aESLFo1o0YgWLRrRohEtWjSiRSNatGhEi0a0aNGIFo1o0aIRLRrRokUjWjSiRYtGtGhEixaNaNGIFi0a0aIRLVo0okUjWrRoRItGtGjRiBaNaNGiES0a0aJFI1o0okWLRrRoRIsWjWjRiBYtGtGiES1aNKJFI1q0aESLRrRo0YgWjWjRohEtGtGiRSNaNKJFi0a0aESLFo1o0YgWLRrRohEtWjSiRSNatGhEi0a0aNGIFo1o0aIRLRrRokUjWjSiRYtGtGhEixaNaNGIFi0a0aIRLVo0okUjWrRoRItGtGjRiBaNaNGiES0a0aJFI1o0okWLRrRoBR1PvtEIvcLLfg8AAAAASUVORK5CYII='
+export const PORTRAIT_CAPTURE = { width: 120, height: 260 }
+
+/** A capture wider than the pane — the axis a width-only fit already bounded,
+ *  so the regression guard for the other direction. 320x200. */
+export const LANDSCAPE_SHOT =
+  'iVBORw0KGgoAAAANSUhEUgAAAUAAAADICAIAAAAWZq/8AAABvUlEQVR42u3TQQkAAAgEwYtjJhMbyxC+hIFJsLCZLuCpSAAGBgwMGBgMDBgYMDBgYDAwYGDAwGBgwMCAgQEDg4EBAwMGBgwMBgYMDBgYDAwYGDAwYGAwMGBgwMCAgcHAgIEBA4OBAQMDBgYMDAYGDAwYGAysAhgYMDBgYDAwYGDAwICBwcCAgQEDg4EBAwMGBgwMBgYMDBgYMDAYGDAwYGAwMGBgwMCAgcHAgIEBAwMGBgMDBgYMDAYGDAwYGDAwGBgwMGBgMDBgYMDAgIHBwICBAQMDBgYDAwYGDAwGBgwMGBgwMBgYMDBgYMDAYGDAwICBwcCAgQEDAwYGAwMGBgwMGBgMDBgYMDAYGDAwYGDAwGBgwMCAgcHAgIEBAwMGBgMDBgYMDBgYDAwYGDAwGBgwMGBgwMBgYMDAgIEBA4OBAQMDBgYDAwYGDAwYGAwMGBgwMBhYBTAwYGDAwGBgwMCAgQEDg4EBAwMGBgMDBgYMDBgYDAwYGDAwYGAwMGBgwMBgYMDAgIEBA4OBAQMDBgYMDAYGDAwYGAwMGBgwMGBgMDBgYMDAYGDAwICBAQODgQEDAwYGDAwGBgwMXCzP6VbflJe/hgAAAABJRU5ErkJggg=='
+
 /** The two streams one player mount consumes. */
 export interface TraceScenario {
   commands: CommandLog[]
@@ -474,6 +488,25 @@ export const domlessTrace: DomlessTrace = {
   assertFlash
 }
 
+/** The DOM-less branch carrying one capture — a native mobile trace, and any
+ *  tall capture: no mutation stream to replay, so the screenshot is all the
+ *  player has to fit. */
+const capturedTrace = (screenshot: string): TraceScenario => ({
+  commands: [
+    commandLog({
+      command: 'url',
+      args: [LOGIN_URL],
+      screenshot,
+      startTime: RUN_START,
+      timestamp: RUN_START + 400
+    })
+  ],
+  mutations: []
+})
+
+export const portraitTrace = capturedTrace(PORTRAIT_SHOT)
+export const landscapeTrace = capturedTrace(LANDSCAPE_SHOT)
+
 /**
  * A document anchor is a childList of one added node with no target; this one
  * carries no url, which is what makes the player fall back for the address bar.

packages/app/test-ui/workbench/player/snapshot.test.ts

@@ -45,6 +45,7 @@ import {
 import {
   CAPTURED_VIEWPORT,
   domlessTrace,
+  landscapeTrace,
   LOGIN_SHOT,
   LOGIN_URL,
   loginTrace,
@@ -53,6 +54,7 @@ import {
   metadataForViewport,
   orphanTrace,
   overlayLabelTrace,
+  portraitTrace,
   preCaptureTrace,
   RECORDING,
   recordedSessionMetadata,
@@ -72,6 +74,8 @@ import {
 const TAG = 'wdio-devtools-browser'
 const ADDRESS_BAR = 'header .truncate'
 const SCREENSHOT = '.screenshot-overlay img'
+/** Box the capture is fitted into, and the one that clips whatever overflows. */
+const SCREENSHOT_PANE = '.screenshot-overlay'
 const PLACEHOLDER = 'wdio-devtools-placeholder'
 const SCREENCAST = 'wdio-devtools-screencast-player'
 const VIEW_BUTTON = '.view-toggle button'
@@ -1453,6 +1457,86 @@ describe('wdio-devtools-browser', () => {
       expect(iframe.style.height).toBe('800px')
       expect(scaleOf(iframe)).toBeCloseTo(0.5, 5)
     })
+
+    /**
+     * The screenshot branch — every DOM-less trace, so every native mobile one —
+     * is fitted by CSS rather than by the sizing pass above, and it was bounded
+     * on the width alone. A portrait capture was scaled up to the pane's width,
+     * overflowed its height, and the overflow:hidden wrapper clipped the rest: a
+     * 1206x2622 phone screen showed 17% of itself at 5.9x in a 1240x457 pane.
+     */
+    describe('fitting a capture with no DOM to replay', () => {
+      /** Sizes the pane and waits for the capture to take a box inside it. No
+       *  transform to wait on here — the fit is the stylesheet's, not the
+       *  player's — so the wait is for the image to have laid out. */
+      async function resizeScreenshotPane(
+        el: Browser,
+        width: number,
+        height: number
+      ): Promise<HTMLImageElement> {
+        const host = el.parentElement
+        if (!host) {
+          throw new Error('the mounted player has no pane to size')
+        }
+        host.style.width = `${width}px`
+        host.style.height = `${height}px`
+        const img = shadow<HTMLImageElement>(el, SCREENSHOT)
+        if (!img) {
+          throw new Error('the player rendered no screenshot')
+        }
+        await waitUntil(
+          () => img.complete && img.getBoundingClientRect().height > 0,
+          'the capture to be laid out in the pane'
+        )
+        return img
+      }
+
+      /** The pane the capture is fitted into, and the box that clips it. */
+      const clipRect = (el: Browser) =>
+        shadow(el, SCREENSHOT_PANE)!.getBoundingClientRect()
+
+      it('holds a portrait capture inside both axes of the pane', async () => {
+        const el = await mountBrowser(portraitTrace)
+        await settle(el)
+
+        const img = await resizeScreenshotPane(
+          el,
+          ...paneFor(el, { w: 400, h: 200 })
+        )
+
+        // Read against the pane rather than the numbers above, which the
+        // player's own chrome and padding eat into. On the width alone this
+        // 120x260 capture took the full pane width and more than four times its
+        // height, and everything past the fold was cut off.
+        const pane = clipRect(el)
+        const box = img.getBoundingClientRect()
+        expect(box.height).toBeLessThanOrEqual(pane.height + 1)
+        expect(box.width).toBeLessThanOrEqual(pane.width + 1)
+        expect(box.height).toBeCloseTo(pane.height, 0)
+        // The painted rect has no DOM box of its own to measure; `contain` is
+        // what letterboxes the capture inside the box asserted above, at its own
+        // shape, rather than cropping it to fill.
+        expect(getComputedStyle(img).objectFit).toBe('contain')
+      })
+
+      it('holds a capture wider than the pane inside it too', async () => {
+        const el = await mountBrowser(landscapeTrace)
+        await settle(el)
+
+        // The axis the old rule did bound: a fit that swapped to the height
+        // alone draws this 320x200 capture 640px wide in a 400px pane.
+        const img = await resizeScreenshotPane(
+          el,
+          ...paneFor(el, { w: 400, h: 400 })
+        )
+
+        const pane = clipRect(el)
+        const box = img.getBoundingClientRect()
+        expect(box.width).toBeLessThanOrEqual(pane.width + 1)
+        expect(box.height).toBeLessThanOrEqual(pane.height + 1)
+        expect(box.width).toBeCloseTo(pane.width, 0)
+      })
+    })
   })
 
   describe('address-bar fallbacks', () => {

packages/app/test-ui/workbench/player/trace-timeline.test.ts

@@ -17,7 +17,13 @@ import '@components/browser/trace-timeline.js'
 
 import { mountWithContext, settle } from '../../support/mount.js'
 import { shadow, shadowAll, text, texts } from '../../support/queries.js'
-import { filmstrip, FRAME_SHOT, loginTrace } from './fixtures.js'
+import {
+  filmstrip,
+  FRAME_SHOT,
+  loginTrace,
+  PORTRAIT_CAPTURE,
+  PORTRAIT_SHOT
+} from './fixtures.js'
 
 const TAG = 'wdio-devtools-trace-timeline'
 const STRIP = '[data-scrub]'
@@ -175,6 +181,74 @@ describe('wdio-devtools-trace-timeline', () => {
       expect(thumbs[0].style.left).toBe('0%')
       expect(isActive(thumbs[0])).toBe(true)
     })
+
+    /**
+     * A thumbnail box is filled by the frame, so its shape decides what survives:
+     * in a fixed 16:9 box a portrait capture was cropped to a horizontal band
+     * through its middle — empty page on a phone screen, so the strip rendered as
+     * blank rectangles. The shape is read from the capture's own pixels, which is
+     * also all a DOM-less trace has: it carries no viewport, and on mobile the
+     * viewport disagrees with the screenshot anyway.
+     */
+    describe('thumbnail shape', () => {
+      /** Height for the strip to lay out in — the component is auto-height, and
+       *  a collapsed strip makes every ratio below 0/0. */
+      const STRIP_HEIGHT = 200
+
+      const shapeOf = (thumb: HTMLElement) => {
+        const box = thumb.getBoundingClientRect()
+        return box.width / box.height
+      }
+
+      async function laidOutStrip(
+        stripFrames: TracePlayerFrame[]
+      ): Promise<Timeline> {
+        const el = await mountTimeline(commands, stripFrames)
+        el.style.height = `${STRIP_HEIGHT}px`
+        el.style.width = '600px'
+        await settle(el)
+        return el
+      }
+
+      const reshot = (screenshot: string) =>
+        frames.map((frame) => ({ ...frame, screenshot }))
+
+      /** Base64 that decodes to bytes no size can be read from. It has to be
+       *  real base64: raw text makes the frame's `data:` url unparseable, and
+       *  Chrome logs that as a SEVERE resource error — which the runner polls
+       *  for every 500ms and fails the whole spec on, whichever test is running
+       *  by then. */
+      const SIZELESS_SHOT = btoa('not-an-image')
+
+      it("takes the capture's own shape", async () => {
+        const el = await laidOutStrip(reshot(PORTRAIT_SHOT))
+
+        const portrait = PORTRAIT_CAPTURE.width / PORTRAIT_CAPTURE.height
+        for (const thumb of shadowAll(el, THUMB)) {
+          expect(shapeOf(thumb)).toBeCloseTo(portrait, 2)
+        }
+      })
+
+      it('keeps nothing cropped away', async () => {
+        const el = await laidOutStrip(reshot(PORTRAIT_SHOT))
+        const thumb = shadowAll(el, THUMB)[0]
+        const img = shadow<HTMLImageElement>(thumb, 'img')!
+
+        // The frame fills its box rather than being covered into it, so the box
+        // holding the capture's shape means the whole capture is on screen.
+        expect(getComputedStyle(img).objectFit).toBe('contain')
+        expect(img.getBoundingClientRect().height).toBeCloseTo(
+          thumb.getBoundingClientRect().height,
+          1
+        )
+      })
+
+      it('falls back to 16:9 when the bytes name no size', async () => {
+        const el = await laidOutStrip(reshot(SIZELESS_SHOT))
+
+        expect(shapeOf(shadowAll(el, THUMB)[0])).toBeCloseTo(16 / 9, 2)
+      })
+    })
   })
 
   describe('action marks', () => {