Aligned multicam timeline for video, time offsets, and multicam fixes

client/dive-common/alignedTimeline.spec.ts

@@ -1,6 +1,7 @@
 import type { FrameImage } from './apispec';
 import {
-  buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapGradient, computeGapSlots,
+  buildAlignedTimeline, buildInverseAlignedIndex, buildOffsetTimeline, canAlign,
+  computeGapGradient, computeGapSlots,
 } from './alignedTimeline';
 
 function frame(timestamp?: number): FrameImage {
@@ -209,3 +210,64 @@ describe('alignedTimeline', () => {
     });
   });
 });
+
+/**
+ * Fixed-rig start offsets: the case timestamps can't cover, because video
+ * frames carry none. EO/IR pairs from the same fixed rig are recorded by
+ * independent encoders, so one can start a fraction of a second after the
+ * other; that lag is constant for the whole recording.
+ */
+describe('buildOffsetTimeline', () => {
+  it('pairs a later-starting camera with the reference instant', () => {
+    // B starts 2 frames later: B's frame 2 is the same instant as A's 0.
+    const result = buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 2 });
+    if (!result.aligned) throw new Error('expected aligned');
+    // Slot 0 is the earliest instant ANY camera saw: B's frame 0, before A began.
+    expect(result.slots[0]).toEqual({ A: undefined, B: 0 });
+    expect(result.slots[2]).toEqual({ A: 0, B: 2 });
+    expect(result.slots[4]).toEqual({ A: 2, B: 4 });
+  });
+
+  it('keeps the union, blanking each camera outside its own coverage', () => {
+    const result = buildOffsetTimeline({ A: 3, B: 3 }, { A: 0, B: 2 });
+    if (!result.aligned) throw new Error('expected aligned');
+    // 2 leading slots before A starts + 3 shared + 0 trailing.
+    expect(result.slots).toHaveLength(5);
+    expect(computeGapSlots(result.slots)).toEqual([0, 1, 3, 4]);
+    // The overlap in the middle has both cameras.
+    expect(result.slots[2]).toEqual({ A: 0, B: 2 });
+  });
+
+  it('round-trips through the inverse index the resolver uses', () => {
+    const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: 1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    const inverse = buildInverseAlignedIndex(result.slots);
+    // Whatever slot holds A's frame 2 must hold B's frame 3 -- the same instant.
+    const slotForA2 = inverse.A.get(2) as number;
+    expect(result.slots[slotForA2].B).toBe(3);
+    expect(inverse.B.get(3)).toBe(slotForA2);
+  });
+
+  it('handles a negative offset (reference is the later camera)', () => {
+    const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: -1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    expect(result.slots[1]).toEqual({ A: 1, B: 0 });
+  });
+
+  it('declines when nothing needs correcting or there is no pair', () => {
+    // All-zero offsets: the positional path already does this, more cheaply.
+    expect(buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 0 })).toEqual({ aligned: false });
+    // A camera with no frames loaded can't be aligned against.
+    expect(buildOffsetTimeline({ A: 5, B: 0 }, { A: 0, B: 2 })).toEqual({ aligned: false });
+    expect(buildOffsetTimeline({ A: 5 }, { A: 3 })).toEqual({ aligned: false });
+  });
+
+  it('treats a missing camera entry as no offset', () => {
+    // A is absent from the offsets map, so it behaves as A: 0 -- identical to
+    // { A: 0, B: 1 }: one leading slot for B's frame 0, then the pairs.
+    const result = buildOffsetTimeline({ A: 3, B: 3 }, { B: 1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    expect(result.slots[0]).toEqual({ A: undefined, B: 0 });
+    expect(result.slots[1]).toEqual({ A: 0, B: 1 });
+  });
+});

client/dive-common/alignedTimeline.ts

@@ -210,3 +210,71 @@ export function computeGapSlots(slots: AlignedSlot[]): number[] {
   });
   return gaps;
 }
+
+/**
+ * A camera's constant start offset, in its own frames: local frame
+ * `slot + offset` shows the same instant as the reference camera's frame
+ * `slot`. A positive offset means this camera starts LATER -- its frame 0
+ * happens before the reference's frame 0, so it must be read further in.
+ */
+export type CameraFrameOffsets = Record<string, number>;
+
+/**
+ * Build a timeline from fixed per-camera start offsets rather than per-frame
+ * timestamps.
+ *
+ * buildAlignedTimeline needs a timestamp on every frame, which only image
+ * sequences carry (parsed from filenames) -- video panes never qualify, so
+ * they scrub in raw-index lockstep and any recording start offset between
+ * two cameras is baked into the review. On a fixed rig that offset is a
+ * single constant, so one number per camera is enough to line them up, and
+ * emitting it as slots means everything downstream (pane seek, gap
+ * indication, cross-camera frame translation) behaves exactly as it does
+ * for a timestamp-aligned dataset.
+ *
+ * Slots span the UNION of the cameras' coverage: where one camera has run
+ * out (or has not started), its entry is undefined, which the existing gap
+ * handling already renders and blanks correctly, rather than silently
+ * trimming footage off the ends.
+ *
+ * Returns { aligned: false } when fewer than two cameras have frames, or
+ * when every offset is zero -- there is nothing to correct then, so the
+ * caller should stay on the cheaper positional path.
+ */
+export function buildOffsetTimeline(
+  cameraFrameCounts: Record<string, number>,
+  offsets: CameraFrameOffsets,
+): TimelineResult {
+  const cameras = Object.keys(cameraFrameCounts)
+    .filter((camera) => cameraFrameCounts[camera] > 0);
+  if (cameras.length < 2) {
+    return { aligned: false };
+  }
+  if (cameras.every((camera) => (offsets[camera] ?? 0) === 0)) {
+    return { aligned: false };
+  }
+  // Slot s shows camera c's local frame s + offset[c]; that frame exists for
+  // s in [-offset[c], count[c] - offset[c]). Take the union across cameras,
+  // then rebase so the emitted slot array is 0-based.
+  const starts = cameras.map((camera) => -(offsets[camera] ?? 0));
+  const ends = cameras.map(
+    (camera) => cameraFrameCounts[camera] - (offsets[camera] ?? 0),
+  );
+  const base = Math.min(...starts);
+  const total = Math.max(...ends) - base;
+  if (total <= 0) {
+    return { aligned: false };
+  }
+  const slots: AlignedSlot[] = new Array(total);
+  for (let index = 0; index < total; index += 1) {
+    const slot: AlignedSlot = {};
+    cameras.forEach((camera) => {
+      const local = index + base + (offsets[camera] ?? 0);
+      slot[camera] = local >= 0 && local < cameraFrameCounts[camera]
+        ? local
+        : undefined;
+    });
+    slots[index] = slot;
+  }
+  return { aligned: true, slots };
+}

client/dive-common/apispec.ts

@@ -199,6 +199,19 @@ interface SaveDetectionsArgs {
   set?: string;
 }
 
+/** Outcome of shifting one camera's stored annotations onto its time offset. */
+interface CameraFrameOffsetResult {
+  camera: string;
+  /** The camera's start offset in its own frames, now both stored and applied. */
+  offset: number;
+  /** Frames the annotations actually moved: the offset minus what was already applied. */
+  delta: number;
+  tracks: number;
+  groups: number;
+  /** Annotations that had nothing left before frame 0 and were deleted. */
+  dropped: number;
+}
+
 interface SaveAttributeArgs {
   delete: string[];
   upsert: Attribute[];
@@ -332,9 +345,13 @@ interface DatasetConfigMutable {
    * role are absent.
    */
   cameraRoles?: Record<string, CameraRole>;
+  /** Per-camera start offset in its own frames, for recorders that started at different times. */
+  cameraFrameOffsets?: Record<string, number>;
+  /** The part of cameraFrameOffsets already applied to each camera's annotations. */
+  cameraFrameOffsetsApplied?: Record<string, number>;
   error?: string;
 }
-const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'typeHierarchy', 'taxonomySources', 'cameraRoles'];
+const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'cameraFrameOffsets', 'cameraFrameOffsetsApplied', 'typeHierarchy', 'taxonomySources', 'cameraRoles'];
 /**
  * Cross-dataset color/style overrides, reused across every dataset when the
  * "shared" color scope is enabled (see clientSettings.typeSettings.colorScope).
@@ -517,6 +534,13 @@ interface Api {
 
   saveDetections(datasetId: string, args: SaveDetectionsArgs): Promise<unknown>;
   saveConfig(datasetId: string, config: DatasetConfigMutable): Promise<unknown>;
+  /**
+   * Shift one camera's stored annotations onto its time offset, in persistence.
+   * Only the part not yet applied moves; the caller reloads the camera afterwards.
+   */
+  applyCameraFrameOffset(
+    datasetId: string, camera: string, offset: number,
+  ): Promise<CameraFrameOffsetResult>;
   saveAttributes(datasetId: string, args: SaveAttributeArgs): Promise<unknown>;
   saveAttributeTrackFilters(datasetId: string,
     args: SaveAttributeTrackFilterArgs): Promise<unknown>;
@@ -931,6 +955,7 @@ export type {
   PipeMetadata,
   Pipelines,
   SaveDetectionsArgs,
+  CameraFrameOffsetResult,
   SaveAttributeArgs,
   SaveAttributeTrackFilterArgs,
   TrainingConfig,

client/dive-common/components/CameraRegistration/RegistrationTools.vue

@@ -648,6 +648,8 @@ export default defineComponent({
           cameraCorrespondences: registration.observations.value,
           cameraTransformTypes: registration.transformTypes.value,
           cameraRegistrationSource: registration.source.value,
+          cameraFrameOffsets: registration.frameOffsets.value,
+          cameraFrameOffsetsApplied: registration.appliedFrameOffsets.value,
         });
         registration.markSaved();
       } finally {

client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts

@@ -84,6 +84,9 @@ function apiWithMetadata({
     loadFrameMetadata: vi.fn(async () => frameMetadata),
     saveDetections: async () => undefined,
     saveConfig: async () => undefined,
+    applyCameraFrameOffset: async () => ({
+      camera: '', offset: 0, delta: 0, tracks: 0, groups: 0, dropped: 0,
+    }),
     saveAttributes: async () => undefined,
     saveAttributeTrackFilters: async () => undefined,
     openFromDisk: async () => ({ canceled: true, filePaths: [] }),

client/dive-common/components/ImportAnnotations.vue

@@ -340,6 +340,7 @@ export default defineComponent({
           meta.cameraCorrespondences,
           meta.cameraTransformTypes,
           meta.cameraRegistrationSource,
+          meta.cameraFrameOffsets,
         );
         if (priorPair) {
           // The panel is open: re-select the imported pair (falling back to

client/dive-common/components/MultiCamTools.spec.ts

@@ -0,0 +1,55 @@
+import { defineComponent, h, ref } from 'vue';
+import { shallowMount } from '@vue/test-utils';
+import MultiCamTools from './MultiCamTools.vue';
+
+const state = vi.hoisted(() => ({
+  readOnlyMode: false,
+  offsetEditLock: false,
+}));
+
+vi.mock('dive-common/apispec', () => ({
+  useApi: () => ({ applyCameraFrameOffset: vi.fn() }),
+}));
+
+vi.mock('vue-media-annotator/provides', () => ({
+  useSelectedCamera: () => ref('left'),
+  useEditingMode: () => ref(false),
+  useTrackFilters: () => ({ enabledAnnotations: ref([]) }),
+  useHandler: () => ({ save: vi.fn(), reloadCameraAnnotations: vi.fn() }),
+  useTime: () => ({ frame: ref(0), frameRate: ref(30) }),
+  useSelectedTrackId: () => ref(null),
+  useCameraStore: () => ({ orderedCameraNames: () => ['left', 'right'] }),
+  useCameraRegistration: () => ({
+    frameOffsets: ref({ right: 3 }),
+    appliedFrameOffsets: ref({}),
+  }),
+  useDatasetId: () => ref('dataset'),
+  usePendingSaveCount: () => ref(0),
+  useReadOnlyMode: () => ref(state.readOnlyMode),
+  useOffsetEditLock: () => ref(state.offsetEditLock),
+}));
+
+function applyButton() {
+  const Host = defineComponent({ setup: () => () => h(MultiCamTools) });
+  const wrapper = shallowMount(Host, { stubs: { MultiCamTools: false } });
+  const button = wrapper.findAll('v-btn').wrappers
+    .find((b) => b.text().includes('Apply to annotations'));
+  if (!button) throw new Error('Apply to annotations button not rendered');
+  return { wrapper, button };
+}
+
+it('keeps Apply enabled when editing is paused only by the pending offset', () => {
+  state.readOnlyMode = true;
+  state.offsetEditLock = true;
+  const { wrapper, button } = applyButton();
+  expect(button.attributes('disabled')).toBeUndefined();
+  expect(wrapper.text()).toContain('Annotation editing is paused until the offset is applied.');
+});
+
+it('disables Apply in a truly read-only view', () => {
+  state.readOnlyMode = true;
+  state.offsetEditLock = false;
+  const { wrapper, button } = applyButton();
+  expect(button.attributes('disabled')).toBeDefined();
+  expect(wrapper.text()).not.toContain('Annotation editing is paused');
+});