diff --git a/client/dive-common/alignedTimeline.spec.ts b/client/dive-common/alignedTimeline.spec.ts index d0d0520fb..821c9b09e 100644 --- a/client/dive-common/alignedTimeline.spec.ts +++ b/client/dive-common/alignedTimeline.spec.ts @@ -1,6 +1,7 @@ import type { FrameImage } from './apispec'; import { - buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapGradient, computeGapSlots, + buildAlignedTimeline, buildInverseAlignedIndex, buildOffsetTimeline, canAlign, + computeGapGradient, computeGapSlots, } from './alignedTimeline'; function frame(timestamp?: number): FrameImage { @@ -209,3 +210,64 @@ describe('alignedTimeline', () => { }); }); }); + +/** + * Fixed-rig start offsets: the case timestamps can't cover, because video + * frames carry none. EO/IR pairs from the same fixed rig are recorded by + * independent encoders, so one can start a fraction of a second after the + * other; that lag is constant for the whole recording. + */ +describe('buildOffsetTimeline', () => { + it('pairs a later-starting camera with the reference instant', () => { + // B starts 2 frames later: B's frame 2 is the same instant as A's 0. + const result = buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 2 }); + if (!result.aligned) throw new Error('expected aligned'); + // Slot 0 is the earliest instant ANY camera saw: B's frame 0, before A began. + expect(result.slots[0]).toEqual({ A: undefined, B: 0 }); + expect(result.slots[2]).toEqual({ A: 0, B: 2 }); + expect(result.slots[4]).toEqual({ A: 2, B: 4 }); + }); + + it('keeps the union, blanking each camera outside its own coverage', () => { + const result = buildOffsetTimeline({ A: 3, B: 3 }, { A: 0, B: 2 }); + if (!result.aligned) throw new Error('expected aligned'); + // 2 leading slots before A starts + 3 shared + 0 trailing. + expect(result.slots).toHaveLength(5); + expect(computeGapSlots(result.slots)).toEqual([0, 1, 3, 4]); + // The overlap in the middle has both cameras. + expect(result.slots[2]).toEqual({ A: 0, B: 2 }); + }); + + it('round-trips through the inverse index the resolver uses', () => { + const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: 1 }); + if (!result.aligned) throw new Error('expected aligned'); + const inverse = buildInverseAlignedIndex(result.slots); + // Whatever slot holds A's frame 2 must hold B's frame 3 -- the same instant. + const slotForA2 = inverse.A.get(2) as number; + expect(result.slots[slotForA2].B).toBe(3); + expect(inverse.B.get(3)).toBe(slotForA2); + }); + + it('handles a negative offset (reference is the later camera)', () => { + const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: -1 }); + if (!result.aligned) throw new Error('expected aligned'); + expect(result.slots[1]).toEqual({ A: 1, B: 0 }); + }); + + it('declines when nothing needs correcting or there is no pair', () => { + // All-zero offsets: the positional path already does this, more cheaply. + expect(buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 0 })).toEqual({ aligned: false }); + // A camera with no frames loaded can't be aligned against. + expect(buildOffsetTimeline({ A: 5, B: 0 }, { A: 0, B: 2 })).toEqual({ aligned: false }); + expect(buildOffsetTimeline({ A: 5 }, { A: 3 })).toEqual({ aligned: false }); + }); + + it('treats a missing camera entry as no offset', () => { + // A is absent from the offsets map, so it behaves as A: 0 -- identical to + // { A: 0, B: 1 }: one leading slot for B's frame 0, then the pairs. + const result = buildOffsetTimeline({ A: 3, B: 3 }, { B: 1 }); + if (!result.aligned) throw new Error('expected aligned'); + expect(result.slots[0]).toEqual({ A: undefined, B: 0 }); + expect(result.slots[1]).toEqual({ A: 0, B: 1 }); + }); +}); diff --git a/client/dive-common/alignedTimeline.ts b/client/dive-common/alignedTimeline.ts index 750566003..08f3d4934 100644 --- a/client/dive-common/alignedTimeline.ts +++ b/client/dive-common/alignedTimeline.ts @@ -210,3 +210,71 @@ export function computeGapSlots(slots: AlignedSlot[]): number[] { }); return gaps; } + +/** + * A camera's constant start offset, in its own frames: local frame + * `slot + offset` shows the same instant as the reference camera's frame + * `slot`. A positive offset means this camera starts LATER -- its frame 0 + * happens before the reference's frame 0, so it must be read further in. + */ +export type CameraFrameOffsets = Record; + +/** + * Build a timeline from fixed per-camera start offsets rather than per-frame + * timestamps. + * + * buildAlignedTimeline needs a timestamp on every frame, which only image + * sequences carry (parsed from filenames) -- video panes never qualify, so + * they scrub in raw-index lockstep and any recording start offset between + * two cameras is baked into the review. On a fixed rig that offset is a + * single constant, so one number per camera is enough to line them up, and + * emitting it as slots means everything downstream (pane seek, gap + * indication, cross-camera frame translation) behaves exactly as it does + * for a timestamp-aligned dataset. + * + * Slots span the UNION of the cameras' coverage: where one camera has run + * out (or has not started), its entry is undefined, which the existing gap + * handling already renders and blanks correctly, rather than silently + * trimming footage off the ends. + * + * Returns { aligned: false } when fewer than two cameras have frames, or + * when every offset is zero -- there is nothing to correct then, so the + * caller should stay on the cheaper positional path. + */ +export function buildOffsetTimeline( + cameraFrameCounts: Record, + offsets: CameraFrameOffsets, +): TimelineResult { + const cameras = Object.keys(cameraFrameCounts) + .filter((camera) => cameraFrameCounts[camera] > 0); + if (cameras.length < 2) { + return { aligned: false }; + } + if (cameras.every((camera) => (offsets[camera] ?? 0) === 0)) { + return { aligned: false }; + } + // Slot s shows camera c's local frame s + offset[c]; that frame exists for + // s in [-offset[c], count[c] - offset[c]). Take the union across cameras, + // then rebase so the emitted slot array is 0-based. + const starts = cameras.map((camera) => -(offsets[camera] ?? 0)); + const ends = cameras.map( + (camera) => cameraFrameCounts[camera] - (offsets[camera] ?? 0), + ); + const base = Math.min(...starts); + const total = Math.max(...ends) - base; + if (total <= 0) { + return { aligned: false }; + } + const slots: AlignedSlot[] = new Array(total); + for (let index = 0; index < total; index += 1) { + const slot: AlignedSlot = {}; + cameras.forEach((camera) => { + const local = index + base + (offsets[camera] ?? 0); + slot[camera] = local >= 0 && local < cameraFrameCounts[camera] + ? local + : undefined; + }); + slots[index] = slot; + } + return { aligned: true, slots }; +} diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 977006bdd..4b4b79b0c 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -199,6 +199,19 @@ interface SaveDetectionsArgs { set?: string; } +/** Outcome of shifting one camera's stored annotations onto its time offset. */ +interface CameraFrameOffsetResult { + camera: string; + /** The camera's start offset in its own frames, now both stored and applied. */ + offset: number; + /** Frames the annotations actually moved: the offset minus what was already applied. */ + delta: number; + tracks: number; + groups: number; + /** Annotations that had nothing left before frame 0 and were deleted. */ + dropped: number; +} + interface SaveAttributeArgs { delete: string[]; upsert: Attribute[]; @@ -332,9 +345,13 @@ interface DatasetConfigMutable { * role are absent. */ cameraRoles?: Record; + /** Per-camera start offset in its own frames, for recorders that started at different times. */ + cameraFrameOffsets?: Record; + /** The part of cameraFrameOffsets already applied to each camera's annotations. */ + cameraFrameOffsetsApplied?: Record; error?: string; } -const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'typeHierarchy', 'taxonomySources', 'cameraRoles']; +const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'cameraFrameOffsets', 'cameraFrameOffsetsApplied', 'typeHierarchy', 'taxonomySources', 'cameraRoles']; /** * Cross-dataset color/style overrides, reused across every dataset when the * "shared" color scope is enabled (see clientSettings.typeSettings.colorScope). @@ -517,6 +534,13 @@ interface Api { saveDetections(datasetId: string, args: SaveDetectionsArgs): Promise; saveConfig(datasetId: string, config: DatasetConfigMutable): Promise; + /** + * Shift one camera's stored annotations onto its time offset, in persistence. + * Only the part not yet applied moves; the caller reloads the camera afterwards. + */ + applyCameraFrameOffset( + datasetId: string, camera: string, offset: number, + ): Promise; saveAttributes(datasetId: string, args: SaveAttributeArgs): Promise; saveAttributeTrackFilters(datasetId: string, args: SaveAttributeTrackFilterArgs): Promise; @@ -931,6 +955,7 @@ export type { PipeMetadata, Pipelines, SaveDetectionsArgs, + CameraFrameOffsetResult, SaveAttributeArgs, SaveAttributeTrackFilterArgs, TrainingConfig, diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 619426bce..60a89546c 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -648,6 +648,8 @@ export default defineComponent({ cameraCorrespondences: registration.observations.value, cameraTransformTypes: registration.transformTypes.value, cameraRegistrationSource: registration.source.value, + cameraFrameOffsets: registration.frameOffsets.value, + cameraFrameOffsetsApplied: registration.appliedFrameOffsets.value, }); registration.markSaved(); } finally { diff --git a/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts b/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts index 76bedc8c5..93737a126 100644 --- a/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts +++ b/client/dive-common/components/DatasetInfo/DatasetInfo.spec.ts @@ -84,6 +84,9 @@ function apiWithMetadata({ loadFrameMetadata: vi.fn(async () => frameMetadata), saveDetections: async () => undefined, saveConfig: async () => undefined, + applyCameraFrameOffset: async () => ({ + camera: '', offset: 0, delta: 0, tracks: 0, groups: 0, dropped: 0, + }), saveAttributes: async () => undefined, saveAttributeTrackFilters: async () => undefined, openFromDisk: async () => ({ canceled: true, filePaths: [] }), diff --git a/client/dive-common/components/ImportAnnotations.vue b/client/dive-common/components/ImportAnnotations.vue index 2b34a4082..4f8c9bf2f 100644 --- a/client/dive-common/components/ImportAnnotations.vue +++ b/client/dive-common/components/ImportAnnotations.vue @@ -340,6 +340,7 @@ export default defineComponent({ meta.cameraCorrespondences, meta.cameraTransformTypes, meta.cameraRegistrationSource, + meta.cameraFrameOffsets, ); if (priorPair) { // The panel is open: re-select the imported pair (falling back to diff --git a/client/dive-common/components/MultiCamTools.spec.ts b/client/dive-common/components/MultiCamTools.spec.ts new file mode 100644 index 000000000..a6ca9ccdc --- /dev/null +++ b/client/dive-common/components/MultiCamTools.spec.ts @@ -0,0 +1,55 @@ +import { defineComponent, h, ref } from 'vue'; +import { shallowMount } from '@vue/test-utils'; +import MultiCamTools from './MultiCamTools.vue'; + +const state = vi.hoisted(() => ({ + readOnlyMode: false, + offsetEditLock: false, +})); + +vi.mock('dive-common/apispec', () => ({ + useApi: () => ({ applyCameraFrameOffset: vi.fn() }), +})); + +vi.mock('vue-media-annotator/provides', () => ({ + useSelectedCamera: () => ref('left'), + useEditingMode: () => ref(false), + useTrackFilters: () => ({ enabledAnnotations: ref([]) }), + useHandler: () => ({ save: vi.fn(), reloadCameraAnnotations: vi.fn() }), + useTime: () => ({ frame: ref(0), frameRate: ref(30) }), + useSelectedTrackId: () => ref(null), + useCameraStore: () => ({ orderedCameraNames: () => ['left', 'right'] }), + useCameraRegistration: () => ({ + frameOffsets: ref({ right: 3 }), + appliedFrameOffsets: ref({}), + }), + useDatasetId: () => ref('dataset'), + usePendingSaveCount: () => ref(0), + useReadOnlyMode: () => ref(state.readOnlyMode), + useOffsetEditLock: () => ref(state.offsetEditLock), +})); + +function applyButton() { + const Host = defineComponent({ setup: () => () => h(MultiCamTools) }); + const wrapper = shallowMount(Host, { stubs: { MultiCamTools: false } }); + const button = wrapper.findAll('v-btn').wrappers + .find((b) => b.text().includes('Apply to annotations')); + if (!button) throw new Error('Apply to annotations button not rendered'); + return { wrapper, button }; +} + +it('keeps Apply enabled when editing is paused only by the pending offset', () => { + state.readOnlyMode = true; + state.offsetEditLock = true; + const { wrapper, button } = applyButton(); + expect(button.attributes('disabled')).toBeUndefined(); + expect(wrapper.text()).toContain('Annotation editing is paused until the offset is applied.'); +}); + +it('disables Apply in a truly read-only view', () => { + state.readOnlyMode = true; + state.offsetEditLock = false; + const { wrapper, button } = applyButton(); + expect(button.attributes('disabled')).toBeDefined(); + expect(wrapper.text()).not.toContain('Annotation editing is paused'); +}); diff --git a/client/dive-common/components/MultiCamTools.vue b/client/dive-common/components/MultiCamTools.vue index dec6e513c..3e1e9a6ca 100644 --- a/client/dive-common/components/MultiCamTools.vue +++ b/client/dive-common/components/MultiCamTools.vue @@ -1,9 +1,14 @@