From 9db4c817e41149e1a508fd6c45b570c805ac8d8e Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Fri, 23 Oct 2026 17:54:05 -0400 Subject: [PATCH] Add --merge-markers-from and --only-keep-processes-with-name-prefix to profiler-edit. This replaces the merge-android-profiles script. It combines the samples of a system-wide simpleperf profile with the markers of a Gecko profile from the same run, for Android startup profiling. The markers are copied into the threads with the same pid and tid, and the profiles are aligned by their monotonic clock start times. The markers of Gecko's "X (JVM)" threads go into the thread of "X". --- src/node-tools/profiler-edit.ts | 98 +++++++++--- src/profile-logic/merge-markers.ts | 231 ++++++++++++++++++++++++++++ src/test/unit/merge-markers.test.ts | 121 +++++++++++++++ src/test/unit/profiler-edit.test.ts | 21 +++ 4 files changed, 453 insertions(+), 18 deletions(-) create mode 100644 src/profile-logic/merge-markers.ts create mode 100644 src/test/unit/merge-markers.test.ts diff --git a/src/node-tools/profiler-edit.ts b/src/node-tools/profiler-edit.ts index a525b77e99..a46a60afd1 100644 --- a/src/node-tools/profiler-edit.ts +++ b/src/node-tools/profiler-edit.ts @@ -51,6 +51,7 @@ import { type SourceReader, } from 'firefox-profiler/profile-logic/embed-js-sources'; import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/source-map'; +import { mergeMarkersFromProfile } from 'firefox-profiler/profile-logic/merge-markers'; /** * A CLI tool for editing profiles. @@ -81,6 +82,10 @@ import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/so * * node node-tools-dist/profiler-edit.js -i samply-profile.json -o out.json.gz \ * --js-sources-from-func-names --map-source-paths mapping.json + * + * node node-tools-dist/profiler-edit.js -i simpleperf-profile.json \ + * -o merged.json.gz --merge-markers-from gecko-profile.json \ + * --only-keep-processes-with-name-prefix org.mozilla.fenix */ export type ProfileSource = @@ -102,6 +107,8 @@ export interface WasmSymbolicationCliSpec { export interface CliOptions { input: ProfileSource; output: string; + mergeMarkersFrom?: ProfileSource; + onlyKeepProcessesWithNamePrefix?: string; symbolicateWithServer?: string; symbolicateWasm: WasmSymbolicationCliSpec[]; mapSourcePaths?: string; @@ -339,9 +346,54 @@ async function loadProfile(source: ProfileSource): Promise { } } +function keepThreads( + profile: Profile, + shouldKeep: (thread: RawThread, threadIndex: ThreadIndex) => boolean +): Profile { + const oldThreadIndexToNew = new Map(); + const keptThreads: RawThread[] = []; + profile.threads.forEach((thread, oldIndex) => { + if (shouldKeep(thread, oldIndex)) { + oldThreadIndexToNew.set(oldIndex, keptThreads.length); + keptThreads.push(thread); + } + }); + return { + ...profile, + threads: keptThreads, + ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew), + }; +} + export async function run(options: CliOptions) { let profile = await loadProfile(options.input); + const processNamePrefix = options.onlyKeepProcessesWithNamePrefix; + if (processNamePrefix !== undefined) { + const before = profile.threads.length; + profile = keepThreads( + profile, + (thread) => thread.processName?.startsWith(processNamePrefix) ?? false + ); + console.log( + `Kept ${profile.threads.length} of ${before} threads in processes whose name starts with ${JSON.stringify(processNamePrefix)}.` + ); + } + + if (options.mergeMarkersFrom !== undefined) { + const markersProfile = await loadProfile(options.mergeMarkersFrom); + const result = mergeMarkersFromProfile(profile, markersProfile); + profile = result.profile; + console.log( + `Merged the markers of ${result.mergedThreadNames.length} threads, aligning the profiles with ${result.usedClock}.` + ); + if (result.unmatchedThreadNames.length !== 0) { + console.log( + `Dropped the markers of ${result.unmatchedThreadNames.length} threads without a matching pid and tid: ${result.unmatchedThreadNames.join(', ')}` + ); + } + } + if (options.symbolicateWithServer !== undefined) { const server = options.symbolicateWithServer; const symbolStore = new SymbolStore({ @@ -434,19 +486,9 @@ export async function run(options: CliOptions) { profile, options.onlyKeepThreadsWithMarkersMatching ); - const oldThreadIndexToNew = new Map(); - const matchingThreads: RawThread[] = []; - profile.threads.forEach((thread, oldIndex) => { - if (matchingThreadIndexes.has(oldIndex)) { - oldThreadIndexToNew.set(oldIndex, matchingThreads.length); - matchingThreads.push(thread); - } - }); - profile = { - ...profile, - threads: matchingThreads, - ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew), - }; + profile = keepThreads(profile, (_thread, threadIndex) => + matchingThreadIndexes.has(threadIndex) + ); console.log( `Kept ${profile.threads.length} of ${before} threads with markers matching ${JSON.stringify(options.onlyKeepThreadsWithMarkersMatching)}.` ); @@ -493,6 +535,12 @@ function collectWasm( return [...previous, { unstrippedWasmPath: value }]; } +function parseFileOrUrl(value: string): ProfileSource { + return /^https?:\/\//i.test(value) + ? { type: 'URL', url: value } + : { type: 'FILE', path: value }; +} + function requireNonEmpty(flagName: string): (value: string) => string { return (value: string) => { if (value === '') { @@ -516,6 +564,16 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { .option('--from-file ', 'Load input from a file') .option('--from-url ', 'Load input from a URL') .option('--from-hash ', 'Load input from a profile hash') + .option( + '--only-keep-processes-with-name-prefix ', + 'Keep only the threads of processes whose name starts with this prefix', + requireNonEmpty('--only-keep-processes-with-name-prefix') + ) + .option( + '--merge-markers-from ', + 'Copy the markers of this profile into the threads with the same pid and tid, e.g. from a Gecko profile into a simpleperf profile of the same run', + requireNonEmpty('--merge-markers-from') + ) .option( '--symbolicate-with-server ', 'Symbolicate frames using this symbol server URL' @@ -556,11 +614,7 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { const sources: ProfileSource[] = []; if (typeof opts.input === 'string' && opts.input !== '') { - if (/^https?:\/\//i.test(opts.input)) { - sources.push({ type: 'URL', url: opts.input }); - } else { - sources.push({ type: 'FILE', path: opts.input }); - } + sources.push(parseFileOrUrl(opts.input)); } if (typeof opts.fromFile === 'string' && opts.fromFile !== '') { sources.push({ type: 'FILE', path: opts.fromFile }); @@ -590,6 +644,14 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { return { input: sources[0], output: opts.output, + mergeMarkersFrom: + typeof opts.mergeMarkersFrom === 'string' + ? parseFileOrUrl(opts.mergeMarkersFrom) + : undefined, + onlyKeepProcessesWithNamePrefix: + typeof opts.onlyKeepProcessesWithNamePrefix === 'string' + ? opts.onlyKeepProcessesWithNamePrefix + : undefined, symbolicateWithServer: typeof opts.symbolicateWithServer === 'string' && opts.symbolicateWithServer !== '' diff --git a/src/profile-logic/merge-markers.ts b/src/profile-logic/merge-markers.ts new file mode 100644 index 0000000000..60a5f5436d --- /dev/null +++ b/src/profile-logic/merge-markers.ts @@ -0,0 +1,231 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { adjustMarkerTimestamps } from './process-profile'; +import { computeStringIndexMarkerFieldsByDataType } from './marker-schema'; +import { + getRawMarkerTableBuilderFromExisting, + finishRawMarkerTableBuilder, +} from './data-structures'; +import { StringTable } from 'firefox-profiler/utils/string-table'; + +import type { + IndexIntoCategoryList, + MarkerPayload, + Milliseconds, + Pid, + Profile, + ProfileMeta, + RawThread, + Tid, +} from 'firefox-profiler/types'; + +const PLATFORM_SPECIFIC_START_TIME_KEYS = [ + 'startTimeAsClockMonotonicNanosecondsSinceBoot', + 'startTimeAsMachAbsoluteTimeNanoseconds', +] as const; + +/** + * Compute the value that needs to be added to a timestamp of the source + * profile to get the timestamp of the same moment in the target profile. + * + * The platform-specific start times use the same clock in both profiles, so + * they give an exact result. The wall-clock start times are only a fallback. + */ +export function computeTimeDeltaBetweenProfiles( + sourceMeta: ProfileMeta, + targetMeta: ProfileMeta +): { delta: Milliseconds; usedClock: string } { + for (const key of PLATFORM_SPECIFIC_START_TIME_KEYS) { + const sourceStart = sourceMeta[key]; + const targetStart = targetMeta[key]; + if (sourceStart !== undefined && targetStart !== undefined) { + return { delta: (sourceStart - targetStart) / 1e6, usedClock: key }; + } + } + return { + delta: sourceMeta.startTime - targetMeta.startTime, + usedClock: 'startTime', + }; +} + +function threadKey(pid: Pid, tid: Tid): string { + return `${pid}:${tid}`; +} + +const JVM_THREAD_SUFFIX = ' (JVM)'; + +/** + * Group the source threads with markers by the key of the target thread they + * belong to. + * + * The Gecko profiler puts the Java markers of an Android thread "X" into a + * separate thread "X (JVM)", whose tid is the Java thread ID. These markers + * belong to the native thread named "X" in the same process. + */ +function groupSourceThreadsByKey( + threads: RawThread[] +): Map { + const tidByProcessAndName = new Map(); + for (const thread of threads) { + if (!thread.name.endsWith(JVM_THREAD_SUFFIX)) { + tidByProcessAndName.set(`${thread.pid}:${thread.name}`, thread.tid); + } + } + + const threadsByKey = new Map(); + for (const thread of threads) { + if (thread.markers.length === 0) { + continue; + } + let tid: Tid | undefined = thread.tid; + if (thread.name.endsWith(JVM_THREAD_SUFFIX)) { + const nativeName = thread.name.slice(0, -JVM_THREAD_SUFFIX.length); + tid = tidByProcessAndName.get(`${thread.pid}:${nativeName}`) ?? tid; + } + const key = threadKey(thread.pid, tid); + const group = threadsByKey.get(key); + if (group === undefined) { + threadsByKey.set(key, [thread]); + } else { + group.push(thread); + } + } + return threadsByKey; +} + +export type MergeMarkersResult = { + profile: Profile; + usedClock: string; + // Names of the source threads whose markers were copied. + mergedThreadNames: string[]; + // Names of the source threads with markers which had no matching thread in + // the target profile. + unmatchedThreadNames: string[]; +}; + +/** + * Copy the markers of the source profile into the target profile, into the + * threads which have the same pid and tid. Source threads without such a + * thread in the target profile are dropped. The markers of Gecko's "X (JVM)" + * threads go into the thread of "X". + * + * This is used for Android startup profiles, which combine the samples of a + * system-wide simpleperf profile with the markers of a Gecko profile from the + * same run. + */ +export function mergeMarkersFromProfile( + target: Profile, + source: Profile +): MergeMarkersResult { + const { delta, usedClock } = computeTimeDeltaBetweenProfiles( + source.meta, + target.meta + ); + + const markerSchema = target.meta.markerSchema.slice(); + const schemaNames = new Set(markerSchema.map((schema) => schema.name)); + for (const schema of source.meta.markerSchema) { + if (!schemaNames.has(schema.name)) { + schemaNames.add(schema.name); + markerSchema.push(schema); + } + } + + const categories = (target.meta.categories ?? []).slice(); + const sourceCategories = source.meta.categories ?? []; + const sourceCategoryToTarget: IndexIntoCategoryList[] = sourceCategories.map( + (category) => { + const index = categories.findIndex((c) => c.name === category.name); + if (index !== -1) { + return index; + } + categories.push(category); + return categories.length - 1; + } + ); + + const stringArray = target.shared.stringArray.slice(); + const stringTable = StringTable.withBackingArray(stringArray); + const sourceStringArray = source.shared.stringArray; + const stringIndexFieldsByDataType = computeStringIndexMarkerFieldsByDataType( + source.meta.markerSchema + ); + + function translateData(data: MarkerPayload | null): MarkerPayload | null { + if (data === null) { + return null; + } + const newData: any = { ...data }; + // The stack indexes point into the source profile's stack table. + delete newData.cause; + const stringIndexFields = data.type + ? stringIndexFieldsByDataType.get(data.type) + : undefined; + if (stringIndexFields !== undefined) { + for (const key of stringIndexFields) { + if (typeof newData[key] === 'number') { + newData[key] = stringTable.indexForString( + sourceStringArray[newData[key]] + ); + } + } + } + return newData; + } + + const sourceThreadsByKey = groupSourceThreadsByKey(source.threads); + + const mergedThreadNames: string[] = []; + const threads = target.threads.map((thread) => { + const key = threadKey(thread.pid, thread.tid); + const sourceThreads = sourceThreadsByKey.get(key); + if (sourceThreads === undefined) { + return thread; + } + sourceThreadsByKey.delete(key); + + const builder = getRawMarkerTableBuilderFromExisting(thread.markers); + for (const sourceThread of sourceThreads) { + mergedThreadNames.push(sourceThread.name); + const sourceMarkers = adjustMarkerTimestamps(sourceThread.markers, delta); + for (let i = 0; i < sourceMarkers.length; i++) { + builder.data.push(translateData(sourceMarkers.data[i])); + builder.name.push( + stringTable.indexForString(sourceStringArray[sourceMarkers.name[i]]) + ); + builder.startTime.push(sourceMarkers.startTime[i]); + builder.endTime.push(sourceMarkers.endTime[i]); + builder.phase.push(sourceMarkers.phase[i]); + builder.category.push( + sourceCategoryToTarget[sourceMarkers.category[i]] ?? 0 + ); + if (builder.threadId !== undefined) { + builder.threadId.push(thread.tid); + } + builder.length++; + } + } + return { ...thread, markers: finishRawMarkerTableBuilder(builder) }; + }); + + const profile: Profile = { + ...target, + meta: { ...target.meta, markerSchema, categories }, + shared: { ...target.shared, stringArray }, + threads, + }; + if (source.pages !== undefined) { + profile.pages = [...(target.pages ?? []), ...source.pages]; + } + + return { + profile, + usedClock, + mergedThreadNames, + unmatchedThreadNames: [...sourceThreadsByKey.values()] + .flat() + .map((thread) => thread.name), + }; +} diff --git a/src/test/unit/merge-markers.test.ts b/src/test/unit/merge-markers.test.ts new file mode 100644 index 0000000000..e98f4b72de --- /dev/null +++ b/src/test/unit/merge-markers.test.ts @@ -0,0 +1,121 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mergeMarkersFromProfile } from '../../profile-logic/merge-markers'; +import { getProfileWithMarkers } from '../fixtures/profiles/processed-profile'; +import { StringTable } from '../../utils/string-table'; + +import type { Profile, RawThread } from 'firefox-profiler/types'; + +function setThreadIds( + profile: Profile, + ids: Array<{ pid: string; tid: number; name: string }> +) { + profile.threads.forEach((thread, i) => { + Object.assign(thread, ids[i]); + }); +} + +function getMarkerNamesAndTimes(profile: Profile, thread: RawThread) { + const { markers } = thread; + const result = []; + for (let i = 0; i < markers.length; i++) { + result.push([ + profile.shared.stringArray[markers.name[i]], + markers.startTime[i], + ]); + } + return result; +} + +describe('mergeMarkersFromProfile', function () { + function setup() { + const target = getProfileWithMarkers([['TargetMarker', 1]], [], []); + setThreadIds(target, [ + { pid: '100', tid: 100, name: 'org.mozilla.fenix' }, + { pid: '100', tid: 101, name: 'Gecko' }, + { pid: '200', tid: 200, name: 'org.mozilla.fenix:tab' }, + ]); + target.meta.startTime = 1000; + target.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 5e9; + + const source = getProfileWithMarkers( + [['GeckoMarker', 2]], + [['UIMarker', 3]], + [['JavaMarker', 4]], + [['OrphanMarker', 5]] + ); + setThreadIds(source, [ + { pid: '100', tid: 101, name: 'GeckoMain' }, + { pid: '100', tid: 100, name: 'AndroidUI' }, + { pid: '100', tid: 1, name: 'AndroidUI (JVM)' }, + { pid: '300', tid: 300, name: 'Orphan' }, + ]); + // The source profile starts 10ms after the target profile. + source.meta.startTime = 1500; + source.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 5.01e9; + return { target, source }; + } + + it('copies the markers into the threads with the same pid and tid', function () { + const { target, source } = setup(); + const { profile, usedClock, unmatchedThreadNames } = + mergeMarkersFromProfile(target, source); + + expect(usedClock).toBe('startTimeAsClockMonotonicNanosecondsSinceBoot'); + expect(getMarkerNamesAndTimes(profile, profile.threads[0])).toEqual([ + ['TargetMarker', 1], + ['UIMarker', 13], + ['JavaMarker', 14], + ]); + expect(getMarkerNamesAndTimes(profile, profile.threads[1])).toEqual([ + ['GeckoMarker', 12], + ]); + expect(profile.threads[2].markers.length).toBe(0); + expect(unmatchedThreadNames).toEqual(['Orphan']); + }); + + it('falls back to the wall-clock start time', function () { + const { target, source } = setup(); + delete target.meta.startTimeAsClockMonotonicNanosecondsSinceBoot; + const { profile, usedClock } = mergeMarkersFromProfile(target, source); + + expect(usedClock).toBe('startTime'); + expect(getMarkerNamesAndTimes(profile, profile.threads[1])).toEqual([ + ['GeckoMarker', 502], + ]); + }); + + it('translates the string indexes in marker payloads', function () { + const { target, source } = setup(); + const sourceStrings = StringTable.withBackingArray( + source.shared.stringArray + ); + source.threads[0].markers.data[0] = { + type: 'Log', + module: 'nsHttp', + name: 'log', + level: sourceStrings.indexForString('Debug'), + message: sourceStrings.indexForString('Hello'), + cause: { tid: 101, time: 2, stack: 0 }, + } as any; + + const { profile } = mergeMarkersFromProfile(target, source); + const { markers } = profile.threads[1]; + const data = markers.data[0] as any; + const { stringArray } = profile.shared; + expect(stringArray[data.level]).toBe('Debug'); + expect(stringArray[data.message]).toBe('Hello'); + // Stack indexes from the source profile would be meaningless. + expect(data.cause).toBeUndefined(); + }); + + it('does not modify the target profile', function () { + const { target, source } = setup(); + const stringCount = target.shared.stringArray.length; + mergeMarkersFromProfile(target, source); + expect(target.shared.stringArray.length).toBe(stringCount); + expect(target.threads[1].markers.length).toBe(0); + }); +}); diff --git a/src/test/unit/profiler-edit.test.ts b/src/test/unit/profiler-edit.test.ts index 96226af9c3..7428857fe8 100644 --- a/src/test/unit/profiler-edit.test.ts +++ b/src/test/unit/profiler-edit.test.ts @@ -164,6 +164,27 @@ describe('makeOptionsFromArgv', function () { expect(options.insertLabelFrames).toEqual('/path/to/labels.toml'); }); + it('recognizes optional --merge-markers-from and --only-keep-processes-with-name-prefix', function () { + const options = makeOptionsFromArgv([ + ...commonArgs, + '-i', + '/path/to/simpleperf.json', + '-o', + '/path/to/output.json', + '--merge-markers-from', + 'https://example.com/gecko.json', + '--only-keep-processes-with-name-prefix', + 'org.mozilla.fenix', + ]); + expect(options.mergeMarkersFrom).toEqual({ + type: 'URL', + url: 'https://example.com/gecko.json', + }); + expect(options.onlyKeepProcessesWithNamePrefix).toEqual( + 'org.mozilla.fenix' + ); + }); + it('throws when no input is provided', function () { expect(() => makeOptionsFromArgv([...commonArgs, '-o', '/path/to/output.json'])