diff --git a/src/node-tools/profiler-edit.ts b/src/node-tools/profiler-edit.ts index a525b77e99..a46a60afd1 100644 --- a/src/node-tools/profiler-edit.ts +++ b/src/node-tools/profiler-edit.ts @@ -51,6 +51,7 @@ import { type SourceReader, } from 'firefox-profiler/profile-logic/embed-js-sources'; import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/source-map'; +import { mergeMarkersFromProfile } from 'firefox-profiler/profile-logic/merge-markers'; /** * A CLI tool for editing profiles. @@ -81,6 +82,10 @@ import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/so * * node node-tools-dist/profiler-edit.js -i samply-profile.json -o out.json.gz \ * --js-sources-from-func-names --map-source-paths mapping.json + * + * node node-tools-dist/profiler-edit.js -i simpleperf-profile.json \ + * -o merged.json.gz --merge-markers-from gecko-profile.json \ + * --only-keep-processes-with-name-prefix org.mozilla.fenix */ export type ProfileSource = @@ -102,6 +107,8 @@ export interface WasmSymbolicationCliSpec { export interface CliOptions { input: ProfileSource; output: string; + mergeMarkersFrom?: ProfileSource; + onlyKeepProcessesWithNamePrefix?: string; symbolicateWithServer?: string; symbolicateWasm: WasmSymbolicationCliSpec[]; mapSourcePaths?: string; @@ -339,9 +346,54 @@ async function loadProfile(source: ProfileSource): Promise { } } +function keepThreads( + profile: Profile, + shouldKeep: (thread: RawThread, threadIndex: ThreadIndex) => boolean +): Profile { + const oldThreadIndexToNew = new Map(); + const keptThreads: RawThread[] = []; + profile.threads.forEach((thread, oldIndex) => { + if (shouldKeep(thread, oldIndex)) { + oldThreadIndexToNew.set(oldIndex, keptThreads.length); + keptThreads.push(thread); + } + }); + return { + ...profile, + threads: keptThreads, + ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew), + }; +} + export async function run(options: CliOptions) { let profile = await loadProfile(options.input); + const processNamePrefix = options.onlyKeepProcessesWithNamePrefix; + if (processNamePrefix !== undefined) { + const before = profile.threads.length; + profile = keepThreads( + profile, + (thread) => thread.processName?.startsWith(processNamePrefix) ?? false + ); + console.log( + `Kept ${profile.threads.length} of ${before} threads in processes whose name starts with ${JSON.stringify(processNamePrefix)}.` + ); + } + + if (options.mergeMarkersFrom !== undefined) { + const markersProfile = await loadProfile(options.mergeMarkersFrom); + const result = mergeMarkersFromProfile(profile, markersProfile); + profile = result.profile; + console.log( + `Merged the markers of ${result.mergedThreadNames.length} threads, aligning the profiles with ${result.usedClock}.` + ); + if (result.unmatchedThreadNames.length !== 0) { + console.log( + `Dropped the markers of ${result.unmatchedThreadNames.length} threads without a matching pid and tid: ${result.unmatchedThreadNames.join(', ')}` + ); + } + } + if (options.symbolicateWithServer !== undefined) { const server = options.symbolicateWithServer; const symbolStore = new SymbolStore({ @@ -434,19 +486,9 @@ export async function run(options: CliOptions) { profile, options.onlyKeepThreadsWithMarkersMatching ); - const oldThreadIndexToNew = new Map(); - const matchingThreads: RawThread[] = []; - profile.threads.forEach((thread, oldIndex) => { - if (matchingThreadIndexes.has(oldIndex)) { - oldThreadIndexToNew.set(oldIndex, matchingThreads.length); - matchingThreads.push(thread); - } - }); - profile = { - ...profile, - threads: matchingThreads, - ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew), - }; + profile = keepThreads(profile, (_thread, threadIndex) => + matchingThreadIndexes.has(threadIndex) + ); console.log( `Kept ${profile.threads.length} of ${before} threads with markers matching ${JSON.stringify(options.onlyKeepThreadsWithMarkersMatching)}.` ); @@ -493,6 +535,12 @@ function collectWasm( return [...previous, { unstrippedWasmPath: value }]; } +function parseFileOrUrl(value: string): ProfileSource { + return /^https?:\/\//i.test(value) + ? { type: 'URL', url: value } + : { type: 'FILE', path: value }; +} + function requireNonEmpty(flagName: string): (value: string) => string { return (value: string) => { if (value === '') { @@ -516,6 +564,16 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { .option('--from-file ', 'Load input from a file') .option('--from-url ', 'Load input from a URL') .option('--from-hash ', 'Load input from a profile hash') + .option( + '--only-keep-processes-with-name-prefix ', + 'Keep only the threads of processes whose name starts with this prefix', + requireNonEmpty('--only-keep-processes-with-name-prefix') + ) + .option( + '--merge-markers-from ', + 'Copy the markers of this profile into the threads with the same pid and tid, e.g. from a Gecko profile into a simpleperf profile of the same run', + requireNonEmpty('--merge-markers-from') + ) .option( '--symbolicate-with-server ', 'Symbolicate frames using this symbol server URL' @@ -556,11 +614,7 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { const sources: ProfileSource[] = []; if (typeof opts.input === 'string' && opts.input !== '') { - if (/^https?:\/\//i.test(opts.input)) { - sources.push({ type: 'URL', url: opts.input }); - } else { - sources.push({ type: 'FILE', path: opts.input }); - } + sources.push(parseFileOrUrl(opts.input)); } if (typeof opts.fromFile === 'string' && opts.fromFile !== '') { sources.push({ type: 'FILE', path: opts.fromFile }); @@ -590,6 +644,14 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions { return { input: sources[0], output: opts.output, + mergeMarkersFrom: + typeof opts.mergeMarkersFrom === 'string' + ? parseFileOrUrl(opts.mergeMarkersFrom) + : undefined, + onlyKeepProcessesWithNamePrefix: + typeof opts.onlyKeepProcessesWithNamePrefix === 'string' + ? opts.onlyKeepProcessesWithNamePrefix + : undefined, symbolicateWithServer: typeof opts.symbolicateWithServer === 'string' && opts.symbolicateWithServer !== '' diff --git a/src/profile-logic/merge-markers.ts b/src/profile-logic/merge-markers.ts new file mode 100644 index 0000000000..60a5f5436d --- /dev/null +++ b/src/profile-logic/merge-markers.ts @@ -0,0 +1,231 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { adjustMarkerTimestamps } from './process-profile'; +import { computeStringIndexMarkerFieldsByDataType } from './marker-schema'; +import { + getRawMarkerTableBuilderFromExisting, + finishRawMarkerTableBuilder, +} from './data-structures'; +import { StringTable } from 'firefox-profiler/utils/string-table'; + +import type { + IndexIntoCategoryList, + MarkerPayload, + Milliseconds, + Pid, + Profile, + ProfileMeta, + RawThread, + Tid, +} from 'firefox-profiler/types'; + +const PLATFORM_SPECIFIC_START_TIME_KEYS = [ + 'startTimeAsClockMonotonicNanosecondsSinceBoot', + 'startTimeAsMachAbsoluteTimeNanoseconds', +] as const; + +/** + * Compute the value that needs to be added to a timestamp of the source + * profile to get the timestamp of the same moment in the target profile. + * + * The platform-specific start times use the same clock in both profiles, so + * they give an exact result. The wall-clock start times are only a fallback. + */ +export function computeTimeDeltaBetweenProfiles( + sourceMeta: ProfileMeta, + targetMeta: ProfileMeta +): { delta: Milliseconds; usedClock: string } { + for (const key of PLATFORM_SPECIFIC_START_TIME_KEYS) { + const sourceStart = sourceMeta[key]; + const targetStart = targetMeta[key]; + if (sourceStart !== undefined && targetStart !== undefined) { + return { delta: (sourceStart - targetStart) / 1e6, usedClock: key }; + } + } + return { + delta: sourceMeta.startTime - targetMeta.startTime, + usedClock: 'startTime', + }; +} + +function threadKey(pid: Pid, tid: Tid): string { + return `${pid}:${tid}`; +} + +const JVM_THREAD_SUFFIX = ' (JVM)'; + +/** + * Group the source threads with markers by the key of the target thread they + * belong to. + * + * The Gecko profiler puts the Java markers of an Android thread "X" into a + * separate thread "X (JVM)", whose tid is the Java thread ID. These markers + * belong to the native thread named "X" in the same process. + */ +function groupSourceThreadsByKey( + threads: RawThread[] +): Map { + const tidByProcessAndName = new Map(); + for (const thread of threads) { + if (!thread.name.endsWith(JVM_THREAD_SUFFIX)) { + tidByProcessAndName.set(`${thread.pid}:${thread.name}`, thread.tid); + } + } + + const threadsByKey = new Map(); + for (const thread of threads) { + if (thread.markers.length === 0) { + continue; + } + let tid: Tid | undefined = thread.tid; + if (thread.name.endsWith(JVM_THREAD_SUFFIX)) { + const nativeName = thread.name.slice(0, -JVM_THREAD_SUFFIX.length); + tid = tidByProcessAndName.get(`${thread.pid}:${nativeName}`) ?? tid; + } + const key = threadKey(thread.pid, tid); + const group = threadsByKey.get(key); + if (group === undefined) { + threadsByKey.set(key, [thread]); + } else { + group.push(thread); + } + } + return threadsByKey; +} + +export type MergeMarkersResult = { + profile: Profile; + usedClock: string; + // Names of the source threads whose markers were copied. + mergedThreadNames: string[]; + // Names of the source threads with markers which had no matching thread in + // the target profile. + unmatchedThreadNames: string[]; +}; + +/** + * Copy the markers of the source profile into the target profile, into the + * threads which have the same pid and tid. Source threads without such a + * thread in the target profile are dropped. The markers of Gecko's "X (JVM)" + * threads go into the thread of "X". + * + * This is used for Android startup profiles, which combine the samples of a + * system-wide simpleperf profile with the markers of a Gecko profile from the + * same run. + */ +export function mergeMarkersFromProfile( + target: Profile, + source: Profile +): MergeMarkersResult { + const { delta, usedClock } = computeTimeDeltaBetweenProfiles( + source.meta, + target.meta + ); + + const markerSchema = target.meta.markerSchema.slice(); + const schemaNames = new Set(markerSchema.map((schema) => schema.name)); + for (const schema of source.meta.markerSchema) { + if (!schemaNames.has(schema.name)) { + schemaNames.add(schema.name); + markerSchema.push(schema); + } + } + + const categories = (target.meta.categories ?? []).slice(); + const sourceCategories = source.meta.categories ?? []; + const sourceCategoryToTarget: IndexIntoCategoryList[] = sourceCategories.map( + (category) => { + const index = categories.findIndex((c) => c.name === category.name); + if (index !== -1) { + return index; + } + categories.push(category); + return categories.length - 1; + } + ); + + const stringArray = target.shared.stringArray.slice(); + const stringTable = StringTable.withBackingArray(stringArray); + const sourceStringArray = source.shared.stringArray; + const stringIndexFieldsByDataType = computeStringIndexMarkerFieldsByDataType( + source.meta.markerSchema + ); + + function translateData(data: MarkerPayload | null): MarkerPayload | null { + if (data === null) { + return null; + } + const newData: any = { ...data }; + // The stack indexes point into the source profile's stack table. + delete newData.cause; + const stringIndexFields = data.type + ? stringIndexFieldsByDataType.get(data.type) + : undefined; + if (stringIndexFields !== undefined) { + for (const key of stringIndexFields) { + if (typeof newData[key] === 'number') { + newData[key] = stringTable.indexForString( + sourceStringArray[newData[key]] + ); + } + } + } + return newData; + } + + const sourceThreadsByKey = groupSourceThreadsByKey(source.threads); + + const mergedThreadNames: string[] = []; + const threads = target.threads.map((thread) => { + const key = threadKey(thread.pid, thread.tid); + const sourceThreads = sourceThreadsByKey.get(key); + if (sourceThreads === undefined) { + return thread; + } + sourceThreadsByKey.delete(key); + + const builder = getRawMarkerTableBuilderFromExisting(thread.markers); + for (const sourceThread of sourceThreads) { + mergedThreadNames.push(sourceThread.name); + const sourceMarkers = adjustMarkerTimestamps(sourceThread.markers, delta); + for (let i = 0; i < sourceMarkers.length; i++) { + builder.data.push(translateData(sourceMarkers.data[i])); + builder.name.push( + stringTable.indexForString(sourceStringArray[sourceMarkers.name[i]]) + ); + builder.startTime.push(sourceMarkers.startTime[i]); + builder.endTime.push(sourceMarkers.endTime[i]); + builder.phase.push(sourceMarkers.phase[i]); + builder.category.push( + sourceCategoryToTarget[sourceMarkers.category[i]] ?? 0 + ); + if (builder.threadId !== undefined) { + builder.threadId.push(thread.tid); + } + builder.length++; + } + } + return { ...thread, markers: finishRawMarkerTableBuilder(builder) }; + }); + + const profile: Profile = { + ...target, + meta: { ...target.meta, markerSchema, categories }, + shared: { ...target.shared, stringArray }, + threads, + }; + if (source.pages !== undefined) { + profile.pages = [...(target.pages ?? []), ...source.pages]; + } + + return { + profile, + usedClock, + mergedThreadNames, + unmatchedThreadNames: [...sourceThreadsByKey.values()] + .flat() + .map((thread) => thread.name), + }; +} diff --git a/src/test/unit/merge-markers.test.ts b/src/test/unit/merge-markers.test.ts new file mode 100644 index 0000000000..e98f4b72de --- /dev/null +++ b/src/test/unit/merge-markers.test.ts @@ -0,0 +1,121 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mergeMarkersFromProfile } from '../../profile-logic/merge-markers'; +import { getProfileWithMarkers } from '../fixtures/profiles/processed-profile'; +import { StringTable } from '../../utils/string-table'; + +import type { Profile, RawThread } from 'firefox-profiler/types'; + +function setThreadIds( + profile: Profile, + ids: Array<{ pid: string; tid: number; name: string }> +) { + profile.threads.forEach((thread, i) => { + Object.assign(thread, ids[i]); + }); +} + +function getMarkerNamesAndTimes(profile: Profile, thread: RawThread) { + const { markers } = thread; + const result = []; + for (let i = 0; i < markers.length; i++) { + result.push([ + profile.shared.stringArray[markers.name[i]], + markers.startTime[i], + ]); + } + return result; +} + +describe('mergeMarkersFromProfile', function () { + function setup() { + const target = getProfileWithMarkers([['TargetMarker', 1]], [], []); + setThreadIds(target, [ + { pid: '100', tid: 100, name: 'org.mozilla.fenix' }, + { pid: '100', tid: 101, name: 'Gecko' }, + { pid: '200', tid: 200, name: 'org.mozilla.fenix:tab' }, + ]); + target.meta.startTime = 1000; + target.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 5e9; + + const source = getProfileWithMarkers( + [['GeckoMarker', 2]], + [['UIMarker', 3]], + [['JavaMarker', 4]], + [['OrphanMarker', 5]] + ); + setThreadIds(source, [ + { pid: '100', tid: 101, name: 'GeckoMain' }, + { pid: '100', tid: 100, name: 'AndroidUI' }, + { pid: '100', tid: 1, name: 'AndroidUI (JVM)' }, + { pid: '300', tid: 300, name: 'Orphan' }, + ]); + // The source profile starts 10ms after the target profile. + source.meta.startTime = 1500; + source.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 5.01e9; + return { target, source }; + } + + it('copies the markers into the threads with the same pid and tid', function () { + const { target, source } = setup(); + const { profile, usedClock, unmatchedThreadNames } = + mergeMarkersFromProfile(target, source); + + expect(usedClock).toBe('startTimeAsClockMonotonicNanosecondsSinceBoot'); + expect(getMarkerNamesAndTimes(profile, profile.threads[0])).toEqual([ + ['TargetMarker', 1], + ['UIMarker', 13], + ['JavaMarker', 14], + ]); + expect(getMarkerNamesAndTimes(profile, profile.threads[1])).toEqual([ + ['GeckoMarker', 12], + ]); + expect(profile.threads[2].markers.length).toBe(0); + expect(unmatchedThreadNames).toEqual(['Orphan']); + }); + + it('falls back to the wall-clock start time', function () { + const { target, source } = setup(); + delete target.meta.startTimeAsClockMonotonicNanosecondsSinceBoot; + const { profile, usedClock } = mergeMarkersFromProfile(target, source); + + expect(usedClock).toBe('startTime'); + expect(getMarkerNamesAndTimes(profile, profile.threads[1])).toEqual([ + ['GeckoMarker', 502], + ]); + }); + + it('translates the string indexes in marker payloads', function () { + const { target, source } = setup(); + const sourceStrings = StringTable.withBackingArray( + source.shared.stringArray + ); + source.threads[0].markers.data[0] = { + type: 'Log', + module: 'nsHttp', + name: 'log', + level: sourceStrings.indexForString('Debug'), + message: sourceStrings.indexForString('Hello'), + cause: { tid: 101, time: 2, stack: 0 }, + } as any; + + const { profile } = mergeMarkersFromProfile(target, source); + const { markers } = profile.threads[1]; + const data = markers.data[0] as any; + const { stringArray } = profile.shared; + expect(stringArray[data.level]).toBe('Debug'); + expect(stringArray[data.message]).toBe('Hello'); + // Stack indexes from the source profile would be meaningless. + expect(data.cause).toBeUndefined(); + }); + + it('does not modify the target profile', function () { + const { target, source } = setup(); + const stringCount = target.shared.stringArray.length; + mergeMarkersFromProfile(target, source); + expect(target.shared.stringArray.length).toBe(stringCount); + expect(target.threads[1].markers.length).toBe(0); + }); +}); diff --git a/src/test/unit/profiler-edit.test.ts b/src/test/unit/profiler-edit.test.ts index 96226af9c3..7428857fe8 100644 --- a/src/test/unit/profiler-edit.test.ts +++ b/src/test/unit/profiler-edit.test.ts @@ -164,6 +164,27 @@ describe('makeOptionsFromArgv', function () { expect(options.insertLabelFrames).toEqual('/path/to/labels.toml'); }); + it('recognizes optional --merge-markers-from and --only-keep-processes-with-name-prefix', function () { + const options = makeOptionsFromArgv([ + ...commonArgs, + '-i', + '/path/to/simpleperf.json', + '-o', + '/path/to/output.json', + '--merge-markers-from', + 'https://example.com/gecko.json', + '--only-keep-processes-with-name-prefix', + 'org.mozilla.fenix', + ]); + expect(options.mergeMarkersFrom).toEqual({ + type: 'URL', + url: 'https://example.com/gecko.json', + }); + expect(options.onlyKeepProcessesWithNamePrefix).toEqual( + 'org.mozilla.fenix' + ); + }); + it('throws when no input is provided', function () { expect(() => makeOptionsFromArgv([...commonArgs, '-o', '/path/to/output.json'])