Add functionality to profiler-edit to merge two profiles, one with samples and one with markers

src/node-tools/profiler-edit.ts

@@ -51,6 +51,7 @@ import {
   type SourceReader,
 } from 'firefox-profiler/profile-logic/embed-js-sources';
 import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/source-map';
+import { mergeMarkersFromProfile } from 'firefox-profiler/profile-logic/merge-markers';
 
 /**
  * A CLI tool for editing profiles.
@@ -81,6 +82,10 @@ import { runSourceMapSymbolicationNode } from 'firefox-profiler/profile-query/so
  *
  *   node node-tools-dist/profiler-edit.js -i samply-profile.json -o out.json.gz \
  *     --js-sources-from-func-names --map-source-paths mapping.json
+ *
+ *   node node-tools-dist/profiler-edit.js -i simpleperf-profile.json \
+ *     -o merged.json.gz --merge-markers-from gecko-profile.json \
+ *     --only-keep-processes-with-name-prefix org.mozilla.fenix
  */
 
 export type ProfileSource =
@@ -102,6 +107,8 @@ export interface WasmSymbolicationCliSpec {
 export interface CliOptions {
   input: ProfileSource;
   output: string;
+  mergeMarkersFrom?: ProfileSource;
+  onlyKeepProcessesWithNamePrefix?: string;
   symbolicateWithServer?: string;
   symbolicateWasm: WasmSymbolicationCliSpec[];
   mapSourcePaths?: string;
@@ -339,9 +346,54 @@ async function loadProfile(source: ProfileSource): Promise<Profile> {
   }
 }
 
+function keepThreads(
+  profile: Profile,
+  shouldKeep: (thread: RawThread, threadIndex: ThreadIndex) => boolean
+): Profile {
+  const oldThreadIndexToNew = new Map<ThreadIndex, ThreadIndex>();
+  const keptThreads: RawThread[] = [];
+  profile.threads.forEach((thread, oldIndex) => {
+    if (shouldKeep(thread, oldIndex)) {
+      oldThreadIndexToNew.set(oldIndex, keptThreads.length);
+      keptThreads.push(thread);
+    }
+  });
+  return {
+    ...profile,
+    threads: keptThreads,
+    ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew),
+  };
+}
+
 export async function run(options: CliOptions) {
   let profile = await loadProfile(options.input);
 
+  const processNamePrefix = options.onlyKeepProcessesWithNamePrefix;
+  if (processNamePrefix !== undefined) {
+    const before = profile.threads.length;
+    profile = keepThreads(
+      profile,
+      (thread) => thread.processName?.startsWith(processNamePrefix) ?? false
+    );
+    console.log(
+      `Kept ${profile.threads.length} of ${before} threads in processes whose name starts with ${JSON.stringify(processNamePrefix)}.`
+    );
+  }
+
+  if (options.mergeMarkersFrom !== undefined) {
+    const markersProfile = await loadProfile(options.mergeMarkersFrom);
+    const result = mergeMarkersFromProfile(profile, markersProfile);
+    profile = result.profile;
+    console.log(
+      `Merged the markers of ${result.mergedThreadNames.length} threads, aligning the profiles with ${result.usedClock}.`
+    );
+    if (result.unmatchedThreadNames.length !== 0) {
+      console.log(
+        `Dropped the markers of ${result.unmatchedThreadNames.length} threads without a matching pid and tid: ${result.unmatchedThreadNames.join(', ')}`
+      );
+    }
+  }
+
   if (options.symbolicateWithServer !== undefined) {
     const server = options.symbolicateWithServer;
     const symbolStore = new SymbolStore({
@@ -434,19 +486,9 @@ export async function run(options: CliOptions) {
       profile,
       options.onlyKeepThreadsWithMarkersMatching
     );
-    const oldThreadIndexToNew = new Map<ThreadIndex, ThreadIndex>();
-    const matchingThreads: RawThread[] = [];
-    profile.threads.forEach((thread, oldIndex) => {
-      if (matchingThreadIndexes.has(oldIndex)) {
-        oldThreadIndexToNew.set(oldIndex, matchingThreads.length);
-        matchingThreads.push(thread);
-      }
-    });
-    profile = {
-      ...profile,
-      threads: matchingThreads,
-      ...remapCountersAndProfilerOverhead(profile, oldThreadIndexToNew),
-    };
+    profile = keepThreads(profile, (_thread, threadIndex) =>
+      matchingThreadIndexes.has(threadIndex)
+    );
     console.log(
       `Kept ${profile.threads.length} of ${before} threads with markers matching ${JSON.stringify(options.onlyKeepThreadsWithMarkersMatching)}.`
     );
@@ -493,6 +535,12 @@ function collectWasm(
   return [...previous, { unstrippedWasmPath: value }];
 }
 
+function parseFileOrUrl(value: string): ProfileSource {
+  return /^https?:\/\//i.test(value)
+    ? { type: 'URL', url: value }
+    : { type: 'FILE', path: value };
+}
+
 function requireNonEmpty(flagName: string): (value: string) => string {
   return (value: string) => {
     if (value === '') {
@@ -516,6 +564,16 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
     .option('--from-file <path>', 'Load input from a file')
     .option('--from-url <url>', 'Load input from a URL')
     .option('--from-hash <hash>', 'Load input from a profile hash')
+    .option(
+      '--only-keep-processes-with-name-prefix <prefix>',
+      'Keep only the threads of processes whose name starts with this prefix',
+      requireNonEmpty('--only-keep-processes-with-name-prefix')
+    )
+    .option(
+      '--merge-markers-from <fileOrUrl>',
+      'Copy the markers of this profile into the threads with the same pid and tid, e.g. from a Gecko profile into a simpleperf profile of the same run',
+      requireNonEmpty('--merge-markers-from')
+    )
     .option(
       '--symbolicate-with-server <url>',
       'Symbolicate frames using this symbol server URL'
@@ -556,11 +614,7 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
 
   const sources: ProfileSource[] = [];
   if (typeof opts.input === 'string' && opts.input !== '') {
-    if (/^https?:\/\//i.test(opts.input)) {
-      sources.push({ type: 'URL', url: opts.input });
-    } else {
-      sources.push({ type: 'FILE', path: opts.input });
-    }
+    sources.push(parseFileOrUrl(opts.input));
   }
   if (typeof opts.fromFile === 'string' && opts.fromFile !== '') {
     sources.push({ type: 'FILE', path: opts.fromFile });
@@ -590,6 +644,14 @@ export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
   return {
     input: sources[0],
     output: opts.output,
+    mergeMarkersFrom:
+      typeof opts.mergeMarkersFrom === 'string'
+        ? parseFileOrUrl(opts.mergeMarkersFrom)
+        : undefined,
+    onlyKeepProcessesWithNamePrefix:
+      typeof opts.onlyKeepProcessesWithNamePrefix === 'string'
+        ? opts.onlyKeepProcessesWithNamePrefix
+        : undefined,
     symbolicateWithServer:
       typeof opts.symbolicateWithServer === 'string' &&
       opts.symbolicateWithServer !== ''

src/profile-logic/merge-markers.ts

@@ -0,0 +1,231 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { adjustMarkerTimestamps } from './process-profile';
+import { computeStringIndexMarkerFieldsByDataType } from './marker-schema';
+import {
+  getRawMarkerTableBuilderFromExisting,
+  finishRawMarkerTableBuilder,
+} from './data-structures';
+import { StringTable } from 'firefox-profiler/utils/string-table';
+
+import type {
+  IndexIntoCategoryList,
+  MarkerPayload,
+  Milliseconds,
+  Pid,
+  Profile,
+  ProfileMeta,
+  RawThread,
+  Tid,
+} from 'firefox-profiler/types';
+
+const PLATFORM_SPECIFIC_START_TIME_KEYS = [
+  'startTimeAsClockMonotonicNanosecondsSinceBoot',
+  'startTimeAsMachAbsoluteTimeNanoseconds',
+] as const;
+
+/**
+ * Compute the value that needs to be added to a timestamp of the source
+ * profile to get the timestamp of the same moment in the target profile.
+ *
+ * The platform-specific start times use the same clock in both profiles, so
+ * they give an exact result. The wall-clock start times are only a fallback.
+ */
+export function computeTimeDeltaBetweenProfiles(
+  sourceMeta: ProfileMeta,
+  targetMeta: ProfileMeta
+): { delta: Milliseconds; usedClock: string } {
+  for (const key of PLATFORM_SPECIFIC_START_TIME_KEYS) {
+    const sourceStart = sourceMeta[key];
+    const targetStart = targetMeta[key];
+    if (sourceStart !== undefined && targetStart !== undefined) {
+      return { delta: (sourceStart - targetStart) / 1e6, usedClock: key };
+    }
+  }
+  return {
+    delta: sourceMeta.startTime - targetMeta.startTime,
+    usedClock: 'startTime',
+  };
+}
+
+function threadKey(pid: Pid, tid: Tid): string {
+  return `${pid}:${tid}`;
+}
+
+const JVM_THREAD_SUFFIX = ' (JVM)';
+
+/**
+ * Group the source threads with markers by the key of the target thread they
+ * belong to.
+ *
+ * The Gecko profiler puts the Java markers of an Android thread "X" into a
+ * separate thread "X (JVM)", whose tid is the Java thread ID. These markers
+ * belong to the native thread named "X" in the same process.
+ */
+function groupSourceThreadsByKey(
+  threads: RawThread[]
+): Map<string, RawThread[]> {
+  const tidByProcessAndName = new Map<string, Tid>();
+  for (const thread of threads) {
+    if (!thread.name.endsWith(JVM_THREAD_SUFFIX)) {
+      tidByProcessAndName.set(`${thread.pid}:${thread.name}`, thread.tid);
+    }
+  }
+
+  const threadsByKey = new Map<string, RawThread[]>();
+  for (const thread of threads) {
+    if (thread.markers.length === 0) {
+      continue;
+    }
+    let tid: Tid | undefined = thread.tid;
+    if (thread.name.endsWith(JVM_THREAD_SUFFIX)) {
+      const nativeName = thread.name.slice(0, -JVM_THREAD_SUFFIX.length);
+      tid = tidByProcessAndName.get(`${thread.pid}:${nativeName}`) ?? tid;
+    }
+    const key = threadKey(thread.pid, tid);
+    const group = threadsByKey.get(key);
+    if (group === undefined) {
+      threadsByKey.set(key, [thread]);
+    } else {
+      group.push(thread);
+    }
+  }
+  return threadsByKey;
+}
+
+export type MergeMarkersResult = {
+  profile: Profile;
+  usedClock: string;
+  // Names of the source threads whose markers were copied.
+  mergedThreadNames: string[];
+  // Names of the source threads with markers which had no matching thread in
+  // the target profile.
+  unmatchedThreadNames: string[];
+};
+
+/**
+ * Copy the markers of the source profile into the target profile, into the
+ * threads which have the same pid and tid. Source threads without such a
+ * thread in the target profile are dropped. The markers of Gecko's "X (JVM)"
+ * threads go into the thread of "X".
+ *
+ * This is used for Android startup profiles, which combine the samples of a
+ * system-wide simpleperf profile with the markers of a Gecko profile from the
+ * same run.
+ */
+export function mergeMarkersFromProfile(
+  target: Profile,
+  source: Profile
+): MergeMarkersResult {
+  const { delta, usedClock } = computeTimeDeltaBetweenProfiles(
+    source.meta,
+    target.meta
+  );
+
+  const markerSchema = target.meta.markerSchema.slice();
+  const schemaNames = new Set(markerSchema.map((schema) => schema.name));
+  for (const schema of source.meta.markerSchema) {
+    if (!schemaNames.has(schema.name)) {
+      schemaNames.add(schema.name);
+      markerSchema.push(schema);
+    }
+  }
+
+  const categories = (target.meta.categories ?? []).slice();
+  const sourceCategories = source.meta.categories ?? [];
+  const sourceCategoryToTarget: IndexIntoCategoryList[] = sourceCategories.map(
+    (category) => {
+      const index = categories.findIndex((c) => c.name === category.name);
+      if (index !== -1) {
+        return index;
+      }
+      categories.push(category);
+      return categories.length - 1;
+    }
+  );
+
+  const stringArray = target.shared.stringArray.slice();
+  const stringTable = StringTable.withBackingArray(stringArray);
+  const sourceStringArray = source.shared.stringArray;
+  const stringIndexFieldsByDataType = computeStringIndexMarkerFieldsByDataType(
+    source.meta.markerSchema
+  );
+
+  function translateData(data: MarkerPayload | null): MarkerPayload | null {
+    if (data === null) {
+      return null;
+    }
+    const newData: any = { ...data };
+    // The stack indexes point into the source profile's stack table.
+    delete newData.cause;
+    const stringIndexFields = data.type
+      ? stringIndexFieldsByDataType.get(data.type)
+      : undefined;
+    if (stringIndexFields !== undefined) {
+      for (const key of stringIndexFields) {
+        if (typeof newData[key] === 'number') {
+          newData[key] = stringTable.indexForString(
+            sourceStringArray[newData[key]]
+          );
+        }
+      }
+    }
+    return newData;
+  }
+
+  const sourceThreadsByKey = groupSourceThreadsByKey(source.threads);
+
+  const mergedThreadNames: string[] = [];
+  const threads = target.threads.map((thread) => {
+    const key = threadKey(thread.pid, thread.tid);
+    const sourceThreads = sourceThreadsByKey.get(key);
+    if (sourceThreads === undefined) {
+      return thread;
+    }
+    sourceThreadsByKey.delete(key);
+
+    const builder = getRawMarkerTableBuilderFromExisting(thread.markers);
+    for (const sourceThread of sourceThreads) {
+      mergedThreadNames.push(sourceThread.name);
+      const sourceMarkers = adjustMarkerTimestamps(sourceThread.markers, delta);
+      for (let i = 0; i < sourceMarkers.length; i++) {
+        builder.data.push(translateData(sourceMarkers.data[i]));
+        builder.name.push(
+          stringTable.indexForString(sourceStringArray[sourceMarkers.name[i]])
+        );
+        builder.startTime.push(sourceMarkers.startTime[i]);
+        builder.endTime.push(sourceMarkers.endTime[i]);
+        builder.phase.push(sourceMarkers.phase[i]);
+        builder.category.push(
+          sourceCategoryToTarget[sourceMarkers.category[i]] ?? 0
+        );
+        if (builder.threadId !== undefined) {
+          builder.threadId.push(thread.tid);
+        }
+        builder.length++;
+      }
+    }
+    return { ...thread, markers: finishRawMarkerTableBuilder(builder) };
+  });
+
+  const profile: Profile = {
+    ...target,
+    meta: { ...target.meta, markerSchema, categories },
+    shared: { ...target.shared, stringArray },
+    threads,
+  };
+  if (source.pages !== undefined) {
+    profile.pages = [...(target.pages ?? []), ...source.pages];
+  }
+
+  return {
+    profile,
+    usedClock,
+    mergedThreadNames,
+    unmatchedThreadNames: [...sourceThreadsByKey.values()]
+      .flat()
+      .map((thread) => thread.name),
+  };
+}