From 4e19f9e893e9cfb30aa0fc12c3b78daa3834a180 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Sun, 8 Jun 2025 18:52:24 -0400 Subject: [PATCH] Share the stackTable, frameTable, funcTable, resourceTable and nativeSymbols across threads. Don't merge frameTable and stackTable in upgrader. This is a trade-off. There are some existing uploaded profiles that are very large. Merging their frameTable and stackTable can take a long time. Advantages of merging: - If reuploaded, the reuploaded profile would be smaller. - Creating the call node table would be faster. Disadvantages of merging: - Long load time any time you open an existing large profile. --- src/actions/receive-profile.ts | 53 +- src/app-logic/constants.ts | 2 +- src/app-logic/url-handling.ts | 21 +- src/profile-logic/data-structures.ts | 37 +- src/profile-logic/global-data-collector.ts | 115 ++- src/profile-logic/import/chrome.ts | 76 +- src/profile-logic/import/dhat.ts | 24 +- src/profile-logic/import/flame-graph.ts | 29 +- src/profile-logic/import/simpleperf.ts | 50 +- src/profile-logic/js-tracer.tsx | 70 +- src/profile-logic/merge-compare.ts | 780 ++++++++---------- src/profile-logic/process-profile.ts | 35 +- .../processed-profile-versioning.ts | 206 +++++ src/profile-logic/profile-compacting.ts | 754 ++++++++++++----- src/profile-logic/profile-data.ts | 254 +++--- src/profile-logic/sanitize.ts | 347 ++++---- src/profile-logic/symbolication.ts | 137 +-- src/profile-logic/tracks.ts | 4 +- src/reducers/profile-view.ts | 31 +- src/selectors/per-thread/composed.ts | 2 +- src/selectors/per-thread/thread.tsx | 43 +- src/selectors/profile.ts | 9 + src/symbolicator-cli/index.ts | 36 +- .../components/CallNodeContextMenu.test.tsx | 15 +- src/test/components/FlameGraph.test.tsx | 2 +- src/test/components/SampleGraph.test.tsx | 4 +- src/test/components/TabSelectorMenu.test.tsx | 18 +- .../components/ThreadActivityGraph.test.tsx | 4 +- src/test/components/TooltipCallnode.test.tsx | 5 +- src/test/components/TrackThread.test.tsx | 2 +- src/test/fixtures/utils.ts | 8 +- src/types/actions.ts | 4 +- src/types/profile.ts | 11 +- 33 files changed, 1824 insertions(+), 1364 deletions(-) diff --git a/src/actions/receive-profile.ts b/src/actions/receive-profile.ts index bfca2b3ba0..e434aea56c 100644 --- a/src/actions/receive-profile.ts +++ b/src/actions/receive-profile.ts @@ -84,10 +84,7 @@ import type { MixedObject, } from 'firefox-profiler/types'; -import type { - FuncToFuncsMap, - SymbolicationStepInfo, -} from '../profile-logic/symbolication'; +import type { SymbolicationStepInfo } from '../profile-logic/symbolication'; import { assertExhaustiveCheck } from '../utils/types'; import { bytesToBase64DataUrl } from 'firefox-profiler/utils/base64'; import type { @@ -337,7 +334,7 @@ export function finalizeFullProfileView( const thread = profile.threads[threadIndex]; const { samples, jsAllocations, nativeAllocations } = thread; hasSamples = [samples, jsAllocations, nativeAllocations].some((table) => - hasUsefulSamples(table?.stack, thread, profile.shared) + hasUsefulSamples(table?.stack, profile.shared) ); if (hasSamples) { break; @@ -446,28 +443,20 @@ export function doneSymbolicating(): Action { // reach the screen because it would be invalidated by the next symbolication update. // So we queue up symbolication steps and run the update from requestIdleCallback. export function bulkProcessSymbolicationSteps( - symbolicationStepsPerThread: Map + symbolicationSteps: SymbolicationStepInfo[] ): ThunkAction { return (dispatch, getState) => { const { threads, shared } = getProfile(getState()); - const oldFuncToNewFuncsMaps: Map = new Map(); - const symbolicatedThreads = threads.map((oldThread, threadIndex) => { - const symbolicationSteps = symbolicationStepsPerThread.get(threadIndex); - if (symbolicationSteps === undefined) { - return oldThread; - } - const { thread, oldFuncToNewFuncsMap } = applySymbolicationSteps( - oldThread, - shared, - symbolicationSteps - ); - oldFuncToNewFuncsMaps.set(threadIndex, oldFuncToNewFuncsMap); - return thread; - }); + const { + threads: symbolicatedThreads, + shared: symbolicatedShared, + oldFuncToNewFuncsMap, + } = applySymbolicationSteps(threads, shared, symbolicationSteps); dispatch({ type: 'BULK_SYMBOLICATION', - oldFuncToNewFuncsMaps, + oldFuncToNewFuncsMap, symbolicatedThreads, + symbolicatedShared, }); }; } @@ -489,12 +478,12 @@ if (typeof window === 'object' && window.requestIdleCallback) { // Queues up symbolication steps and bulk-processes them from requestIdleCallback, // in order to improve UI responsiveness during symbolication. class SymbolicationStepQueue { - _updates: Map; + _updates: SymbolicationStepInfo[]; _updateObservers: Array<() => void>; _requestedUpdate: boolean; constructor() { - this._updates = new Map(); + this._updates = []; this._updateObservers = []; this._requestedUpdate = false; } @@ -512,7 +501,7 @@ class SymbolicationStepQueue { _dispatchUpdate(dispatch: Dispatch) { const updates = this._updates; const observers = this._updateObservers; - this._updates = new Map(); + this._updates = []; this._updateObservers = []; this._requestedUpdate = false; @@ -525,17 +514,11 @@ class SymbolicationStepQueue { enqueueSingleSymbolicationStep( dispatch: Dispatch, - threadIndex: ThreadIndex, symbolicationStepInfo: SymbolicationStepInfo, completionHandler: () => void ) { this._scheduleUpdate(dispatch); - let threadSteps = this._updates.get(threadIndex); - if (threadSteps === undefined) { - threadSteps = []; - this._updates.set(threadIndex, threadSteps); - } - threadSteps.push(symbolicationStepInfo); + this._updates.push(symbolicationStepInfo); this._updateObservers.push(completionHandler); } } @@ -671,15 +654,11 @@ export async function doSymbolicateProfile( await symbolicateProfile( profile, symbolStore, - ( - threadIndex: ThreadIndex, - symbolicationStepInfo: SymbolicationStepInfo - ) => { + (symbolicationStepInfo: SymbolicationStepInfo) => { completionPromises.push( new Promise((resolve) => { _symbolicationStepQueueSingleton.enqueueSingleSymbolicationStep( dispatch, - threadIndex, symbolicationStepInfo, () => resolve(undefined) ); @@ -1575,7 +1554,7 @@ export function changeTabFilter(tabID: TabID | null): ThunkAction { const thread = profile.threads[threadIndex]; const { samples, jsAllocations, nativeAllocations } = thread; hasSamples = [samples, jsAllocations, nativeAllocations].some((table) => - hasUsefulSamples(table?.stack, thread, profile.shared) + hasUsefulSamples(table?.stack, profile.shared) ); if (hasSamples) { break; diff --git a/src/app-logic/constants.ts b/src/app-logic/constants.ts index be047a25b2..4679913070 100644 --- a/src/app-logic/constants.ts +++ b/src/app-logic/constants.ts @@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 32; // The current version of the "processed" profile format. // Please don't forget to update the processed profile format changelog in // `docs-developer/CHANGELOG-formats.md`. -export const PROCESSED_PROFILE_VERSION = 58; +export const PROCESSED_PROFILE_VERSION = 59; // The following are the margin sizes for the left and right of the timeline. Independent // components need to share these values. diff --git a/src/app-logic/url-handling.ts b/src/app-logic/url-handling.ts index de0818abe4..56d8a298bc 100644 --- a/src/app-logic/url-handling.ts +++ b/src/app-logic/url-handling.ts @@ -28,7 +28,7 @@ import type { DataSource, Pid, Profile, - RawThread, + RawProfileSharedData, IndexIntoStackTable, TabID, TrackIndex, @@ -898,10 +898,6 @@ const _upgraders: { return; } - // The transform stack is for the selected thread. - // At the time this upgrader was written, there was only one selected thread. - const thread = profile.threads[selectedThread]; - for (let i = 0; i < transforms.length; i++) { const transform = transforms[i]; if ( @@ -922,7 +918,7 @@ const _upgraders: { // To be correct, we would need to apply all previous transforms and find // the right stack in the filtered thread. const callNodeStackIndex = getStackIndexFromVersion3JSCallNodePath( - thread, + profile.shared, transform.callNodePath ); if (callNodeStackIndex === null) { @@ -930,7 +926,7 @@ const _upgraders: { continue; } transform.callNodePath = getVersion4JSCallNodePathFromStackIndex( - thread, + profile.shared, callNodeStackIndex ); } @@ -1129,8 +1125,7 @@ const _upgraders: { return; } - const threadIndex = selectedThreads[0]; - const funcTableLength = profile.threads[threadIndex].funcTable.length; + const funcTableLength = profile.shared.funcTable.length; // cr-{implementation}-{resourceIndex}-{wrongFuncIndex} // -> cr-{implementation}-{resourceIndex}-{correctFuncIndex} @@ -1214,10 +1209,10 @@ for (const destVersionStr of Object.keys(_upgraders)) { // This should only be used for the URL upgrader, typically this // operation would use a call node index rather than a stack. function getStackIndexFromVersion3JSCallNodePath( - thread: RawThread, + shared: RawProfileSharedData, oldCallNodePath: CallNodePath ): IndexIntoStackTable | null { - const { stackTable, funcTable, frameTable } = thread; + const { stackTable, funcTable, frameTable } = shared; const stackIndexDepth: Map = new Map(); stackIndexDepth.set(null, -1); @@ -1260,10 +1255,10 @@ function getStackIndexFromVersion3JSCallNodePath( // Constructs the new JS CallNodePath from given stackIndex and returns it. // This should only be used for the URL upgrader. function getVersion4JSCallNodePathFromStackIndex( - thread: RawThread, + shared: RawProfileSharedData, stackIndex: IndexIntoStackTable ): CallNodePath { - const { funcTable, stackTable, frameTable } = thread; + const { funcTable, stackTable, frameTable } = shared; const callNodePath = []; let nextStackIndex: IndexIntoStackTable | null = stackIndex; while (nextStackIndex !== null) { diff --git a/src/profile-logic/data-structures.ts b/src/profile-logic/data-structures.ts index dfb9994e8e..592c3c4d03 100644 --- a/src/profile-logic/data-structures.ts +++ b/src/profile-logic/data-structures.ts @@ -8,6 +8,7 @@ import { } from '../app-logic/constants'; import type { + RawProfileSharedData, RawThread, RawSamplesTable, SamplesTable, @@ -73,6 +74,20 @@ export function getEmptyRawStackTable(): RawStackTable { }; } +export function shallowCloneRawStackTable( + stackTable: RawStackTable +): RawStackTable { + return { + // Important! + // If modifying this structure, please update all callers of this function to ensure + // that they are pushing on correctly to the data structure. These pushes may not + // be caught by the type system. + frame: stackTable.frame.slice(), + prefix: stackTable.prefix.slice(), + length: stackTable.length, + }; +} + /** * Returns an empty samples table with eventDelay field instead of responsiveness. * eventDelay is a new field and it replaced responsiveness. We should still @@ -398,11 +413,6 @@ export function getEmptyThread(overrides?: Partial): RawThread { // Creating samples with event delay since it's the new samples table. samples: getEmptySamplesTableWithEventDelay(), markers: getEmptyRawMarkerTable(), - stackTable: getEmptyRawStackTable(), - frameTable: getEmptyFrameTable(), - funcTable: getEmptyFuncTable(), - resourceTable: getEmptyResourceTable(), - nativeSymbols: getEmptyNativeSymbolTable(), }; return { @@ -411,6 +421,18 @@ export function getEmptyThread(overrides?: Partial): RawThread { }; } +export function getEmptySharedData(): RawProfileSharedData { + return { + stackTable: getEmptyRawStackTable(), + frameTable: getEmptyFrameTable(), + funcTable: getEmptyFuncTable(), + resourceTable: getEmptyResourceTable(), + nativeSymbols: getEmptyNativeSymbolTable(), + sources: getEmptySourceTable(), + stringArray: [], + }; +} + export function getEmptyProfile(): Profile { return { meta: { @@ -438,10 +460,7 @@ export function getEmptyProfile(): Profile { }, libs: [], pages: [], - shared: { - stringArray: [], - sources: getEmptySourceTable(), - }, + shared: getEmptySharedData(), threads: [], }; } diff --git a/src/profile-logic/global-data-collector.ts b/src/profile-logic/global-data-collector.ts index c23f0af7c9..c78011e7bb 100644 --- a/src/profile-logic/global-data-collector.ts +++ b/src/profile-logic/global-data-collector.ts @@ -3,7 +3,15 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { StringTable } from '../utils/string-table'; -import { getEmptySourceTable } from './data-structures'; +import { + getEmptyFrameTable, + getEmptyFuncTable, + getEmptyNativeSymbolTable, + getEmptyResourceTable, + getEmptySourceTable, + getEmptyStackTable, + resourceTypes, +} from './data-structures'; import type { Lib, @@ -13,6 +21,13 @@ import type { IndexIntoSourceTable, RawProfileSharedData, SourceTable, + FrameTable, + RawStackTable, + FuncTable, + ResourceTable, + NativeSymbolTable, + IndexIntoResourceTable, + IndexIntoFuncTable, } from 'firefox-profiler/types'; /** @@ -29,7 +44,14 @@ export class GlobalDataCollector { _stringArray: string[] = []; _stringTable: StringTable = StringTable.withBackingArray(this._stringArray); _sources: SourceTable = getEmptySourceTable(); + _frameTable: FrameTable = getEmptyFrameTable(); + _stackTable: RawStackTable = getEmptyStackTable(); + _funcTable: FuncTable = getEmptyFuncTable(); + _resourceTable: ResourceTable = getEmptyResourceTable(); + _nativeSymbols: NativeSymbolTable = getEmptyNativeSymbolTable(); + _funcKeyToFuncIndex: Map = new Map(); _uuidToSourceIndex: Map = new Map(); + _originToResourceIndex: Map = new Map(); _filenameToSourceIndex: Map = new Map(); @@ -56,6 +78,31 @@ export class GlobalDataCollector { return index; } + indexForFunc( + name: IndexIntoStringTable, + isJS: boolean, + relevantForJS: boolean, + resource: IndexIntoResourceTable | -1, + source: IndexIntoSourceTable | null, + lineNumber: number | null, + columnNumber: number | null + ): IndexIntoFuncTable { + const funcKey = `${name}-${isJS}-${relevantForJS}-${resource}-${source}-${lineNumber}-${columnNumber}`; + let funcIndex = this._funcKeyToFuncIndex.get(funcKey); + if (funcIndex === undefined) { + funcIndex = this._funcTable.length++; + this._funcTable.name[funcIndex] = name; + this._funcTable.isJS[funcIndex] = isJS; + this._funcTable.relevantForJS[funcIndex] = relevantForJS; + this._funcTable.resource[funcIndex] = resource; + this._funcTable.source[funcIndex] = source; + this._funcTable.lineNumber[funcIndex] = lineNumber; + this._funcTable.columnNumber[funcIndex] = columnNumber; + this._funcKeyToFuncIndex.set(funcKey, funcIndex); + } + return funcIndex; + } + // Return the global index for this source, adding it to the global list if // necessary. indexForSource(uuid: string | null, filename: string): IndexIntoSourceTable { @@ -85,18 +132,80 @@ export class GlobalDataCollector { return index; } + // Returns the resource index for a "url" or "webhost" resource which is created + // on demand based on the script URI. + indexForURIResource(scriptURI: string) { + // Figure out the origin and host. + let origin; + let host; + try { + const url = new URL(scriptURI); + if ( + !( + url.protocol === 'http:' || + url.protocol === 'https:' || + url.protocol === 'moz-extension:' + ) + ) { + throw new Error('not a webhost or extension protocol'); + } + origin = url.origin; + host = url.host; + } catch (_e) { + origin = scriptURI; + host = null; + } + + let resourceIndex = this._originToResourceIndex.get(origin); + if (resourceIndex !== undefined) { + return resourceIndex; + } + + const resourceTable = this._resourceTable; + + resourceIndex = resourceTable.length++; + this._originToResourceIndex.set(origin, resourceIndex); + if (host) { + // This is a webhost URL. + resourceTable.lib[resourceIndex] = null; + resourceTable.name[resourceIndex] = + this._stringTable.indexForString(origin); + resourceTable.host[resourceIndex] = + this._stringTable.indexForString(host); + resourceTable.type[resourceIndex] = resourceTypes.webhost; + } else { + // This is a URL, but it doesn't point to something on the web, e.g. a + // chrome url. + resourceTable.lib[resourceIndex] = null; + resourceTable.name[resourceIndex] = + this._stringTable.indexForString(scriptURI); + resourceTable.host[resourceIndex] = null; + resourceTable.type[resourceIndex] = resourceTypes.url; + } + return resourceIndex; + } + getStringTable(): StringTable { return this._stringTable; } - getSources(): SourceTable { - return this._sources; + getFrameTable(): FrameTable { + return this._frameTable; + } + + getStackTable(): RawStackTable { + return this._stackTable; } // Package up all de-duplicated global tables so that they can be embedded in // the profile. finish(): { libs: Lib[]; shared: RawProfileSharedData } { const shared: RawProfileSharedData = { + stackTable: this._stackTable, + frameTable: this._frameTable, + funcTable: this._funcTable, + resourceTable: this._resourceTable, + nativeSymbols: this._nativeSymbols, stringArray: this._stringArray, sources: this._sources, }; diff --git a/src/profile-logic/import/chrome.ts b/src/profile-logic/import/chrome.ts index 9026a6fd20..84274fcd83 100644 --- a/src/profile-logic/import/chrome.ts +++ b/src/profile-logic/import/chrome.ts @@ -6,9 +6,7 @@ import type { Profile, RawThread, RawStackTable, - IndexIntoFuncTable, IndexIntoStackTable, - IndexIntoResourceTable, MixedObject, } from 'firefox-profiler/types'; @@ -22,7 +20,7 @@ import { INTERVAL_END, } from 'firefox-profiler/app-logic/constants'; -import { getOrCreateURIResource, getTimeRangeForThread } from '../profile-data'; +import { getTimeRangeForThread } from '../profile-data'; import { GlobalDataCollector } from '../global-data-collector'; // Chrome Tracing Event Spec: @@ -269,9 +267,7 @@ export function attemptToConvertChromeProfile( type ThreadInfo = { thread: RawThread; - funcKeyToFuncId: Map; nodeIdToStackId: Map; - originToResourceIndex: Map; lastSeenTime: number; lastSampledTime: number; pid: number; @@ -403,8 +399,6 @@ function getThreadInfo( const threadInfo: ThreadInfo = { thread, nodeIdToStackId, - funcKeyToFuncId: new Map(), - originToResourceIndex: new Map(), lastSeenTime: chunk.ts / 1000, lastSampledTime: 0, pid: chunk.pid, @@ -506,6 +500,9 @@ async function processTracingEvents( const globalDataCollector = new GlobalDataCollector(); const stringTable = globalDataCollector.getStringTable(); + const frameTable = globalDataCollector.getFrameTable(); + const stackTable = globalDataCollector.getStackTable(); + let profileEvents: (ProfileEvent | CpuProfileEvent)[] = (eventsByName.get( 'Profile' ) || []) as ProfileEvent[]; @@ -531,8 +528,7 @@ async function processTracingEvents( profile, profileEvent ); - const { thread, funcKeyToFuncId, nodeIdToStackId, originToResourceIndex } = - threadInfo; + const { thread, nodeIdToStackId } = threadInfo; let profileChunks: any[] = []; if (profileEvent.name === 'Profile') { @@ -566,13 +562,7 @@ async function processTracingEvents( continue; } - const { - funcTable, - frameTable, - stackTable, - samples: samplesTable, - resourceTable, - } = thread; + const { samples: samplesTable } = thread; if (nodes) { const parentMap = new Map(); @@ -616,43 +606,27 @@ async function processTracingEvents( } const { functionName } = callFrame; - const funcKey = `${functionName}:${url || ''}:${lineNumber || 0}:${ - columnNumber || 0 - }`; const { category, isJS, relevantForJS } = getFunctionInfo( functionName, url !== undefined || lineNumber !== undefined ); - let funcId = funcKeyToFuncId.get(funcKey); - - if (funcId === undefined) { - // The function did not exist. - funcId = funcTable.length++; - funcTable.isJS.push(isJS); - funcTable.relevantForJS.push(relevantForJS); - const name = functionName !== '' ? functionName : '(anonymous)'; - funcTable.name.push(stringTable.indexForString(name)); - funcTable.resource.push( - isJS - ? getOrCreateURIResource( - url || '', - resourceTable, - stringTable, - originToResourceIndex - ) - : -1 - ); - funcTable.source.push( - isJS && url ? globalDataCollector.indexForSource(null, url) : null - ); - funcTable.lineNumber.push( - lineNumber === undefined ? null : lineNumber - ); - funcTable.columnNumber.push( - columnNumber === undefined ? null : columnNumber - ); - funcKeyToFuncId.set(funcKey, funcId); - } + const name = stringTable.indexForString( + functionName !== '' ? functionName : '(anonymous)' + ); + const source = + isJS && url ? globalDataCollector.indexForSource(null, url) : null; + const resource = isJS + ? globalDataCollector.indexForURIResource(url || '') + : -1; + const funcId = globalDataCollector.indexForFunc( + name, + isJS, + relevantForJS, + resource, + source, + lineNumber === undefined ? null : lineNumber, + columnNumber === undefined ? null : columnNumber + ); // Node indexes start at 1, while frame indexes start at 0. const frameIndex = nodeIndex - 1; @@ -712,9 +686,7 @@ async function processTracingEvents( } } - for (const thread of profile.threads) { - assertStackOrdering(thread.stackTable); - } + assertStackOrdering(stackTable); await extractScreenshots( threadInfoByPidAndTid, diff --git a/src/profile-logic/import/dhat.ts b/src/profile-logic/import/dhat.ts index ec07e36a0c..b2969a56d6 100644 --- a/src/profile-logic/import/dhat.ts +++ b/src/profile-logic/import/dhat.ts @@ -187,7 +187,7 @@ export function attemptToConvertDhat(json: unknown): Profile | null { const stringTable = globalDataCollector.getStringTable(); const allocationsTable = getEmptyUnbalancedNativeAllocationsTable(); - const { funcTable, stackTable, frameTable } = getEmptyThread(); + const { funcTable, stackTable, frameTable } = profile.shared; const funcKeyToFuncIndex = new Map(); @@ -376,28 +376,6 @@ export function attemptToConvertDhat(json: unknown): Profile | null { thread.tid = i; thread.name = name; - thread.funcTable.name = funcTable.name.slice(); - thread.funcTable.isJS = funcTable.isJS.slice(); - thread.funcTable.relevantForJS = funcTable.relevantForJS.slice(); - thread.funcTable.resource = funcTable.resource.slice(); - thread.funcTable.source = funcTable.source.slice(); - thread.funcTable.lineNumber = funcTable.lineNumber.slice(); - thread.funcTable.columnNumber = funcTable.columnNumber.slice(); - thread.funcTable.length = funcTable.length; - - thread.frameTable.address = frameTable.address.slice(); - thread.frameTable.line = frameTable.line.slice(); - thread.frameTable.column = frameTable.column.slice(); - thread.frameTable.category = frameTable.category.slice(); - thread.frameTable.subcategory = frameTable.subcategory.slice(); - thread.frameTable.innerWindowID = frameTable.innerWindowID.slice(); - thread.frameTable.func = frameTable.func.slice(); - thread.frameTable.length = frameTable.length; - - thread.stackTable.frame = stackTable.frame.slice(); - thread.stackTable.prefix = stackTable.prefix.slice(); - thread.stackTable.length = stackTable.length; - thread.nativeAllocations = { time: allocationsTable.time.slice(), stack: allocationsTable.stack.slice(), diff --git a/src/profile-logic/import/flame-graph.ts b/src/profile-logic/import/flame-graph.ts index bbc2385f2c..c941f2137b 100644 --- a/src/profile-logic/import/flame-graph.ts +++ b/src/profile-logic/import/flame-graph.ts @@ -7,7 +7,6 @@ import type { CategoryList, IndexIntoCategoryList, IndexIntoFrameTable, - IndexIntoFuncTable, IndexIntoStackTable, Profile, } from 'firefox-profiler/types/profile'; @@ -60,12 +59,13 @@ export function convertFlameGraphProfile(profileText: string): Profile { tid: 0, }); - const { frameTable, funcTable, stackTable, samples } = thread; + const frameTable = globalDataCollector.getFrameTable(); + const stackTable = globalDataCollector.getStackTable(); + const { samples } = thread; // Maps to deduplicate stacks, frames, and functions. const stackMap = new Map(); const frameMap = new Map(); - const funcMap = new Map(); function getOrCreateStack( frameIndex: IndexIntoFrameTable, @@ -100,19 +100,16 @@ export function convertFlameGraphProfile(profileText: string): Profile { } // Create or get function. - let funcIndex = funcMap.get(cleanedName); - if (funcIndex === undefined) { - funcIndex = funcTable.length; - funcTable.isJS.push(false); - funcTable.relevantForJS.push(false); - funcTable.name.push(stringTable.indexForString(cleanedName)); - funcTable.resource.push(-1); - funcTable.source.push(null); - funcTable.lineNumber.push(null); - funcTable.columnNumber.push(null); - funcTable.length++; - funcMap.set(cleanedName, funcIndex); - } + const nameIndex = stringTable.indexForString(cleanedName); + const funcIndex = globalDataCollector.indexForFunc( + nameIndex, + false, + false, + -1, + null, + null, + null + ); // Create frame. frameIndex = frameTable.length; diff --git a/src/profile-logic/import/simpleperf.ts b/src/profile-logic/import/simpleperf.ts index d6111b93be..152830e49c 100644 --- a/src/profile-logic/import/simpleperf.ts +++ b/src/profile-logic/import/simpleperf.ts @@ -16,6 +16,7 @@ import type { ResourceTable, RawSamplesTable, Profile, + RawProfileSharedData, RawThread, RawStackTable, } from 'firefox-profiler/types/profile'; @@ -215,6 +216,27 @@ class FirefoxSampleTable { } } +class FirefoxSharedData { + stringArray = []; + stringTable = StringTable.withBackingArray(this.stringArray); + stackTable = new FirefoxSampleTable(this.stringTable); + frameTable = new FirefoxFrameTable(this.stringTable); + funcTable = new FirefoxFuncTable(this.stringTable); + resourceTable = new FirefoxResourceTable(this.stringTable); + + toJson(): RawProfileSharedData { + return { + stackTable: this.stackTable.toJson(), + frameTable: this.frameTable.toJson(), + funcTable: this.funcTable.toJson(), + resourceTable: this.resourceTable.toJson(), + nativeSymbols: getEmptyNativeSymbolTable(), + sources: getEmptySourceTable(), + stringArray: this.stringArray, + }; + } +} + class FirefoxThread { name: string; isMainThread: boolean; @@ -233,19 +255,18 @@ class FirefoxThread { cpuClockEventId: number = -1; - constructor(thread: report.IThread, stringTable: StringTable) { + constructor(thread: report.IThread, shared: FirefoxSharedData) { this.tid = thread.threadId!; this.pid = thread.processId!; this.isMainThread = thread.threadId === thread.processId; this.name = thread.threadName ?? ''; - this.strings = stringTable; - - this.stackTable = new FirefoxSampleTable(this.strings); - this.frameTable = new FirefoxFrameTable(this.strings); - this.funcTable = new FirefoxFuncTable(this.strings); - this.resourceTable = new FirefoxResourceTable(this.strings); + this.strings = shared.stringTable; + this.stackTable = shared.stackTable; + this.frameTable = shared.frameTable; + this.funcTable = shared.funcTable; + this.resourceTable = shared.resourceTable; } toJson(): RawThread { @@ -262,11 +283,6 @@ class FirefoxThread { tid: this.tid, samples: this.sampleTable, markers: getEmptyRawMarkerTable(), - stackTable: this.stackTable.toJson(), - frameTable: this.frameTable.toJson(), - funcTable: this.funcTable.toJson(), - resourceTable: this.resourceTable.toJson(), - nativeSymbols: getEmptyNativeSymbolTable(), }; } @@ -362,17 +378,13 @@ class FirefoxProfile { sampleCount: number = 0; lostCount: number = 0; - stringArray = []; - stringTable = StringTable.withBackingArray(this.stringArray); + shared = new FirefoxSharedData(); toJson(): Profile { return { meta: this.getProfileMeta(), libs: [], - shared: { - stringArray: this.stringArray, - sources: getEmptySourceTable(), - }, + shared: this.shared.toJson(), threads: this.threads.map((thread) => thread.toJson()), }; } @@ -450,7 +462,7 @@ class FirefoxProfile { } addThread(thread: report.IThread) { - const firefoxThread = new FirefoxThread(thread, this.stringTable); + const firefoxThread = new FirefoxThread(thread, this.shared); this.threads.push(firefoxThread); this.threadMap.set(thread.threadId!, firefoxThread); } diff --git a/src/profile-logic/js-tracer.tsx b/src/profile-logic/js-tracer.tsx index 43fb52c42b..17af033347 100644 --- a/src/profile-logic/js-tracer.tsx +++ b/src/profile-logic/js-tracer.tsx @@ -2,25 +2,23 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { - getEmptyFrameTable, - getEmptyRawStackTable, getEmptySamplesTableWithEventDelay, getEmptyRawMarkerTable, } from './data-structures'; -import type { StringTable } from '../utils/string-table'; +import { StringTable } from '../utils/string-table'; import { ensureExists } from '../utils/types'; import type { JsTracerTable, IndexIntoStringTable, IndexIntoJsTracerEvents, IndexIntoFuncTable, + RawProfileSharedData, RawThread, IndexIntoStackTable, RawSamplesTable, CategoryList, JsTracerTiming, Microseconds, - SourceTable, } from 'firefox-profiler/types'; // See the function below for more information. @@ -32,12 +30,11 @@ type ScriptLocationToFuncIndex = Map; * This operation can fail, as there is no guarantee that every location in the JS * tracer information was sampled. */ -function getScriptLocationToFuncIndex( - thread: RawThread, - stringTable: StringTable, - sources: SourceTable -): ScriptLocationToFuncIndex { - const { funcTable } = thread; +function getScriptLocationToFuncIndex({ + funcTable, + sources, + stringArray, +}: RawProfileSharedData): ScriptLocationToFuncIndex { const scriptLocationToFuncIndex: ScriptLocationToFuncIndex = new Map(); for (let funcIndex = 0; funcIndex < funcTable.length; funcIndex++) { if (!funcTable.isJS[funcIndex]) { @@ -48,7 +45,7 @@ function getScriptLocationToFuncIndex( const sourceIndex = funcTable.source[funcIndex]; if (column !== null && line !== null && sourceIndex !== null) { const urlIndex = sources.filename[sourceIndex]; - const fileName = stringTable.getString(urlIndex); + const fileName = stringArray[urlIndex]; const key = `${fileName}:${line}:${column}`; if (scriptLocationToFuncIndex.has(key)) { // Multiple functions map to this script location. @@ -78,21 +75,16 @@ function getScriptLocationToFuncIndex( */ export function getJsTracerTiming( jsTracer: JsTracerTable, - thread: RawThread, - stringTable: StringTable, - sources: SourceTable + shared: RawProfileSharedData ): JsTracerTiming[] { const jsTracerTiming: JsTracerTiming[] = []; - const { funcTable } = thread; // This has already been computed by the conversion of the JS tracer structure to // a thread, but it's probably not worth the complexity of caching this object. // Just recompute it. - const scriptLocationToFuncIndex = getScriptLocationToFuncIndex( - thread, - stringTable, - sources - ); + const scriptLocationToFuncIndex = getScriptLocationToFuncIndex(shared); + + const { funcTable, stringArray } = shared; // Go through all of the events. for ( @@ -106,7 +98,7 @@ export function getJsTracerTiming( // By default we use the display name from JS tracer, but we may update it if // we can figure out more information about it. - let displayName = stringTable.getString(stringIndex); + let displayName = stringArray[stringIndex]; // We may have deduced the funcIndex in the scriptLocationToFuncIndex Map. let funcIndex: null | IndexIntoFuncTable = null; @@ -126,9 +118,9 @@ export function getJsTracerTiming( } else { // Update the information with the function that was found. funcIndex = funcIndexInMap; - displayName = `ƒ ${stringTable.getString( - funcTable.name[funcIndex] - )} ${displayName}`; + displayName = `ƒ ${ + stringArray[funcTable.name[funcIndex]] + } ${displayName}`; } } } @@ -499,35 +491,29 @@ export function getJsTracerLeafTiming( */ export function convertJsTracerToThreadWithoutSamples( fromThread: RawThread, + shared: RawProfileSharedData, stringTable: StringTable, jsTracer: JsTracerFixed, - categories: CategoryList, - sources: SourceTable + categories: CategoryList ): { thread: RawThread; stackMap: Map; } { - // Create a new thread, with empty information, but preserve some of the existing - // thread information. - const frameTable = getEmptyFrameTable(); - const stackTable = getEmptyRawStackTable(); const samples: RawSamplesTable = { ...getEmptySamplesTableWithEventDelay(), weight: [], weightType: 'tracing-ms', }; const markers = getEmptyRawMarkerTable(); - const funcTable = { ...fromThread.funcTable }; const thread: RawThread = { ...fromThread, markers, - funcTable, - stackTable, - frameTable, samples, }; + const { funcTable, frameTable, stackTable } = shared; + // Keep a stack of js tracer events, and end timings, that will be used to find // the stack prefixes. Once a JS tracer event starts past another event end, the // stack will be "popped" by decrementing the unmatchedIndex. @@ -545,11 +531,7 @@ export function convertJsTracerToThreadWithoutSamples( if (otherCategory === -1) { throw new Error("Expected to find an 'Other' category."); } - const scriptLocationToFuncIndex = getScriptLocationToFuncIndex( - thread, - stringTable, - sources - ); + const scriptLocationToFuncIndex = getScriptLocationToFuncIndex(shared); // Go through all of the JS tracer events, and build up the func, stack, and // frame tables. @@ -742,18 +724,18 @@ export function getJsTracerFixed(jsTracer: JsTracerTable): JsTracerFixed { */ export function convertJsTracerToThread( fromThread: RawThread, + shared: RawProfileSharedData, jsTracer: JsTracerTable, - categories: CategoryList, - stringTable: StringTable, - sources: SourceTable + categories: CategoryList ): RawThread { const jsTracerFixed = getJsTracerFixed(jsTracer); + const stringTable = StringTable.withBackingArray(shared.stringArray); const { thread, stackMap } = convertJsTracerToThreadWithoutSamples( fromThread, + shared, stringTable, jsTracerFixed, - categories, - sources + categories ); thread.samples = getSelfTimeSamplesFromJsTracer( stringTable, diff --git a/src/profile-logic/merge-compare.ts b/src/profile-logic/merge-compare.ts index bfd339b3e3..4fda86d465 100644 --- a/src/profile-logic/merge-compare.ts +++ b/src/profile-logic/merge-compare.ts @@ -26,6 +26,7 @@ import { getTimeRangeForThread, getTimeRangeIncludingAllThreads, computeTimeColumnForRawSamplesTable, + updateRawThreadStacks, } from './profile-data'; import { filterRawMarkerTableToRange, @@ -47,7 +48,6 @@ import type { IndexIntoLibs, IndexIntoNativeSymbolTable, IndexIntoStackTable, - IndexIntoSamplesTable, IndexIntoStringTable, IndexIntoSourceTable, FuncTable, @@ -64,7 +64,6 @@ import type { DerivedMarkerInfo, RawMarkerTable, MarkerPayload, - MarkerIndex, Milliseconds, Tid, } from 'firefox-profiler/types'; @@ -144,14 +143,14 @@ export function mergeProfilesForDiffing( const { stringArray: newStringArray, - translationMaps: translationMapForStrings, + translationMaps: translationMapsForStrings, } = mergeStringArrays(profiles.map((profile) => profile.shared.stringArray)); // Then merge sources. const { sources: newSources, translationMaps: translationMapForSources } = mergeSources( profiles.map((profile) => profile.shared.sources ?? null), - translationMapForStrings + translationMapsForStrings ); // Then merge libs. @@ -160,7 +159,49 @@ export function mergeProfilesForDiffing( ); resultProfile.libs = newLibs; + const { + resourceTable: newResourceTable, + translationMaps: translationMapsForResources, + } = combineResourceTables( + profiles, + translationMapsForStrings, + translationMapsForLibs + ); + const { + nativeSymbols: newNativeSymbols, + translationMaps: translationMapsForNativeSymbols, + } = combineNativeSymbolTables( + profiles, + translationMapsForStrings, + translationMapsForLibs + ); + const { funcTable: newFuncTable, translationMaps: translationMapsForFuncs } = + combineFuncTables( + profiles, + translationMapsForResources, + translationMapForSources, + translationMapsForStrings + ); + const { + frameTable: newFrameTable, + translationMaps: translationMapsForFrames, + } = combineFrameTables( + profiles, + translationMapsForFuncs, + translationMapsForNativeSymbols, + translationMapsForCategories + ); + const { + stackTable: newStackTable, + translationMaps: translationMapsForStacks, + } = combineStackTables(profiles, translationMapsForFrames); + resultProfile.shared = { + stackTable: newStackTable, + frameTable: newFrameTable, + funcTable: newFuncTable, + nativeSymbols: newNativeSymbols, + resourceTable: newResourceTable, stringArray: newStringArray, sources: newSources, }; @@ -190,65 +231,24 @@ export function mergeProfilesForDiffing( transformStacks[i] = profileSpecific.transforms[selectedThreadIndex]; implementationFilters.push(profileSpecific.implementation); - // We adjust the categories using the maps computed above. - // TODO issue #2151: Also adjust subcategories. - thread.frameTable = { - ...thread.frameTable, - category: adjustNullableCategories( - thread.frameTable.category, - translationMapsForCategories[i] - ), - }; - thread.funcTable = { - ...thread.funcTable, - name: adjustStringIndexes( - thread.funcTable.name, - translationMapForStrings[i] - ), - source: adjustNullableSourceIndexes( - thread.funcTable.source, - translationMapForSources[i] - ), - }; - thread.resourceTable = { - ...thread.resourceTable, - name: adjustStringIndexes( - thread.resourceTable.name, - translationMapForStrings[i] - ), - host: adjustNullableStringIndexes( - thread.resourceTable.host, - translationMapForStrings[i] - ), - lib: adjustResourceTableLibs( - thread.resourceTable.lib, - translationMapsForLibs[i] - ), - }; - thread.nativeSymbols = { - ...thread.nativeSymbols, - name: adjustStringIndexes( - thread.nativeSymbols.name, - translationMapForStrings[i] - ), - libIndex: adjustNativeSymbolLibs( - thread.nativeSymbols.libIndex, - translationMapsForLibs[i] - ), - }; thread.markers = { ...thread.markers, name: adjustStringIndexes( thread.markers.name, - translationMapForStrings[i] + translationMapsForStrings[i] ), data: adjustMarkerDataStringIndexes( thread.markers.data, - translationMapForStrings[i], + translationMapsForStrings[i], stringIndexMarkerFieldsByDataType ), }; + const translationMapForStacks = translationMapsForStacks[i]; + [thread] = updateRawThreadStacks([thread], (stackIndex) => + _mapNullableStack(stackIndex, translationMapForStacks) + ); + // Make sure that screenshot markers make it into the merged profile, even // if they're not on the selected thread. thread.markers = getThreadMarkersAndScreenshotMarkers( @@ -428,10 +428,6 @@ type TranslationMapForStacks = Map; type TranslationMapForLibs = Map; type TranslationMapForStrings = Map; type TranslationMapForSources = Map; -type TranslationMapForSamples = Map< - IndexIntoSamplesTable, - IndexIntoSamplesTable ->; /** * Merges several categories lists into one, resolving duplicates if necessary. @@ -545,76 +541,6 @@ function mergeSources( }; } -/** - * Adjusts the category indices in a category list using a translation map. - */ -function adjustResourceTableLibs( - libs: Array, // type of ResourceTable.libs - translationMap: TranslationMapForLibs -): Array { - return libs.map((lib) => { - if (lib === null) { - return lib; - } - const result = translationMap.get(lib); - if (result === undefined) { - throw new Error( - stripIndent` - Lib with index ${lib} hasn't been found in the translation map. - This shouldn't happen and indicates a bug in the profiler's code. - ` - ); - } - return result; - }); -} - -// Same as above, but without the " | null" in the type, to make flow happy. -function adjustNativeSymbolLibs( - libs: Array, // type of ResourceTable.libs - translationMap: TranslationMapForLibs -): Array { - return libs.map((lib) => { - const result = translationMap.get(lib); - if (result === undefined) { - throw new Error( - stripIndent` - Lib with index ${lib} hasn't been found in the translation map. - This shouldn't happen and indicates a bug in the profiler's code. - ` - ); - } - return result; - }); -} - -/** - * Adjusts the category indices in a category list using a translation map. - * This is just like the previous function, except the input and output arrays - * can have null values. There are 2 different functions to keep our type - * safety. - */ -function adjustNullableCategories( - categories: ReadonlyArray, - translationMap: TranslationMapForCategories -): Array { - return categories.map((category) => { - if (category === null) { - return null; - } - const result = translationMap.get(category); - if (result === undefined) { - throw new Error( - stripIndent` - Category with index ${category} hasn't been found in the translation map. - This shouldn't happen and indicates a bug in the profiler's code. - ` - ); - } - return result; - }); -} - function adjustStringIndexes( stringIndexes: ReadonlyArray, translationMap: TranslationMapForStrings @@ -673,48 +599,6 @@ function adjustMarkerDataStringIndexes( }); } -function adjustNullableStringIndexes( - stringIndexes: ReadonlyArray, - translationMap: TranslationMapForStrings -): Array { - return stringIndexes.map((stringIndex) => { - if (stringIndex === null) { - return null; - } - const newStringIndex = translationMap.get(stringIndex); - if (newStringIndex === undefined) { - throw new Error( - stripIndent` - String with index ${stringIndex} hasn't been found in the translation map. - This shouldn't happen and indicates a bug in the profiler's code. - ` - ); - } - return newStringIndex; - }); -} - -function adjustNullableSourceIndexes( - sourceIndexes: ReadonlyArray, - translationMap: TranslationMapForSources -): Array { - return sourceIndexes.map((sourceIndex) => { - if (sourceIndex === null) { - return null; - } - const newSourceIndex = translationMap.get(sourceIndex); - if (newSourceIndex === undefined) { - throw new Error( - stripIndent` - Source with index ${sourceIndex} hasn't been found in the translation map. - This shouldn't happen and indicates a bug in the profiler's code. - ` - ); - } - return newSourceIndex; - }); -} - /** * This combines the library lists from multiple profiles. It returns a merged * Lib array, along with a translation maps that can be used in other functions @@ -752,12 +636,200 @@ function mergeLibs(libsPerProfile: Lib[][]): { return { libs: newLibTable, translationMaps }; } +function _mapLib( + libIndex: IndexIntoLibs, + translationMap: TranslationMapForLibs +): IndexIntoLibs { + const newLibIndex = translationMap.get(libIndex); + if (newLibIndex === undefined) { + throw new Error( + stripIndent` + Lib with index ${libIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newLibIndex; +} + +function _mapNullableLib( + libIndex: IndexIntoLibs | null, + translationMap: TranslationMapForLibs +): IndexIntoLibs | null { + return libIndex !== null ? _mapLib(libIndex, translationMap) : null; +} + +function _mapString( + stringIndex: IndexIntoStringTable, + translationMap: TranslationMapForStrings +): IndexIntoStringTable { + const newStringIndex = translationMap.get(stringIndex); + if (newStringIndex === undefined) { + throw new Error( + stripIndent` + String with index ${stringIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newStringIndex; +} + +function _mapNullableString( + stringIndex: IndexIntoStringTable | null, + translationMap: TranslationMapForStrings +): IndexIntoStringTable | null { + return stringIndex !== null ? _mapString(stringIndex, translationMap) : null; +} + +function _mapNullableSource( + sourceIndex: IndexIntoSourceTable | null, + translationMap: TranslationMapForSources +): IndexIntoStringTable | null { + return sourceIndex !== null ? _mapSource(sourceIndex, translationMap) : null; +} + +function _mapSource( + sourceIndex: IndexIntoSourceTable, + translationMap: TranslationMapForStrings +): IndexIntoStringTable { + const newSourceIndex = translationMap.get(sourceIndex); + if (newSourceIndex === undefined) { + throw new Error( + stripIndent` + Source with index ${sourceIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newSourceIndex; +} + +function _mapFuncResource( + resourceIndex: IndexIntoResourceTable | -1, + translationMap: TranslationMapForResources +): IndexIntoResourceTable | -1 { + if (resourceIndex === -1) { + return -1; + } + + const newResourceIndex = translationMap.get(resourceIndex); + if (newResourceIndex === undefined) { + throw new Error( + stripIndent` + Resource with index ${resourceIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newResourceIndex; +} + +function _mapFunc( + funcIndex: IndexIntoFuncTable, + translationMap: TranslationMapForFuncs +): IndexIntoFuncTable { + const newFuncIndex = translationMap.get(funcIndex); + if (newFuncIndex === undefined) { + throw new Error( + stripIndent` + Func with index ${funcIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newFuncIndex; +} + +function _mapFrame( + frameIndex: IndexIntoFrameTable, + translationMap: TranslationMapForFrames +): IndexIntoFrameTable { + const newFrameIndex = translationMap.get(frameIndex); + if (newFrameIndex === undefined) { + throw new Error( + stripIndent` + Func with index ${frameIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newFrameIndex; +} + +function _mapNullableNativeSymbol( + nativeSymbolIndex: IndexIntoLibs | null, + translationMap: TranslationMapForNativeSymbols +): IndexIntoLibs | null { + if (nativeSymbolIndex === null) { + return null; + } + + const newNativeSymbolIndex = translationMap.get(nativeSymbolIndex); + if (newNativeSymbolIndex === undefined) { + throw new Error( + stripIndent` + Native symbol with index ${nativeSymbolIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newNativeSymbolIndex; +} + +function _mapNullableCategory( + categoryIndex: IndexIntoCategoryList | null, + translationMap: TranslationMapForCategories +): IndexIntoCategoryList | null { + if (categoryIndex === null) { + return null; + } + + const newCategoryIndex = translationMap.get(categoryIndex); + if (newCategoryIndex === undefined) { + throw new Error( + stripIndent` + Category with index ${categoryIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newCategoryIndex; +} + +function _mapStack( + stackIndex: IndexIntoStackTable, + translationMap: TranslationMapForStacks +): IndexIntoStackTable { + const newStackIndex = translationMap.get(stackIndex); + if (newStackIndex === undefined) { + throw new Error( + stripIndent` + Stack with index ${stackIndex} hasn't been found in the translation map. + This shouldn't happen and indicates a bug in the profiler's code. + ` + ); + } + return newStackIndex; +} + +function _mapNullableStack( + stackIndex: IndexIntoStackTable | null, + translationMap: TranslationMapForStacks +): IndexIntoStackTable | null { + return stackIndex !== null ? _mapStack(stackIndex, translationMap) : null; +} + /** - * This combines the resource tables for a list of threads. It returns the new + * This combines the resource tables for a list of profiles. It returns the new * resource table with the translation maps to be used in subsequent merging * functions. */ -function combineResourceTables(threads: ReadonlyArray): { +function combineResourceTables( + profiles: ReadonlyArray, + translationMapsForStrings: TranslationMapForStrings[], + translationMapsForLibs: TranslationMapForLibs[] +): { resourceTable: ResourceTable; translationMaps: TranslationMapForResources[]; } { @@ -765,14 +837,25 @@ function combineResourceTables(threads: ReadonlyArray): { const translationMaps: TranslationMapForResources[] = []; const newResourceTable = getEmptyResourceTable(); - threads.forEach((thread) => { + profiles.forEach((profile, profileIndex) => { + const translationMapForLibs = translationMapsForLibs[profileIndex]; + const translationMapForStrings = translationMapsForStrings[profileIndex]; const translationMap: TranslationMapForResources = new Map(); - const { resourceTable } = thread; + const { resourceTable } = profile.shared; for (let i = 0; i < resourceTable.length; i++) { - const libIndex = resourceTable.lib[i]; - const nameIndex = resourceTable.name[i]; - const hostIndex = resourceTable.host[i]; + const libIndex = _mapNullableLib( + resourceTable.lib[i], + translationMapForLibs + ); + const nameIndex = _mapString( + resourceTable.name[i], + translationMapForStrings + ); + const hostIndex = _mapNullableString( + resourceTable.host[i], + translationMapForStrings + ); const type = resourceTable.type[i]; // Duplicate search. @@ -800,9 +883,13 @@ function combineResourceTables(threads: ReadonlyArray): { } /** - * This combines the nativeSymbols tables for the threads. + * This combines the nativeSymbols tables for the profiles. */ -function combineNativeSymbolTables(threads: ReadonlyArray): { +function combineNativeSymbolTables( + profiles: ReadonlyArray, + translationMapsForStrings: TranslationMapForStrings[], + translationMapsForLibs: TranslationMapForLibs[] +): { nativeSymbols: NativeSymbolTable; translationMaps: TranslationMapForNativeSymbols[]; } { @@ -811,13 +898,21 @@ function combineNativeSymbolTables(threads: ReadonlyArray): { const translationMaps: TranslationMapForNativeSymbols[] = []; const newNativeSymbols = getEmptyNativeSymbolTable(); - threads.forEach((thread) => { + profiles.forEach((profile, profileIndex) => { + const translationMapForLibs = translationMapsForLibs[profileIndex]; + const translationMapForStrings = translationMapsForStrings[profileIndex]; const translationMap: TranslationMapForNativeSymbols = new Map(); - const { nativeSymbols } = thread; + const { nativeSymbols } = profile.shared; for (let i = 0; i < nativeSymbols.length; i++) { - const libIndex = nativeSymbols.libIndex[i]; - const nameIndex = nativeSymbols.name[i]; + const libIndex = _mapLib( + nativeSymbols.libIndex[i], + translationMapForLibs + ); + const nameIndex = _mapString( + nativeSymbols.name[i], + translationMapForStrings + ); const address = nativeSymbols.address[i]; const functionSize = nativeSymbols.functionSize[i]; @@ -848,37 +943,38 @@ function combineNativeSymbolTables(threads: ReadonlyArray): { } /** - * This combines the function tables for a list of threads. It returns the new + * This combines the function tables for a list of profiles. It returns the new * function table with the translation maps to be used in subsequent merging * functions. */ function combineFuncTables( + profiles: ReadonlyArray, translationMapsForResources: TranslationMapForResources[], - threads: ReadonlyArray + translationMapsForSources: TranslationMapForSources[], + translationMapsForStrings: TranslationMapForStrings[] ): { funcTable: FuncTable; translationMaps: TranslationMapForFuncs[] } { const mapOfInsertedFuncs = new Map(); const translationMaps: TranslationMapForFuncs[] = []; const newFuncTable = getEmptyFuncTable(); - threads.forEach((thread, threadIndex) => { - const { funcTable } = thread; + profiles.forEach((profile, profileIndex) => { + const { funcTable } = profile.shared; const translationMap: TranslationMapForFuncs = new Map(); - const resourceTranslationMap = translationMapsForResources[threadIndex]; + const translationMapForResources = + translationMapsForResources[profileIndex]; + const translationMapForStrings = translationMapsForStrings[profileIndex]; + const translationMapForSources = translationMapsForSources[profileIndex]; for (let i = 0; i < funcTable.length; i++) { - const sourceIndex = funcTable.source[i]; - const resourceIndex = funcTable.resource[i]; - const newResourceIndex = - resourceIndex >= 0 - ? resourceTranslationMap.get(funcTable.resource[i]) - : -1; - if (newResourceIndex === undefined) { - throw new Error(stripIndent` - We couldn't find the resource of func ${i} in the translation map. - This is a programming error. - `); - } - const nameIndex = funcTable.name[i]; + const sourceIndex = _mapNullableSource( + funcTable.source[i], + translationMapForSources + ); + const resourceIndex = _mapFuncResource( + funcTable.resource[i], + translationMapForResources + ); + const nameIndex = _mapString(funcTable.name[i], translationMapForStrings); const lineNumber = funcTable.lineNumber[i]; // Entries in this table can be either: @@ -889,7 +985,7 @@ function combineFuncTables( // number as well. // 3. Label frames: they have no resource, only a name. So we can't do // better than this. - const funcKey = [nameIndex, newResourceIndex, lineNumber].join('#'); + const funcKey = [nameIndex, resourceIndex, lineNumber].join('#'); const insertedFuncIndex = mapOfInsertedFuncs.get(funcKey); if (insertedFuncIndex !== undefined) { translationMap.set(i, insertedFuncIndex); @@ -900,7 +996,7 @@ function combineFuncTables( newFuncTable.isJS.push(funcTable.isJS[i]); newFuncTable.name.push(nameIndex); - newFuncTable.resource.push(newResourceIndex); + newFuncTable.resource.push(resourceIndex); newFuncTable.relevantForJS.push(funcTable.relevantForJS[i]); newFuncTable.source.push(sourceIndex); newFuncTable.lineNumber.push(lineNumber); @@ -916,54 +1012,48 @@ function combineFuncTables( } /** - * This combines the frame tables for a list of threads. It returns the new + * This combines the frame tables for a list of profiles. It returns the new * frame table with the translation maps to be used in subsequent merging * functions. - * Note that we don't try to merge the frames of the source threads, because + * Note that we don't try to merge the frames of the source profiles, because * that's not needed to get a diffing call tree. */ function combineFrameTables( + profiles: ReadonlyArray, translationMapsForFuncs: TranslationMapForFuncs[], translationMapsForNativeSymbols: TranslationMapForNativeSymbols[], - threads: ReadonlyArray + translationMapsForCategories: TranslationMapForCategories[] ): { frameTable: FrameTable; translationMaps: TranslationMapForFrames[] } { const translationMaps: TranslationMapForFrames[] = []; const newFrameTable = getEmptyFrameTable(); - threads.forEach((thread, threadIndex) => { - const { frameTable } = thread; + profiles.forEach((profile, profileIndex) => { + const { frameTable } = profile.shared; const translationMap: TranslationMapForFrames = new Map(); - const funcTranslationMap = translationMapsForFuncs[threadIndex]; - const nativeSymbolTranslationMap = - translationMapsForNativeSymbols[threadIndex]; - + const translationMapForFuncs = translationMapsForFuncs[profileIndex]; + const translationMapForNativeSymbols = + translationMapsForNativeSymbols[profileIndex]; + const translationMapForCategories = + translationMapsForCategories[profileIndex]; for (let i = 0; i < frameTable.length; i++) { - const newFunc = funcTranslationMap.get(frameTable.func[i]); - if (newFunc === undefined) { - throw new Error(stripIndent` - We couldn't find the function of frame ${i} in the translation map. - This is a programming error. - `); - } - - const nativeSymbol = frameTable.nativeSymbol[i]; - const newNativeSymbol = - nativeSymbol === null - ? null - : nativeSymbolTranslationMap.get(nativeSymbol); - if (newNativeSymbol === undefined) { - throw new Error(stripIndent` - We couldn't find the nativeSymbol of frame ${i} in the translation map. - This is a programming error. - `); - } + const func = _mapFunc(frameTable.func[i], translationMapForFuncs); + const nativeSymbol = _mapNullableNativeSymbol( + frameTable.nativeSymbol[i], + translationMapForNativeSymbols + ); + const category = _mapNullableCategory( + frameTable.category[i], + translationMapForCategories + ); + // TODO issue #2151: Also adjust subcategories. + const subcategory = frameTable.subcategory[i]; newFrameTable.address.push(frameTable.address[i]); newFrameTable.inlineDepth.push(frameTable.inlineDepth[i]); - newFrameTable.category.push(frameTable.category[i]); - newFrameTable.subcategory.push(frameTable.subcategory[i]); - newFrameTable.nativeSymbol.push(newNativeSymbol); - newFrameTable.func.push(newFunc); + newFrameTable.category.push(category); + newFrameTable.subcategory.push(subcategory); + newFrameTable.nativeSymbol.push(nativeSymbol); + newFrameTable.func.push(func); newFrameTable.innerWindowID.push(frameTable.innerWindowID[i]); newFrameTable.line.push(frameTable.line[i]); newFrameTable.column.push(frameTable.column[i]); @@ -979,33 +1069,29 @@ function combineFrameTables( } /** - * This combines the stack tables for a list of threads. It returns the new + * This combines the stack tables for a list of profiles. It returns the new * stack table with the translation maps to be used in subsequent merging * functions. - * Note that we don't try to merge the stacks of the source threads, because + * Note that we don't try to merge the stacks of the source profiles, because * that's not needed to get a diffing call tree. */ function combineStackTables( - translationMapsForFrames: TranslationMapForFrames[], - threads: ReadonlyArray + profiles: ReadonlyArray, + translationMapsForFrames: TranslationMapForFrames[] ): { stackTable: RawStackTable; translationMaps: TranslationMapForStacks[] } { const translationMaps: TranslationMapForStacks[] = []; const newStackTable = getEmptyRawStackTable(); - threads.forEach((thread, threadIndex) => { - const { stackTable } = thread; - const translationMap: TranslationMapForStacks = new Map(); - const frameTranslationMap = translationMapsForFrames[threadIndex]; + profiles.forEach((profile, profileIndex) => { + const { stackTable } = profile.shared; + const translationMap = new Map(); + const translationMapForFrames = translationMapsForFrames[profileIndex]; for (let i = 0; i < stackTable.length; i++) { - const newFrameIndex = frameTranslationMap.get(stackTable.frame[i]); - if (newFrameIndex === undefined) { - throw new Error(stripIndent` - We couldn't find the frame of stack ${i} in the translation map. - This is a programming error. - `); - } - + const frameIndex = _mapFrame( + stackTable.frame[i], + translationMapForFrames + ); const prefix = stackTable.prefix[i]; const newPrefix = prefix === null ? null : translationMap.get(prefix); if (newPrefix === undefined) { @@ -1015,7 +1101,7 @@ function combineStackTables( `); } - newStackTable.frame.push(newFrameIndex); + newStackTable.frame.push(frameIndex); newStackTable.prefix.push(newPrefix); translationMap.set(i, newStackTable.length); @@ -1037,13 +1123,11 @@ function combineStackTables( * subsequent merging functions, if necessary. */ function combineSamplesDiffing( - translationMapsForStacks: TranslationMapForStacks[], threadsAndWeightMultipliers: [ ThreadAndWeightMultiplier, ThreadAndWeightMultiplier, ] -): { samples: RawSamplesTable; translationMaps: TranslationMapForSamples[] } { - const translationMaps: TranslationMapForSamples[] = [new Map(), new Map()]; +): RawSamplesTable { const [ { thread: { samples: samples1, tid: tid1 }, @@ -1081,18 +1165,7 @@ function combineSamplesDiffing( if (nextSampleIsFromThread1) { // Next sample is from thread 1. - const stackIndex = samples1.stack[i]; - const newStackIndex = - stackIndex === null - ? null - : translationMapsForStacks[0].get(stackIndex); - if (newStackIndex === undefined) { - throw new Error(stripIndent` - We couldn't find the stack of sample ${i} in the translation map. - This is a programming error. - `); - } - newSamples.stack.push(newStackIndex); + newSamples.stack.push(samples1.stack[i]); // Diffing event delay values doesn't make sense since interleaved values // of eventDelay/responsiveness don't mean anything. newSamples.eventDelay!.push(null); @@ -1104,23 +1177,11 @@ function combineSamplesDiffing( const sampleWeight = samples1.weight ? samples1.weight[i] : 1; newWeight.push(-weightMultiplier1 * sampleWeight); - translationMaps[0].set(i, newSamples.length); newSamples.length++; i++; } else { // Next sample is from thread 2. - const stackIndex = samples2.stack[j]; - const newStackIndex = - stackIndex === null - ? null - : translationMapsForStacks[1].get(stackIndex); - if (newStackIndex === undefined) { - throw new Error(stripIndent` - We couldn't find the stack of sample ${j} in the translation map. - This is a programming error. - `); - } - newSamples.stack.push(newStackIndex); + newSamples.stack.push(samples2.stack[j]); // Diffing event delay values doesn't make sense since interleaved values // of eventDelay/responsiveness don't mean anything. newSamples.eventDelay!.push(null); @@ -1129,16 +1190,12 @@ function combineSamplesDiffing( const sampleWeight = samples2.weight ? samples2.weight[j] : 1; newWeight.push(weightMultiplier2 * sampleWeight); - translationMaps[1].set(j, newSamples.length); newSamples.length++; j++; } } - return { - samples: newSamples, - translationMaps, - }; + return newSamples; } type ThreadAndWeightMultiplier = { @@ -1160,32 +1217,7 @@ function getComparisonThread( ): RawThread { const threads = threadsAndWeightMultipliers.map((item) => item.thread); - const { - resourceTable: newResourceTable, - translationMaps: translationMapsForResources, - } = combineResourceTables(threads); - const { - nativeSymbols: newNativeSymbols, - translationMaps: translationMapsForNativeSymbols, - } = combineNativeSymbolTables(threads); - const { funcTable: newFuncTable, translationMaps: translationMapsForFuncs } = - combineFuncTables(translationMapsForResources, threads); - const { - frameTable: newFrameTable, - translationMaps: translationMapsForFrames, - } = combineFrameTables( - translationMapsForFuncs, - translationMapsForNativeSymbols, - threads - ); - const { - stackTable: newStackTable, - translationMaps: translationMapsForStacks, - } = combineStackTables(translationMapsForFrames, threads); - const { samples: newSamples } = combineSamplesDiffing( - translationMapsForStacks, - threadsAndWeightMultipliers - ); + const newSamples = combineSamplesDiffing(threadsAndWeightMultipliers); const mergedThread = { processType: 'comparison', @@ -1211,11 +1243,6 @@ function getComparisonThread( isMainThread: true, samples: newSamples, markers: getEmptyRawMarkerTable(), - stackTable: newStackTable, - frameTable: newFrameTable, - funcTable: newFuncTable, - resourceTable: newResourceTable, - nativeSymbols: newNativeSymbols, }; return mergedThread; @@ -1228,40 +1255,8 @@ function getComparisonThread( * TODO: Overlapping threads will not look great due to #2783. */ export function mergeThreads(threads: RawThread[]): RawThread { - // Combine the table we would need. - const { - resourceTable: newResourceTable, - translationMaps: translationMapsForResources, - } = combineResourceTables(threads); - const { - nativeSymbols: newNativeSymbols, - translationMaps: translationMapsForNativeSymbols, - } = combineNativeSymbolTables(threads); - const { funcTable: newFuncTable, translationMaps: translationMapsForFuncs } = - combineFuncTables(translationMapsForResources, threads); - const { - frameTable: newFrameTable, - translationMaps: translationMapsForFrames, - } = combineFrameTables( - translationMapsForFuncs, - translationMapsForNativeSymbols, - threads - ); - const { - stackTable: newStackTable, - translationMaps: translationMapsForStacks, - } = combineStackTables(translationMapsForFrames, threads); - - // Combine the samples for merging. - const newSamples = combineSamplesForMerging( - translationMapsForStacks, - threads - ); - - const { markerTable: newMarkers } = mergeMarkers( - translationMapsForStacks, - threads - ); + const newSamples = combineSamplesForMerging(threads); + const newMarkers = mergeMarkers(threads); let processStartupTime = Infinity; let processShutdownTime = -Infinity; @@ -1297,11 +1292,6 @@ export function mergeThreads(threads: RawThread[]): RawThread { isMainThread: true, samples: newSamples, markers: newMarkers, - stackTable: newStackTable, - frameTable: newFrameTable, - funcTable: newFuncTable, - nativeSymbols: newNativeSymbols, - resourceTable: newResourceTable, }; return mergedThread; @@ -1316,10 +1306,7 @@ export function mergeThreads(threads: RawThread[]): RawThread { * It returns the new sample table with the translation maps to be used in * subsequent merging functions, if necessary. */ -function combineSamplesForMerging( - translationMapsForStacks: TranslationMapForStacks[], - threads: RawThread[] -): RawSamplesTable { +function combineSamplesForMerging(threads: RawThread[]): RawSamplesTable { const samplesPerThread: RawSamplesTable[] = threads.map( (thread) => thread.samples ); @@ -1379,19 +1366,7 @@ function combineSamplesForMerging( const sourceThreadSampleIndex: number = nextSampleIndexPerThread[sourceThreadIndex]; - const stackIndex: number | null = - sourceThreadSamples.stack[sourceThreadSampleIndex]; - const newStackIndex = - stackIndex === null - ? null - : translationMapsForStacks[sourceThreadIndex].get(stackIndex); - if (newStackIndex === undefined) { - throw new Error(stripIndent` - We couldn't find the stack of sample ${sourceThreadSampleIndex} in the translation map. - This is a programming error. - `); - } - newSamples.stack.push(newStackIndex); + newSamples.stack.push(sourceThreadSamples.stack[sourceThreadSampleIndex]); // It doesn't make sense to combine event delay values. We need to use jank markers // from independent threads instead. ensureExists(newSamples.eventDelay).push(null); @@ -1409,77 +1384,34 @@ function combineSamplesForMerging( return newSamples; } -type TranslationMapForMarkers = Map; - /** * Merge markers from different threads. And update the new string table while doing it. */ -function mergeMarkers( - translationMapsForStacks: TranslationMapForStacks[], - threads: RawThread[] -): { - markerTable: RawMarkerTable; - translationMaps: TranslationMapForMarkers[]; -} { - const newThreadId: Tid[] = []; - const newMarkerTable = { ...getEmptyRawMarkerTable(), threadId: newThreadId }; - - const translationMaps: TranslationMapForMarkers[] = []; +function mergeMarkers(threads: RawThread[]): RawMarkerTable { + const newThreadId: Array = []; + const newMarkerTable: RawMarkerTable = { + ...getEmptyRawMarkerTable(), + threadId: newThreadId, + }; - threads.forEach((thread, threadIndex) => { - const translationMapForStacks = translationMapsForStacks[threadIndex]; - const translationMap: TranslationMapForMarkers = new Map(); + threads.forEach((thread) => { const { markers } = thread; for (let markerIndex = 0; markerIndex < markers.length; markerIndex++) { - // We need to move the name string to the new string table if doesn't exist. - const nameIndex = markers.name[markerIndex]; - - // Move marker data to the new marker table - const oldData = markers.data[markerIndex]; - - if (oldData && 'cause' in oldData && oldData.cause) { - // The old data has a cause, we need to convert the stack. - const oldStack = oldData.cause.stack; - const newStack = - oldStack !== null ? translationMapForStacks.get(oldStack) : null; - if (newStack === undefined) { - throw new Error( - `Missing old stack entry ${oldStack} in the translation map.` - ); - } - - newMarkerTable.data.push({ - ...oldData, - cause: { - ...oldData.cause, - stack: newStack, - }, - }); - } else { - newMarkerTable.data.push(oldData); - } - - newMarkerTable.name.push(nameIndex); + newMarkerTable.name.push(markers.name[markerIndex]); + newMarkerTable.data.push(markers.data[markerIndex]); newMarkerTable.startTime.push(markers.startTime[markerIndex]); newMarkerTable.endTime.push(markers.endTime[markerIndex]); newMarkerTable.phase.push(markers.phase[markerIndex]); newMarkerTable.category.push(markers.category[markerIndex]); newThreadId.push( - markers.threadId - ? (markers.threadId[markerIndex] ?? thread.tid) - : thread.tid + markers.threadId ? markers.threadId[markerIndex] : thread.tid ); - - // Set the translation map and increase the table length. - translationMap.set(markerIndex, newMarkerTable.length); newMarkerTable.length++; } - - translationMaps.push(translationMap); }); - return { markerTable: newMarkerTable, translationMaps }; + return newMarkerTable; } /** diff --git a/src/profile-logic/process-profile.ts b/src/profile-logic/process-profile.ts index 2339e28824..a9963966e4 100644 --- a/src/profile-logic/process-profile.ts +++ b/src/profile-logic/process-profile.ts @@ -36,7 +36,6 @@ import { } from '../app-logic/constants'; import { getFriendlyThreadName, - getOrCreateURIResource, nudgeReturnAddresses, } from '../profile-logic/profile-data'; import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema'; @@ -512,14 +511,8 @@ function _extractJsFunction( return null; } - const { - funcTable, - stringTable, - resourceTable, - originToResourceIndex, - globalDataCollector, - geckoSourceTable, - } = extractionInfo; + const { funcTable, stringTable, globalDataCollector, geckoSourceTable } = + extractionInfo; // Case 4: JS function - A match was found in the location string in the format // of a JS function. @@ -527,12 +520,7 @@ function _extractJsFunction( jsMatch; const scriptURI = _getRealScriptURI(rawScriptURI); - const resourceIndex = getOrCreateURIResource( - scriptURI, - resourceTable, - stringTable, - originToResourceIndex - ); + const resourceIndex = globalDataCollector.indexForURIResource(scriptURI); // Process the source index if it's provided. let processedSourceIndex = null; @@ -1224,11 +1212,6 @@ function _processThread( tid: thread.tid, pid: `${thread.pid}`, pausedRanges: pausedRanges || [], - frameTable, - funcTable, - nativeSymbols, - resourceTable, - stackTable, markers, samples, }; @@ -1291,7 +1274,7 @@ function _processThread( processJsTracer(); - return nudgeReturnAddresses(newThread); + return newThread; } /** @@ -1838,7 +1821,8 @@ export function processGeckoProfile(geckoProfile: GeckoProfile): Profile { const profileGatheringLog = { ...(geckoProfile.profileGatheringLog || {}) }; const stringTable = globalDataCollector.getStringTable(); - const sources = globalDataCollector.getSources(); + + const { libs, shared } = globalDataCollector.finish(); // Convert JS tracer information into their own threads. This mutates // the threads array. @@ -1848,10 +1832,9 @@ export function processGeckoProfile(geckoProfile: GeckoProfile): Profile { const friendlyThreadName = getFriendlyThreadName(threads, thread); const jsTracerThread = convertJsTracerToThread( thread, + shared, jsTracer, - geckoProfile.meta.categories, - stringTable, - sources + geckoProfile.meta.categories ); jsTracerThread.isJsTracer = true; jsTracerThread.name = `JS Tracer of ${friendlyThreadName}`; @@ -1867,8 +1850,6 @@ export function processGeckoProfile(geckoProfile: GeckoProfile): Profile { processVisualMetrics(threads, meta, pages, stringTable); } - const { libs, shared } = globalDataCollector.finish(); - const result = { meta, libs, diff --git a/src/profile-logic/processed-profile-versioning.ts b/src/profile-logic/processed-profile-versioning.ts index 2448b486e7..b60cb381aa 100644 --- a/src/profile-logic/processed-profile-versioning.ts +++ b/src/profile-logic/processed-profile-versioning.ts @@ -2693,6 +2693,212 @@ const _upgraders: { } profile.shared.sources = sourceTable; }, + [59]: (profile) => { + // The following tables are now shared across all threads: + // - stackTable + // - frameTable + // - funcTable + // - resourceTable + // - nativeSymbols + // They are now stored in profile.shared. + const funcTableMap = new Map(); + const resourceTableMap = new Map(); + const nativeSymbolsMap = new Map(); + const newStackTable = { + frame: [] as Array, + prefix: [] as Array, + length: 0, + }; + const newFrameTable = { + address: [] as Array, + inlineDepth: [] as Array, + category: [] as Array, + subcategory: [] as Array, + func: [] as Array, + nativeSymbol: [] as Array, + innerWindowID: [] as Array, + line: [] as Array, + column: [] as Array, + length: 0, + }; + const newFuncTable = { + name: [] as Array, + isJS: [] as Array, + relevantForJS: [] as Array, + resource: [] as Array, + source: [] as Array, + lineNumber: [] as Array, + columnNumber: [] as Array, + length: 0, + }; + const newResourceTable = { + type: [] as Array, + lib: [] as Array, + name: [] as Array, + host: [] as Array, + length: 0, + }; + const newNativeSymbols = { + libIndex: [] as Array, + address: [] as Array, + name: [] as Array, + functionSize: [] as Array, + length: 0, + }; + for (const thread of profile.threads) { + const { + stackTable, + frameTable, + funcTable, + resourceTable, + nativeSymbols, + samples, + markers, + } = thread; + const stackTableIndexMap = new Int32Array(stackTable.length); + const frameTableIndexMap = new Int32Array(frameTable.length); + const funcTableIndexMap = new Int32Array(funcTable.length); + const resourceTableIndexMap = new Int32Array(resourceTable.length); + const nativeSymbolsIndexMap = new Int32Array(nativeSymbols.length); + (function integrateIntoSharedNativeSymbols() { + for (let i = 0; i < nativeSymbols.length; i++) { + const libIndex = nativeSymbols.libIndex[i]; + const address = nativeSymbols.address[i]; + const key = `${libIndex}-${address}`; + let newIndex = nativeSymbolsMap.get(key); + if (newIndex === undefined) { + newIndex = newNativeSymbols.length++; + nativeSymbolsMap.set(key, newIndex); + newNativeSymbols.libIndex[newIndex] = libIndex; + newNativeSymbols.address[newIndex] = address; + newNativeSymbols.name[newIndex] = nativeSymbols.name[i]; + newNativeSymbols.functionSize[newIndex] = + nativeSymbols.functionSize[i]; + } + nativeSymbolsIndexMap[i] = newIndex; + } + })(); + (function integrateIntoSharedResources() { + for (let i = 0; i < resourceTable.length; i++) { + const type = resourceTable.type[i]; + const lib = resourceTable.lib[i]; + const name = resourceTable.name[i]; + const host = resourceTable.host[i]; + const key = `${type}-${lib !== null ? lib : ''}-${name}-${host !== null ? host : ''}`; + let newIndex = resourceTableMap.get(key); + if (newIndex === undefined) { + newIndex = newResourceTable.length++; + resourceTableMap.set(key, newIndex); + newResourceTable.type[newIndex] = type; + newResourceTable.lib[newIndex] = lib; + newResourceTable.name[newIndex] = name; + newResourceTable.host[newIndex] = host; + } + resourceTableIndexMap[i] = newIndex; + } + })(); + (function integrateIntoSharedFuncTable() { + for (let i = 0; i < funcTable.length; i++) { + const name = funcTable.name[i]; + const isJS = funcTable.isJS[i]; + const relevantForJS = funcTable.relevantForJS[i]; + const oldResource = funcTable.resource[i]; + const resource = + oldResource !== -1 ? resourceTableIndexMap[oldResource] : -1; + const source = funcTable.source[i]; + const lineNumber = funcTable.lineNumber[i]; + const columnNumber = funcTable.columnNumber[i]; + const key = `${name}-${isJS}-${relevantForJS}-${resource}-${source !== null ? source : ''}-${lineNumber !== null ? lineNumber : ''}-${columnNumber !== null ? columnNumber : ''}`; + let newIndex = funcTableMap.get(key); + if (newIndex === undefined) { + newIndex = newFuncTable.length++; + funcTableMap.set(key, newIndex); + newFuncTable.name[newIndex] = name; + newFuncTable.isJS[newIndex] = isJS; + newFuncTable.relevantForJS[newIndex] = relevantForJS; + newFuncTable.resource[newIndex] = resource; + newFuncTable.source[newIndex] = source; + newFuncTable.lineNumber[newIndex] = lineNumber; + newFuncTable.columnNumber[newIndex] = columnNumber; + } + funcTableIndexMap[i] = newIndex; + } + })(); + (function integrateIntoSharedFrameTable() { + // Just copy over all frames from all threads into the shared frameTable + // without doing any deduplication. + // We will end up with duplicated frames, but that's ok. + // For a correct call tree we just need a deduplicated *func* table; it's + // ok if we have duplicated frames because the computation of the call node + // table will do the right thing anyway. + // Deduplicating frames might make the computation of the call node table + // a bit faster, but it would mean we'd have a very slow upgrader, so we'd + // rather keep the upgrader fast. + for (let i = 0; i < frameTable.length; i++) { + const newIndex = newFrameTable.length++; + newFrameTable.address[newIndex] = frameTable.address[i]; + newFrameTable.inlineDepth[newIndex] = frameTable.inlineDepth[i]; + newFrameTable.category[newIndex] = frameTable.category[i]; + newFrameTable.subcategory[newIndex] = frameTable.subcategory[i]; + newFrameTable.func[newIndex] = funcTableIndexMap[frameTable.func[i]]; + const oldNativeSymbol = frameTable.nativeSymbol[i]; + const nativeSymbol = + oldNativeSymbol !== null + ? nativeSymbolsIndexMap[oldNativeSymbol] + : null; + newFrameTable.nativeSymbol[newIndex] = nativeSymbol; + newFrameTable.innerWindowID[newIndex] = frameTable.innerWindowID[i]; + newFrameTable.line[newIndex] = frameTable.line[i]; + newFrameTable.column[newIndex] = frameTable.column[i]; + frameTableIndexMap[i] = newIndex; + } + })(); + (function integrateIntoSharedStackTable() { + // Don't deduplicate stacks; same reason as above. + for (let i = 0; i < stackTable.length; i++) { + const frame = frameTableIndexMap[stackTable.frame[i]]; + const oldPrefix = stackTable.prefix[i]; + const prefix = + oldPrefix !== null ? stackTableIndexMap[oldPrefix] : null; + const newIndex = newStackTable.length++; + newStackTable.frame[newIndex] = frame; + newStackTable.prefix[newIndex] = prefix; + stackTableIndexMap[i] = newIndex; + } + })(); + (function translateSamples() { + for (let i = 0; i < samples.length; i++) { + const oldStack = samples.stack[i]; + const stack = oldStack !== null ? stackTableIndexMap[oldStack] : null; + samples.stack[i] = stack; + } + })(); + (function translateMarkers() { + for (let i = 0; i < markers.length; i++) { + const data = markers.data[i]; + if (!data || !data.cause) { + continue; + } + const oldStack = data.cause.stack; + const stack = + oldStack !== null && oldStack !== undefined + ? stackTableIndexMap[oldStack] + : null; + data.cause.stack = stack; + } + })(); + delete thread.stackTable; + delete thread.frameTable; + delete thread.funcTable; + delete thread.resourceTable; + delete thread.nativeSymbols; + } + profile.shared.stackTable = newStackTable; + profile.shared.frameTable = newFrameTable; + profile.shared.funcTable = newFuncTable; + profile.shared.resourceTable = newResourceTable; + profile.shared.nativeSymbols = newNativeSymbols; + }, // If you add a new upgrader here, please document the change in // `docs-developer/CHANGELOG-formats.md`. }; diff --git a/src/profile-logic/profile-compacting.ts b/src/profile-logic/profile-compacting.ts index 49bd963c15..cb85f7a8b7 100644 --- a/src/profile-logic/profile-compacting.ts +++ b/src/profile-logic/profile-compacting.ts @@ -2,6 +2,14 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { + getEmptyRawStackTable, + getEmptyFrameTable, + getEmptyFuncTable, + getEmptyResourceTable, + getEmptyNativeSymbolTable, + getEmptySourceTable, +} from './data-structures'; import { computeStringIndexMarkerFieldsByDataType } from './marker-schema'; import type { @@ -9,28 +17,61 @@ import type { RawThread, RawProfileSharedData, RawMarkerTable, + IndexIntoStackTable, + RawStackTable, + FrameTable, FuncTable, ResourceTable, NativeSymbolTable, + RawSamplesTable, + NativeAllocationsTable, + JsAllocationsTable, + Lib, SourceTable, } from 'firefox-profiler/types'; export type CompactedProfileWithTranslationMaps = { profile: Profile; - oldStringToNewStringPlusOne: Int32Array; + translationMaps: TranslationMaps; +}; + +type ReferencedProfileData = { + referencedStacks: Uint8Array; + referencedFrames: Uint8Array; + referencedFuncs: Uint8Array; + referencedResources: Uint8Array; + referencedNativeSymbols: Uint8Array; + referencedSources: Uint8Array; + referencedStrings: Uint8Array; + referencedLibs: Uint8Array; +}; + +type TranslationMaps = { + oldStackToNewStackPlusOne: Int32Array; + oldFrameToNewFramePlusOne: Int32Array; + oldFuncToNewFuncPlusOne: Int32Array; + oldResourceToNewResourcePlusOne: Int32Array; + oldNativeSymbolToNewNativeSymbolPlusOne: Int32Array; oldSourceToNewSourcePlusOne: Int32Array; + oldStringToNewStringPlusOne: Int32Array; + oldLibToNewLibPlusOne: Int32Array; }; /** - * Returns a new profile with all unreferenced strings and sources removed. + * Returns a new profile with all unreferenced data removed. * - * Since the string table and source table are shared between all threads, if - * the user asks for a thread to be removed during sanitization, by default - * we'd keep the strings and sources from the removed threads in the profile. + * The markers and samples in the profile are the "GC roots". All other data + * tables exist only to make the marker and sample data meaningful. + * (Here, sample data includes allocation samples from thread.jsAllocations and + * thread.nativeAllocations.) * - * By calling this function, you can get a profile with adjusted string and - * source tables where those unused strings and sources from the removed - * threads have been removed. + * When a profile is uploaded, we allow removing parts of the uploaded data, + * for example by restricting to a time range (which removes samples and markers + * outside of the time range) or by removing entire threads. + * + * computeCompactedProfile makes it so that, once those threads / samples / markers + * are removed, we don't keep around any stacks / frames / strings / etc. which + * were only used by the removed threads / samples / markers. */ export function computeCompactedProfile( profile: Profile @@ -38,164 +79,290 @@ export function computeCompactedProfile( const stringIndexMarkerFieldsByDataType = computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema); - // Step 1: Gather all references of strings. - const referencedStrings = _gatherStringReferencesInProfile( + // Step 1: Gather all references. + const referencedData = _gatherReferencesInProfile( profile, stringIndexMarkerFieldsByDataType ); - // Step 2: Gather all references of sources. - const referencedSources = _gatherSourceReferencesInProfile(profile); - - // Step 3: Adjust all tables to use new string and source indexes. - return _createProfileWithTranslatedIndexes( + // Step 2: Create new tables for everything, skipping unreferenced entries. + return _createCompactedProfile( profile, - referencedStrings, - referencedSources, + referencedData, stringIndexMarkerFieldsByDataType ); } -function _gatherStringReferencesInProfile( +function _gatherReferencesInProfile( profile: Profile, stringIndexMarkerFieldsByDataType: Map -): Uint8Array { - const referencedStrings = new Uint8Array(profile.shared.stringArray.length); +): ReferencedProfileData { + const { shared, threads } = profile; + const referencedSharedData: ReferencedProfileData = { + referencedStacks: new Uint8Array(shared.stackTable.length), + referencedFrames: new Uint8Array(shared.frameTable.length), + referencedFuncs: new Uint8Array(shared.funcTable.length), + referencedResources: new Uint8Array(shared.resourceTable.length), + referencedNativeSymbols: new Uint8Array(shared.nativeSymbols.length), + referencedSources: new Uint8Array(shared.sources.length), + referencedLibs: new Uint8Array(profile.libs.length), + referencedStrings: new Uint8Array(shared.stringArray.length), + }; - for (const thread of profile.threads) { - _gatherStringReferencesInThread( + for (const thread of threads) { + _gatherReferencesInThread( thread, - referencedStrings, - stringIndexMarkerFieldsByDataType, - profile.shared.sources ?? null + referencedSharedData, + stringIndexMarkerFieldsByDataType ); } - return referencedStrings; -} - -function _gatherSourceReferencesInProfile(profile: Profile): Uint8Array { - const referencedSources = new Uint8Array(profile.shared.sources.length); - - for (const thread of profile.threads) { - _gatherSourceReferencesInThread(thread, referencedSources); - } - - return referencedSources; -} + _gatherReferencesInStackTable(shared.stackTable, referencedSharedData); + _gatherReferencesInFrameTable(shared.frameTable, referencedSharedData); + _gatherReferencesInFuncTable(shared.funcTable, referencedSharedData); + _gatherReferencesInResourceTable(shared.resourceTable, referencedSharedData); + _gatherReferencesInNativeSymbols(shared.nativeSymbols, referencedSharedData); + _gatherReferencesInSources(shared.sources, referencedSharedData); -function _gatherSourceReferencesInThread( - thread: RawThread, - referencedSources: Uint8Array -) { - for (let i = 0; i < thread.funcTable.length; i++) { - const sourceIndex = thread.funcTable.source[i]; - if (sourceIndex !== null) { - referencedSources[sourceIndex] = 1; - } - } + return referencedSharedData; } -function _createProfileWithTranslatedIndexes( +function _createCompactedProfile( profile: Profile, - referencedStrings: Uint8Array, - referencedSources: Uint8Array, + referencedSharedData: ReferencedProfileData, stringIndexMarkerFieldsByDataType: Map ): CompactedProfileWithTranslationMaps { - const { newStringArray, oldStringToNewStringPlusOne } = - _createCompactedStringArray(profile.shared.stringArray, referencedStrings); - - const { newSources, oldSourceToNewSourcePlusOne } = - _createCompactedSourceTable( - profile.shared.sources, - referencedSources, - oldStringToNewStringPlusOne - ); + const { shared } = profile; + const translationMaps: TranslationMaps = { + oldStackToNewStackPlusOne: new Int32Array(shared.stackTable.length), + oldFrameToNewFramePlusOne: new Int32Array(shared.frameTable.length), + oldFuncToNewFuncPlusOne: new Int32Array(shared.funcTable.length), + oldResourceToNewResourcePlusOne: new Int32Array( + shared.resourceTable.length + ), + oldNativeSymbolToNewNativeSymbolPlusOne: new Int32Array( + shared.nativeSymbols.length + ), + oldSourceToNewSourcePlusOne: new Int32Array(shared.sources.length), + oldStringToNewStringPlusOne: new Int32Array(shared.stringArray.length), + oldLibToNewLibPlusOne: new Int32Array(profile.libs.length), + }; - const newThreads = profile.threads.map((thread) => - _createThreadWithTranslatedIndexes( - thread, - oldStringToNewStringPlusOne, - oldSourceToNewSourcePlusOne, - stringIndexMarkerFieldsByDataType - ) + const newStringArray = _createCompactedStringArray( + shared.stringArray, + referencedSharedData, + translationMaps + ); + const newLibs = _createCompactedLibs( + profile.libs, + referencedSharedData, + translationMaps + ); + const newSources = _createCompactedSources( + shared.sources, + referencedSharedData, + translationMaps + ); + const newNativeSymbols = _createCompactedNativeSymbols( + shared.nativeSymbols, + referencedSharedData, + translationMaps + ); + const newResourceTable = _createCompactedResourceTable( + shared.resourceTable, + referencedSharedData, + translationMaps + ); + const newFuncTable = _createCompactedFuncTable( + shared.funcTable, + referencedSharedData, + translationMaps + ); + const newFrameTable = _createCompactedFrameTable( + shared.frameTable, + referencedSharedData, + translationMaps + ); + const newStackTable = _createCompactedStackTable( + shared.stackTable, + referencedSharedData, + translationMaps + ); + + const newThreads = profile.threads.map( + (thread): RawThread => + _createCompactedThread( + thread, + translationMaps, + stringIndexMarkerFieldsByDataType + ) ); const newShared: RawProfileSharedData = { stringArray: newStringArray, sources: newSources, + nativeSymbols: newNativeSymbols, + resourceTable: newResourceTable, + funcTable: newFuncTable, + frameTable: newFrameTable, + stackTable: newStackTable, }; const newProfile: Profile = { ...profile, + libs: newLibs, shared: newShared, threads: newThreads, }; return { profile: newProfile, - oldStringToNewStringPlusOne, - oldSourceToNewSourcePlusOne, + translationMaps, }; } -function _gatherStringReferencesInThread( +function _gatherReferencesInThread( thread: RawThread, - referencedStrings: Uint8Array, - stringIndexMarkerFieldsByDataType: Map, - sources: SourceTable + referencedSharedData: ReferencedProfileData, + stringIndexMarkerFieldsByDataType: Map ) { + _gatherReferencesInSamples(thread.samples, referencedSharedData); + if (thread.jsAllocations) { + _gatherReferencesInJsAllocations( + thread.jsAllocations, + referencedSharedData + ); + } + if (thread.nativeAllocations) { + _gatherReferencesInNativeAllocations( + thread.nativeAllocations, + referencedSharedData + ); + } _gatherReferencesInMarkers( thread.markers, - referencedStrings, - stringIndexMarkerFieldsByDataType + stringIndexMarkerFieldsByDataType, + referencedSharedData ); - - _gatherReferencesInFuncTable(thread.funcTable, referencedStrings, sources); - _gatherReferencesInResourceTable(thread.resourceTable, referencedStrings); - _gatherReferencesInNativeSymbols(thread.nativeSymbols, referencedStrings); } -function _createThreadWithTranslatedIndexes( +function _createCompactedThread( thread: RawThread, - oldStringToNewStringPlusOne: Int32Array, - oldSourceToNewSourcePlusOne: Int32Array, + translationMaps: TranslationMaps, stringIndexMarkerFieldsByDataType: Map ): RawThread { - const newNativeSymbols = _createNativeSymbolsWithTranslatedStringIndexes( - thread.nativeSymbols, - oldStringToNewStringPlusOne - ); - const newResourceTable = _createResourceTableWithTranslatedStringIndexes( - thread.resourceTable, - oldStringToNewStringPlusOne - ); - const newFuncTable = _createFuncTableWithTranslatedIndexes( - thread.funcTable, - oldStringToNewStringPlusOne, - oldSourceToNewSourcePlusOne - ); - const newMarkers = _createMarkersWithTranslatedStringIndexes( + const newSamples = _createCompactedSamples(thread.samples, translationMaps); + const newJsAllocations = thread.jsAllocations + ? _createCompactedJsAllocations(thread.jsAllocations, translationMaps) + : undefined; + const newNativeAllocations = thread.nativeAllocations + ? _createCompactedNativeAllocations( + thread.nativeAllocations, + translationMaps + ) + : undefined; + const newMarkers = _createCompactedMarkers( thread.markers, - oldStringToNewStringPlusOne, + translationMaps, stringIndexMarkerFieldsByDataType ); const newThread: RawThread = { ...thread, - nativeSymbols: newNativeSymbols, - resourceTable: newResourceTable, - funcTable: newFuncTable, + samples: newSamples, + jsAllocations: newJsAllocations, + nativeAllocations: newNativeAllocations, markers: newMarkers, }; return newThread; } +function _gatherReferencesInSamples( + samples: RawSamplesTable, + references: ReferencedProfileData +) { + _gatherReferencesInStackCol(samples.stack, references); +} + +function _createCompactedSamples( + samples: RawSamplesTable, + translationMaps: TranslationMaps +): RawSamplesTable { + return { + ...samples, + stack: _translateStackCol(samples.stack, translationMaps), + }; +} + +function _gatherReferencesInJsAllocations( + jsAllocations: JsAllocationsTable, + references: ReferencedProfileData +) { + _gatherReferencesInStackCol(jsAllocations.stack, references); +} + +function _createCompactedJsAllocations( + jsAllocations: JsAllocationsTable, + translationMaps: TranslationMaps +): JsAllocationsTable { + return { + ...jsAllocations, + stack: _translateStackCol(jsAllocations.stack, translationMaps), + }; +} + +function _gatherReferencesInNativeAllocations( + nativeAllocations: NativeAllocationsTable, + references: ReferencedProfileData +) { + _gatherReferencesInStackCol(nativeAllocations.stack, references); +} + +function _createCompactedNativeAllocations( + nativeAllocations: NativeAllocationsTable, + translationMaps: TranslationMaps +): NativeAllocationsTable { + return { + ...nativeAllocations, + stack: _translateStackCol(nativeAllocations.stack, translationMaps), + }; +} + +function _gatherReferencesInStackCol( + stackCol: Array, + references: ReferencedProfileData +) { + const { referencedStacks } = references; + for (let i = 0; i < stackCol.length; i++) { + const stack = stackCol[i]; + if (stack !== null) { + referencedStacks[stack] = 1; + } + } +} + +function _translateStackCol( + stackCol: Array, + translationMaps: TranslationMaps +): Array { + const { oldStackToNewStackPlusOne } = translationMaps; + const newStackCol = stackCol.slice(); + + for (let i = 0; i < stackCol.length; i++) { + const stack = stackCol[i]; + newStackCol[i] = + stack !== null ? oldStackToNewStackPlusOne[stack] - 1 : null; + } + + return newStackCol; +} + function _gatherReferencesInMarkers( markers: RawMarkerTable, - referencedStrings: Uint8Array, - stringIndexMarkerFieldsByDataType: Map + stringIndexMarkerFieldsByDataType: Map, + references: ReferencedProfileData ) { + const { referencedStacks, referencedStrings } = references; for (let i = 0; i < markers.length; i++) { referencedStrings[markers.name[i]] = 1; @@ -204,6 +371,13 @@ function _gatherReferencesInMarkers( continue; } + if ('cause' in data && data.cause) { + const stack = data.cause.stack; + if (stack !== null) { + referencedStacks[stack] = 1; + } + } + if (data.type) { const stringIndexMarkerFields = stringIndexMarkerFieldsByDataType.get( data.type @@ -220,11 +394,13 @@ function _gatherReferencesInMarkers( } } -function _createMarkersWithTranslatedStringIndexes( +function _createCompactedMarkers( markers: RawMarkerTable, - oldStringToNewStringPlusOne: Int32Array, + translationMaps: TranslationMaps, stringIndexMarkerFieldsByDataType: Map ): RawMarkerTable { + const { oldStackToNewStackPlusOne, oldStringToNewStringPlusOne } = + translationMaps; const newDataCol = markers.data.slice(); const newNameCol = markers.name.slice(); for (let i = 0; i < markers.length; i++) { @@ -236,6 +412,19 @@ function _createMarkersWithTranslatedStringIndexes( } let newData = data; + if ('cause' in newData && newData.cause) { + const stack = newData.cause.stack; + if (stack !== null) { + newData = { + ...newData, + cause: { + ...newData.cause, + stack: oldStackToNewStackPlusOne[stack] - 1, + }, + }; + } + } + if (data.type) { const stringIndexMarkerFields = stringIndexMarkerFieldsByDataType.get( data.type @@ -263,117 +452,327 @@ function _createMarkersWithTranslatedStringIndexes( }; } +function _gatherReferencesInStackTable( + stackTable: RawStackTable, + references: ReferencedProfileData +) { + const { referencedStacks, referencedFrames } = references; + for (let i = stackTable.length - 1; i >= 0; i--) { + if (referencedStacks[i] === 0) { + continue; + } + + const prefix = stackTable.prefix[i]; + if (prefix !== null) { + referencedStacks[prefix] = 1; + } + referencedFrames[stackTable.frame[i]] = 1; + } +} + +function _createCompactedStackTable( + stackTable: RawStackTable, + { referencedStacks }: ReferencedProfileData, + translationMaps: TranslationMaps +): RawStackTable { + const { oldStackToNewStackPlusOne, oldFrameToNewFramePlusOne } = + translationMaps; + const newStackTable = getEmptyRawStackTable(); + for (let i = 0; i < stackTable.length; i++) { + if (referencedStacks[i] === 0) { + continue; + } + + const prefix = stackTable.prefix[i]; + + const newIndex = newStackTable.length++; + newStackTable.prefix[newIndex] = + prefix !== null ? oldStackToNewStackPlusOne[prefix] - 1 : null; + newStackTable.frame[newIndex] = + oldFrameToNewFramePlusOne[stackTable.frame[i]] - 1; + + oldStackToNewStackPlusOne[i] = newIndex + 1; + } + + return newStackTable; +} + +function _gatherReferencesInFrameTable( + frameTable: FrameTable, + references: ReferencedProfileData +) { + const { referencedFrames, referencedFuncs, referencedNativeSymbols } = + references; + for (let i = 0; i < frameTable.length; i++) { + if (referencedFrames[i] === 0) { + continue; + } + + referencedFuncs[frameTable.func[i]] = 1; + + const nativeSymbol = frameTable.nativeSymbol[i]; + if (nativeSymbol !== null) { + referencedNativeSymbols[nativeSymbol] = 1; + } + } +} + +function _createCompactedFrameTable( + frameTable: FrameTable, + { referencedFrames }: ReferencedProfileData, + translationMaps: TranslationMaps +): FrameTable { + const { + oldFrameToNewFramePlusOne, + oldFuncToNewFuncPlusOne, + oldNativeSymbolToNewNativeSymbolPlusOne, + } = translationMaps; + const newFrameTable = getEmptyFrameTable(); + for (let i = 0; i < frameTable.length; i++) { + if (referencedFrames[i] === 0) { + continue; + } + + const nativeSymbol = frameTable.nativeSymbol[i]; + + const newIndex = newFrameTable.length++; + newFrameTable.address[newIndex] = frameTable.address[i]; + newFrameTable.inlineDepth[newIndex] = frameTable.inlineDepth[i]; + newFrameTable.category[newIndex] = frameTable.category[i]; + newFrameTable.subcategory[newIndex] = frameTable.subcategory[i]; + newFrameTable.func[newIndex] = + oldFuncToNewFuncPlusOne[frameTable.func[i]] - 1; + newFrameTable.nativeSymbol[newIndex] = + nativeSymbol !== null + ? oldNativeSymbolToNewNativeSymbolPlusOne[nativeSymbol] - 1 + : null; + newFrameTable.innerWindowID[newIndex] = frameTable.innerWindowID[i]; + newFrameTable.line[newIndex] = frameTable.line[i]; + newFrameTable.column[newIndex] = frameTable.column[i]; + + oldFrameToNewFramePlusOne[i] = newIndex + 1; + } + + return newFrameTable; +} + function _gatherReferencesInFuncTable( funcTable: FuncTable, - referencedStrings: Uint8Array, - sources: SourceTable + references: ReferencedProfileData ) { + const { + referencedFuncs, + referencedStrings, + referencedSources, + referencedResources, + } = references; for (let i = 0; i < funcTable.length; i++) { + if (referencedFuncs[i] === 0) { + continue; + } + referencedStrings[funcTable.name[i]] = 1; - const sourceIndex = funcTable.source[i]; - if (sourceIndex !== null) { - const urlIndex = sources.filename[sourceIndex]; - referencedStrings[urlIndex] = 1; + const source = funcTable.source[i]; + if (source !== null) { + referencedSources[source] = 1; + } + + const resource = funcTable.resource[i]; + if (resource !== -1) { + referencedResources[resource] = 1; } } } -function _createFuncTableWithTranslatedIndexes( +function _createCompactedFuncTable( funcTable: FuncTable, - oldStringToNewStringPlusOne: Int32Array, - oldSourceToNewSourcePlusOne: Int32Array + { referencedFuncs }: ReferencedProfileData, + translationMaps: TranslationMaps ): FuncTable { - const newFuncTableNameCol = funcTable.name.slice(); - const newFuncTableSourceCol = funcTable.source.slice(); + const { + oldFuncToNewFuncPlusOne, + oldResourceToNewResourcePlusOne, + oldSourceToNewSourcePlusOne, + oldStringToNewStringPlusOne, + } = translationMaps; + const newFuncTable = getEmptyFuncTable(); for (let i = 0; i < funcTable.length; i++) { - const name = funcTable.name[i]; - newFuncTableNameCol[i] = oldStringToNewStringPlusOne[name] - 1; - - // Translate source indexes to new compacted source table. - const sourceIndex = funcTable.source[i]; - if (sourceIndex !== null) { - const newSourceIndexPlusOne = oldSourceToNewSourcePlusOne[sourceIndex]; - newFuncTableSourceCol[i] = newSourceIndexPlusOne - 1; + if (referencedFuncs[i] === 0) { + continue; } + + const resource = funcTable.resource[i]; + const source = funcTable.source[i]; + + const newIndex = newFuncTable.length++; + newFuncTable.name[newIndex] = + oldStringToNewStringPlusOne[funcTable.name[i]] - 1; + newFuncTable.isJS[newIndex] = funcTable.isJS[i]; + newFuncTable.relevantForJS[newIndex] = funcTable.relevantForJS[i]; + newFuncTable.resource[newIndex] = + resource !== -1 ? oldResourceToNewResourcePlusOne[resource] - 1 : -1; + newFuncTable.source[newIndex] = + source !== null ? oldSourceToNewSourcePlusOne[source] - 1 : null; + newFuncTable.lineNumber[newIndex] = funcTable.lineNumber[i]; + newFuncTable.columnNumber[newIndex] = funcTable.columnNumber[i]; + + oldFuncToNewFuncPlusOne[i] = newIndex + 1; } - const newFuncTable = { - ...funcTable, - name: newFuncTableNameCol, - source: newFuncTableSourceCol, - }; return newFuncTable; } function _gatherReferencesInResourceTable( resourceTable: ResourceTable, - referencedStrings: Uint8Array + references: ReferencedProfileData ) { + const { referencedResources, referencedStrings, referencedLibs } = references; for (let i = 0; i < resourceTable.length; i++) { + if (referencedResources[i] === 0) { + continue; + } + referencedStrings[resourceTable.name[i]] = 1; const host = resourceTable.host[i]; if (host !== null) { referencedStrings[host] = 1; } + + const lib = resourceTable.lib[i]; + if (lib !== null) { + referencedLibs[lib] = 1; + } } } -function _createResourceTableWithTranslatedStringIndexes( +function _createCompactedResourceTable( resourceTable: ResourceTable, - oldStringToNewStringPlusOne: Int32Array + { referencedResources }: ReferencedProfileData, + translationMaps: TranslationMaps ): ResourceTable { - const newResourceTableNameCol = resourceTable.name.slice(); - const newResourceTableHostCol = resourceTable.host.slice(); + const { + oldResourceToNewResourcePlusOne, + oldStringToNewStringPlusOne, + oldLibToNewLibPlusOne, + } = translationMaps; + const newResourceTable = getEmptyResourceTable(); for (let i = 0; i < resourceTable.length; i++) { - const name = newResourceTableNameCol[i]; - newResourceTableNameCol[i] = oldStringToNewStringPlusOne[name] - 1; + if (referencedResources[i] === 0) { + continue; + } + + const host = resourceTable.host[i]; + const lib = resourceTable.lib[i]; - const host = newResourceTableHostCol[i]; - newResourceTableHostCol[i] = + const newIndex = newResourceTable.length++; + newResourceTable.name[newIndex] = + oldStringToNewStringPlusOne[resourceTable.name[i]] - 1; + newResourceTable.host[newIndex] = host !== null ? oldStringToNewStringPlusOne[host] - 1 : null; + newResourceTable.lib[newIndex] = + lib !== null ? oldLibToNewLibPlusOne[lib] - 1 : null; + newResourceTable.type[newIndex] = resourceTable.type[i]; + + oldResourceToNewResourcePlusOne[i] = newIndex + 1; } - const newResourceTable = { - ...resourceTable, - name: newResourceTableNameCol, - host: newResourceTableHostCol, - }; return newResourceTable; } function _gatherReferencesInNativeSymbols( nativeSymbols: NativeSymbolTable, - referencedStrings: Uint8Array + references: ReferencedProfileData ) { + const { referencedNativeSymbols, referencedStrings, referencedLibs } = + references; for (let i = 0; i < nativeSymbols.length; i++) { + if (referencedNativeSymbols[i] === 0) { + continue; + } + referencedStrings[nativeSymbols.name[i]] = 1; + referencedLibs[nativeSymbols.libIndex[i]] = 1; } } -function _createNativeSymbolsWithTranslatedStringIndexes( +function _createCompactedNativeSymbols( nativeSymbols: NativeSymbolTable, - oldStringToNewStringPlusOne: Int32Array + { referencedNativeSymbols }: ReferencedProfileData, + translationMaps: TranslationMaps ): NativeSymbolTable { - const newNativeSymbolsNameCol = nativeSymbols.name.slice(); + const { + oldNativeSymbolToNewNativeSymbolPlusOne, + oldStringToNewStringPlusOne, + oldLibToNewLibPlusOne, + } = translationMaps; + const newNativeSymbols = getEmptyNativeSymbolTable(); for (let i = 0; i < nativeSymbols.length; i++) { - newNativeSymbolsNameCol[i] = - oldStringToNewStringPlusOne[newNativeSymbolsNameCol[i]] - 1; + if (referencedNativeSymbols[i] === 0) { + continue; + } + + const newIndex = newNativeSymbols.length++; + newNativeSymbols.name[newIndex] = + oldStringToNewStringPlusOne[nativeSymbols.name[i]] - 1; + newNativeSymbols.libIndex[newIndex] = + oldLibToNewLibPlusOne[nativeSymbols.libIndex[i]] - 1; + newNativeSymbols.address[newIndex] = nativeSymbols.address[i]; + newNativeSymbols.functionSize[newIndex] = nativeSymbols.functionSize[i]; + + oldNativeSymbolToNewNativeSymbolPlusOne[i] = newIndex + 1; } - const newNativeSymbols = { - ...nativeSymbols, - name: newNativeSymbolsNameCol, - }; return newNativeSymbols; } +function _gatherReferencesInSources( + sources: SourceTable, + references: ReferencedProfileData +) { + const { referencedSources, referencedStrings } = references; + for (let i = 0; i < sources.length; i++) { + if (referencedSources[i] === 0) { + continue; + } + + referencedStrings[sources.filename[i]] = 1; + } +} + +function _createCompactedSources( + sources: SourceTable, + { referencedSources }: ReferencedProfileData, + translationMaps: TranslationMaps +): SourceTable { + const { + oldNativeSymbolToNewNativeSymbolPlusOne, + oldStringToNewStringPlusOne, + } = translationMaps; + const newSources = getEmptySourceTable(); + for (let i = 0; i < sources.length; i++) { + if (referencedSources[i] === 0) { + continue; + } + + const newIndex = newSources.length++; + newSources.filename[newIndex] = + oldStringToNewStringPlusOne[sources.filename[i]] - 1; + newSources.uuid[newIndex] = sources.uuid[i]; + + oldNativeSymbolToNewNativeSymbolPlusOne[i] = newIndex + 1; + } + + return newSources; +} + function _createCompactedStringArray( stringArray: string[], - referencedStrings: Uint8Array -): { newStringArray: string[]; oldStringToNewStringPlusOne: Int32Array } { - const oldStringToNewStringPlusOne = new Int32Array(stringArray.length); + { referencedStrings }: ReferencedProfileData, + translationMaps: TranslationMaps +): string[] { + const { oldStringToNewStringPlusOne } = translationMaps; let nextIndex = 0; const newStringArray = []; for (let i = 0; i < stringArray.length; i++) { @@ -386,46 +785,27 @@ function _createCompactedStringArray( oldStringToNewStringPlusOne[i] = newIndex + 1; } - return { newStringArray, oldStringToNewStringPlusOne }; + return newStringArray; } -function _createCompactedSourceTable( - sourceTable: SourceTable, - referencedSources: Uint8Array, - oldStringToNewStringPlusOne: Int32Array -): { newSources: SourceTable; oldSourceToNewSourcePlusOne: Int32Array } { - const oldSourceToNewSourcePlusOne = new Int32Array(sourceTable.length); +function _createCompactedLibs( + libs: Lib[], + referencedSharedData: ReferencedProfileData, + translationMaps: TranslationMaps +): Lib[] { + const { referencedLibs } = referencedSharedData; + const { oldLibToNewLibPlusOne } = translationMaps; let nextIndex = 0; - const newUuid = []; - const newFilename = []; - - for (let i = 0; i < sourceTable.length; i++) { - if (referencedSources[i] === 0) { + const newLibs = []; + for (let i = 0; i < libs.length; i++) { + if (referencedLibs[i] === 0) { continue; } const newIndex = nextIndex++; - newUuid[newIndex] = sourceTable.uuid[i]; - - // Translate the filename string index - const oldFilenameIndex = sourceTable.filename[i]; - const newFilenameIndexPlusOne = - oldStringToNewStringPlusOne[oldFilenameIndex]; - if (newFilenameIndexPlusOne === 0) { - throw new Error( - `String index ${oldFilenameIndex} was not found in the translation map` - ); - } - newFilename[newIndex] = newFilenameIndexPlusOne - 1; - - oldSourceToNewSourcePlusOne[i] = newIndex + 1; + newLibs[newIndex] = libs[i]; + oldLibToNewLibPlusOne[i] = newIndex + 1; } - const newSources: SourceTable = { - length: nextIndex, - uuid: newUuid, - filename: newFilename, - }; - - return { newSources, oldSourceToNewSourcePlusOne }; + return newLibs; } diff --git a/src/profile-logic/profile-data.ts b/src/profile-logic/profile-data.ts index 21af66fcd8..e6b1e346f5 100644 --- a/src/profile-logic/profile-data.ts +++ b/src/profile-logic/profile-data.ts @@ -1586,11 +1586,9 @@ export function computeTimeColumnForRawSamplesTable( */ export function hasUsefulSamples( sampleStacks: Array | undefined, - thread: RawThread, shared: RawProfileSharedData ): boolean { - const { stringArray } = shared; - const { stackTable, frameTable, funcTable } = thread; + const { stackTable, frameTable, funcTable, stringArray } = shared; if ( sampleStacks === undefined || sampleStacks.length === 0 || @@ -2328,6 +2326,10 @@ export function createThreadFromDerivedTables( rawThread: RawThread, samples: SamplesTable, stackTable: StackTable, + frameTable: FrameTable, + funcTable: FuncTable, + nativeSymbols: NativeSymbolTable, + resourceTable: ResourceTable, stringTable: StringTable, sources: SourceTable ): Thread { @@ -2349,10 +2351,6 @@ export function createThreadFromDerivedTables( jsAllocations, nativeAllocations, markers, - frameTable, - funcTable, - resourceTable, - nativeSymbols, jsTracer, isPrivateBrowsing, userContextId, @@ -2377,10 +2375,6 @@ export function createThreadFromDerivedTables( jsAllocations, nativeAllocations, markers, - frameTable, - funcTable, - resourceTable, - nativeSymbols, jsTracer, isPrivateBrowsing, userContextId, @@ -2388,6 +2382,10 @@ export function createThreadFromDerivedTables( // These fields are derived: samples, stackTable, + frameTable, + funcTable, + resourceTable, + nativeSymbols, stringTable, sources, }; @@ -2504,23 +2502,17 @@ export function updateThreadStacks( } /** - * Updates the stackTable and all references to stacks in the raw thread. + * Updates all references to stacks in the raw threads. * * This function is used by symbolication, which acts on the raw thread. */ export function updateRawThreadStacks( - thread: RawThread, - newStackTable: RawStackTable, + threads: RawThread[], convertStack: ( oldStack: IndexIntoStackTable | null ) => IndexIntoStackTable | null -): RawThread { - return updateRawThreadStacksSeparate( - thread, - newStackTable, - convertStack, - convertStack - ); +): RawThread[] { + return updateRawThreadStacksSeparate(threads, convertStack, convertStack); } /** @@ -2535,8 +2527,25 @@ export function updateRawThreadStacks( * which act on the raw thread. */ export function updateRawThreadStacksSeparate( + threads: RawThread[], + convertStack: ( + oldStack: IndexIntoStackTable | null + ) => IndexIntoStackTable | null, + convertSyncBacktraceStack: ( + oldStack: IndexIntoStackTable | null + ) => IndexIntoStackTable | null +): RawThread[] { + return threads.map((thread) => + updateSingleRawThreadStacksSeparate( + thread, + convertStack, + convertSyncBacktraceStack + ) + ); +} + +export function updateSingleRawThreadStacksSeparate( thread: RawThread, - newStackTable: RawStackTable, convertStack: ( oldStack: IndexIntoStackTable | null ) => IndexIntoStackTable | null, @@ -2576,7 +2585,6 @@ export function updateRawThreadStacksSeparate( ...thread, samples: newSamples, markers: newMarkers, - stackTable: newStackTable, }; if (jsAllocations) { @@ -3360,59 +3368,6 @@ export function extractProfileFilterPageData( return pageDataByTabID; } -// Returns the resource index for a "url" or "webhost" resource which is created -// on demand based on the script URI. -export function getOrCreateURIResource( - scriptURI: string, - resourceTable: ResourceTable, - stringTable: StringTable, - originToResourceIndex: Map -): IndexIntoResourceTable { - // Figure out the origin and host. - let origin; - let host; - try { - const url = new URL(scriptURI); - if ( - !( - url.protocol === 'http:' || - url.protocol === 'https:' || - url.protocol === 'moz-extension:' - ) - ) { - throw new Error('not a webhost or extension protocol'); - } - origin = url.origin; - host = url.host; - } catch (_e) { - origin = scriptURI; - host = null; - } - - let resourceIndex = originToResourceIndex.get(origin); - if (resourceIndex !== undefined) { - return resourceIndex; - } - - resourceIndex = resourceTable.length++; - originToResourceIndex.set(origin, resourceIndex); - if (host) { - // This is a webhost URL. - resourceTable.lib[resourceIndex] = null; - resourceTable.name[resourceIndex] = stringTable.indexForString(origin); - resourceTable.host[resourceIndex] = stringTable.indexForString(host); - resourceTable.type[resourceIndex] = resourceTypes.webhost; - } else { - // This is a URL, but it doesn't point to something on the web, e.g. a - // chrome url. - resourceTable.lib[resourceIndex] = null; - resourceTable.name[resourceIndex] = stringTable.indexForString(scriptURI); - resourceTable.host[resourceIndex] = null; - resourceTable.type[resourceIndex] = resourceTypes.url; - } - return resourceIndex; -} - /** * See the ThreadsKey type for an explanation. */ @@ -3465,10 +3420,25 @@ export type StackReferences = { * samples, and stacks referenced by sync backtraces (e.g. marker causes). * The two have slightly different properties, see the type definition. */ -export function gatherStackReferences(thread: RawThread): StackReferences { +export function gatherStackReferences(threads: RawThread[]): StackReferences { const samplingSelfStacks: Set = new Set(); const syncBacktraceSelfStacks: Set = new Set(); + for (const thread of threads) { + _gatherSingleThreadStackReferences( + thread, + samplingSelfStacks, + syncBacktraceSelfStacks + ); + } + return { samplingSelfStacks, syncBacktraceSelfStacks }; +} + +export function _gatherSingleThreadStackReferences( + thread: RawThread, + samplingSelfStacks: Set, + syncBacktraceSelfStacks: Set +) { const { samples, markers, jsAllocations, nativeAllocations } = thread; // Samples @@ -3509,8 +3479,6 @@ export function gatherStackReferences(thread: RawThread): StackReferences { } } } - - return { samplingSelfStacks, syncBacktraceSelfStacks }; } /** @@ -3636,11 +3604,12 @@ export function gatherStackReferences(thread: RawThread): StackReferences { * used in both contexts. If we detect that this happened, we need to duplicate * the frame and the stack node and pick the right one depending on the use. */ -export function nudgeReturnAddresses(thread: RawThread): RawThread { - const { samplingSelfStacks, syncBacktraceSelfStacks } = - gatherStackReferences(thread); +export function nudgeReturnAddresses(profile: Profile): Profile { + const { samplingSelfStacks, syncBacktraceSelfStacks } = gatherStackReferences( + profile.threads + ); - const { stackTable, frameTable } = thread; + const { stackTable, frameTable } = profile.shared; // Collect frames that were obtained from the instruction pointer. // These are the top ("self") frames of stacks from sampling. @@ -3683,7 +3652,7 @@ export function nudgeReturnAddresses(thread: RawThread): RawThread { if (ipFrames.size === 0 && returnAddressFrames.size === 0) { // Nothing to do, use the original thread. - return thread; + return profile; } // Create the new frame table. @@ -3769,17 +3738,25 @@ export function nudgeReturnAddresses(thread: RawThread): RawThread { } } - const newThread: RawThread = { - ...thread, + const newShared: RawProfileSharedData = { + ...profile.shared, frameTable: newFrameTable, + stackTable: newStackTable, }; - return updateRawThreadStacksSeparate( - newThread, - newStackTable, + const newThreads = updateRawThreadStacksSeparate( + profile.threads, getMapStackUpdater(mapForSamplingSelfStacks), getMapStackUpdater(mapForBacktraceSelfStacks) ); + + const newProfile: Profile = { + ...profile, + shared: newShared, + threads: newThreads, + }; + + return newProfile; } /** @@ -3791,37 +3768,34 @@ export function findAddressProofForFile( sourceIndex: IndexIntoSourceTable ): AddressProof | null { const { libs } = profile; - for (const thread of profile.threads) { - const { frameTable, funcTable, resourceTable } = thread; - const func = funcTable.source.indexOf(sourceIndex); - if (func === -1) { - continue; - } - const frame = frameTable.func.indexOf(func); - if (frame === -1) { - continue; - } - const address = frameTable.address[frame]; - if (address === null) { - continue; - } - const resource = funcTable.resource[func]; - if (resourceTable.type[resource] !== resourceTypes.library) { - continue; - } - const libIndex = resourceTable.lib[resource]; - if (libIndex === null) { - continue; - } - const lib = libs[libIndex]; - const { debugName, breakpadId } = lib; - return { - debugName, - breakpadId, - address, - }; + const { frameTable, funcTable, resourceTable } = profile.shared; + const func = funcTable.source.indexOf(sourceIndex); + if (func === -1) { + return null; } - return null; + const frame = frameTable.func.indexOf(func); + if (frame === -1) { + return null; + } + const address = frameTable.address[frame]; + if (address === null) { + return null; + } + const resource = funcTable.resource[func]; + if (resourceTable.type[resource] !== resourceTypes.library) { + return null; + } + const libIndex = resourceTable.lib[resource]; + if (libIndex === null) { + return null; + } + const lib = libs[libIndex]; + const { debugName, breakpadId } = lib; + return { + debugName, + breakpadId, + address, + }; } /** @@ -4114,12 +4088,14 @@ export function computeTabToThreadIndexesMap( // very cheap, but it'll allow us to not compute this information every // time when we need it. for (let threadIdx = 0; threadIdx < threads.length; threadIdx++) { - const thread = threads[threadIdx]; + const { usedInnerWindowIDs } = threads[threadIdx]; + if (usedInnerWindowIDs === undefined) { + continue; + } // First go over the innerWindowIDs of the samples. - for (let i = 0; i < thread.frameTable.length; i++) { - const innerWindowID = thread.frameTable.innerWindowID[i]; - if (innerWindowID === null || innerWindowID === 0) { + for (const innerWindowID of usedInnerWindowIDs) { + if (innerWindowID === 0) { // Zero value also means null for innerWindowID. continue; } @@ -4138,38 +4114,6 @@ export function computeTabToThreadIndexesMap( } threadIndexes.add(threadIdx); } - - // Then go over the markers to find their innerWindowIDs. - for (let i = 0; i < thread.markers.length; i++) { - const markerData = thread.markers.data[i]; - - if (!markerData) { - continue; - } - - if ( - 'innerWindowID' in markerData && - markerData.innerWindowID !== null && - markerData.innerWindowID !== undefined && - // Zero value also means null for innerWindowID. - markerData.innerWindowID !== 0 - ) { - const innerWindowID = markerData.innerWindowID; - const tabID = innerWindowIDToTabMap.get(innerWindowID); - if (tabID === undefined) { - // We couldn't find the tab of this innerWindowID, this should - // never happen, it might indicate a bug in Firefox. - continue; - } - - let threadIndexes = tabToThreadIndexesMap.get(tabID); - if (!threadIndexes) { - threadIndexes = new Set(); - tabToThreadIndexesMap.set(tabID, threadIndexes); - } - threadIndexes.add(threadIdx); - } - } } return tabToThreadIndexesMap; diff --git a/src/profile-logic/sanitize.ts b/src/profile-logic/sanitize.ts index e9eefdd8f7..84b2dd6cd9 100644 --- a/src/profile-logic/sanitize.ts +++ b/src/profile-logic/sanitize.ts @@ -36,6 +36,7 @@ import type { MarkerSchemaByName, RawCounter, ProfilerOverhead, + IndexIntoResourceTable, } from 'firefox-profiler/types'; export type SanitizeProfileResult = { @@ -45,6 +46,9 @@ export type SanitizeProfileResult = { readonly isSanitized: boolean; }; +// Some constants to make it easier to read. +const PRIVATE_BROWSING_STACK = 1; + /** * Take a processed profile with PII that user wants to be removed and remove the * thread data depending on that PII status. Look at `RemoveProfileInformation` @@ -116,6 +120,174 @@ export function sanitizePII( stringArray = stringArray.map((s) => removeURLs(s)); } const stringTable = StringTable.withBackingArray(stringArray); + const newShared = { + ...profile.shared, + stringArray, + }; + + let stackFlags: Uint8Array | null = null; + + if (windowIdFromPrivateBrowsing.size > 0) { + // In this block, we'll remove everything related to frame table entries + // that have a innerWindowID with a isPrivateBrowsing flag, or that come + // from other tabs than the one we want to keep. + + // This map holds the information about the frame indexes that should be + // sanitized, with their functions as a key, so that we can easily change + // all frames if we need to. + const sanitizedFuncIndexesToFrameIndex: Map< + IndexIntoFuncTable, + IndexIntoFrameTable[] + > = new Map(); + // This set holds all func indexes that shouldn't be sanitized. This will be + // intersected with the previous map's keys to know which functions need to + // be split in 2. + const funcIndexesToBeKept = new Set(); + + const { frameTable, funcTable, resourceTable, stackTable } = newShared; + for (let frameIndex = 0; frameIndex < frameTable.length; frameIndex++) { + const innerWindowID = frameTable.innerWindowID[frameIndex]; + const funcIndex = frameTable.func[frameIndex]; + + const isPrivateBrowsing = + innerWindowID && windowIdFromPrivateBrowsing.has(innerWindowID); + if (isPrivateBrowsing) { + // The function pointed by this frame should be sanitized. + let sanitizedFrameIndexes = + sanitizedFuncIndexesToFrameIndex.get(funcIndex); + if (!sanitizedFrameIndexes) { + sanitizedFrameIndexes = []; + sanitizedFuncIndexesToFrameIndex.set( + funcIndex, + sanitizedFrameIndexes + ); + } + sanitizedFrameIndexes.push(frameIndex); + } else { + // The function pointed by this frame should be kept. + funcIndexesToBeKept.add(funcIndex); + } + } + + if (sanitizedFuncIndexesToFrameIndex.size) { + const resourcesToBeSanitized = new Set(); + + const newFuncTable = (newShared.funcTable = + shallowCloneFuncTable(funcTable)); + const newFrameTable = (newShared.frameTable = { + ...frameTable, + innerWindowID: frameTable.innerWindowID.slice(), + func: frameTable.func.slice(), + line: frameTable.line.slice(), + column: frameTable.column.slice(), + }); + + for (const [ + funcIndex, + frameIndexes, + ] of sanitizedFuncIndexesToFrameIndex.entries()) { + if (funcIndexesToBeKept.has(funcIndex)) { + // This function is used by both private and non-private data, therefore + // we split this function into 2 sanitized and unsanitized functions. + const sanitizedFuncIndex = newFuncTable.length; + const name = stringTable.indexForString( + `` + ); + newFuncTable.name.push(name); + newFuncTable.isJS.push(funcTable.isJS[funcIndex]); + newFuncTable.relevantForJS.push(funcTable.isJS[funcIndex]); + newFuncTable.resource.push(-1); + newFuncTable.source.push(null); + newFuncTable.lineNumber.push(null); + newFuncTable.columnNumber.push(null); + newFuncTable.length++; + + frameIndexes.forEach( + (frameIndex) => + (newFrameTable.func[frameIndex] = sanitizedFuncIndex) + ); + } else { + // This function is used only by private data, so we can change it + // directly. + const name = stringTable.indexForString(``); + newFuncTable.name[funcIndex] = name; + + newFuncTable.source[funcIndex] = null; + if (newFuncTable.resource[funcIndex] >= 0) { + resourcesToBeSanitized.add(newFuncTable.resource[funcIndex]); + } + newFuncTable.resource[funcIndex] = -1; + newFuncTable.lineNumber[funcIndex] = null; + newFuncTable.columnNumber[funcIndex] = null; + } + + // In both cases, nullify some information in all frames. + frameIndexes.forEach((frameIndex) => { + newFrameTable.line[frameIndex] = null; + newFrameTable.column[frameIndex] = null; + newFrameTable.innerWindowID[frameIndex] = null; + }); + } + + if (resourcesToBeSanitized.size) { + const newResourceTable = (newShared.resourceTable = { + ...resourceTable, + lib: resourceTable.lib.slice(), + name: resourceTable.name.slice(), + host: resourceTable.host.slice(), + }); + const remainingResources = new Set( + newFuncTable.resource + ); + for (const resourceIndex of resourcesToBeSanitized) { + if (!remainingResources.has(resourceIndex)) { + // This resource was used only by sanitized functions. Sanitize it + // as well. + const name = stringTable.indexForString( + `` + ); + newResourceTable.name[resourceIndex] = name; + newResourceTable.lib[resourceIndex] = null; + newResourceTable.host[resourceIndex] = null; + } + } + } + } + + // First we'll loop the stack table and populate a typed array with a value + // that is a flag that's inherited by children. This is possible because + // when iterating we visit parents before their children. + // There can be 3 values: + // - 0 is neutral, this means this stack isn't private browsing and isn't + // the tab id we want to keep. + // - 1 means that this stack comes from private browsing. + // They won't be set if the related sanitization isn't set in PIIToBeRemoved. + stackFlags = new Uint8Array(stackTable.length); + + for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) { + const prefix = stackTable.prefix[stackIndex]; + if (prefix !== null) { + // Inherit the prefix value + stackFlags[stackIndex] = stackFlags[prefix]; + if (stackFlags[stackIndex] === PRIVATE_BROWSING_STACK) { + // Because private browsing is the strongest value, we can skip + // the rest of the processing. + continue; + } + } + + const frameIndex = stackTable.frame[stackIndex]; + const innerWindowID = frameTable.innerWindowID[frameIndex]; + if (!innerWindowID) { + continue; + } + + const isPrivateBrowsing = windowIdFromPrivateBrowsing.has(innerWindowID); + if (isPrivateBrowsing) { + stackFlags[stackIndex] = PRIVATE_BROWSING_STACK; + } + } + } let removingCounters = false; const newProfile: Profile = { @@ -127,10 +299,7 @@ export function sanitizePII( : profile.meta.extensions, }, pages: pages, - shared: { - stringArray, - sources: profile.shared.sources, - }, + shared: newShared, threads: profile.threads.reduce((acc, thread, threadIndex) => { const newThread: RawThread | null = sanitizeThreadPII( thread, @@ -139,7 +308,8 @@ export function sanitizePII( threadIndex, PIIToBeRemoved, windowIdFromPrivateBrowsing, - markerSchemaByName + markerSchemaByName, + stackFlags ); // Filtering out the current thread if it's null. @@ -243,7 +413,8 @@ function sanitizeThreadPII( threadIndex: number, PIIToBeRemoved: RemoveProfileInformation, windowIdFromPrivateBrowsing: Set, - markerSchemaByName: MarkerSchemaByName + markerSchemaByName: MarkerSchemaByName, + stackFlags: Uint8Array | null ): RawThread | null { if (PIIToBeRemoved.shouldRemoveThreads.has(threadIndex)) { // If this is a hidden thread, remove the thread immediately. @@ -411,174 +582,14 @@ function sanitizeThreadPII( delete newThread['eTLD+1']; } - if (windowIdFromPrivateBrowsing.size > 0) { - // In this block, we'll remove everything related to frame table entries - // that have a innerWindowID with a isPrivateBrowsing flag. - - // This map holds the information about the frame indexes that should be - // sanitized, with their functions as a key, so that we can easily change - // all frames if we need to. - const sanitizedFuncIndexesToFrameIndex: Map< - IndexIntoFuncTable, - IndexIntoFrameTable[] - > = new Map(); - // This set holds all func indexes that shouldn't be sanitized. This will be - // intersected with the previous map's keys to know which functions need to - // be split in 2. - const funcIndexesToBeKept = new Set(); - - const { frameTable, funcTable, resourceTable, stackTable, samples } = - newThread; - for (let frameIndex = 0; frameIndex < frameTable.length; frameIndex++) { - const innerWindowID = frameTable.innerWindowID[frameIndex]; - const funcIndex = frameTable.func[frameIndex]; - - const isPrivateBrowsing = - innerWindowID && windowIdFromPrivateBrowsing.has(innerWindowID); - if (isPrivateBrowsing) { - // The function pointed by this frame should be sanitized. - let sanitizedFrameIndexes = - sanitizedFuncIndexesToFrameIndex.get(funcIndex); - if (!sanitizedFrameIndexes) { - sanitizedFrameIndexes = []; - sanitizedFuncIndexesToFrameIndex.set( - funcIndex, - sanitizedFrameIndexes - ); - } - sanitizedFrameIndexes.push(frameIndex); - } else { - // The function pointed by this frame should be kept. - funcIndexesToBeKept.add(funcIndex); - } - } - - if (sanitizedFuncIndexesToFrameIndex.size) { - const resourcesToBeSanitized = new Set(); - - const newFuncTable = (newThread.funcTable = - shallowCloneFuncTable(funcTable)); - const newFrameTable = (newThread.frameTable = { - ...frameTable, - innerWindowID: frameTable.innerWindowID.slice(), - func: frameTable.func.slice(), - line: frameTable.line.slice(), - column: frameTable.column.slice(), - }); - - for (const [ - funcIndex, - frameIndexes, - ] of sanitizedFuncIndexesToFrameIndex.entries()) { - if (funcIndexesToBeKept.has(funcIndex)) { - // This function is used by both private and non-private data, therefore - // we split this function into 2 sanitized and unsanitized functions. - const sanitizedFuncIndex = newFuncTable.length; - const name = stringTable.indexForString( - `` - ); - newFuncTable.name.push(name); - newFuncTable.isJS.push(funcTable.isJS[funcIndex]); - newFuncTable.relevantForJS.push(funcTable.isJS[funcIndex]); - newFuncTable.resource.push(-1); - newFuncTable.source.push(null); - newFuncTable.lineNumber.push(null); - newFuncTable.columnNumber.push(null); - newFuncTable.length++; - - frameIndexes.forEach( - (frameIndex) => - (newFrameTable.func[frameIndex] = sanitizedFuncIndex) - ); - } else { - // This function is used only by private data, so we can change it - // directly. - const name = stringTable.indexForString(``); - newFuncTable.name[funcIndex] = name; - - newFuncTable.source[funcIndex] = null; - if (newFuncTable.resource[funcIndex] >= 0) { - resourcesToBeSanitized.add(newFuncTable.resource[funcIndex]); - } - newFuncTable.resource[funcIndex] = -1; - newFuncTable.lineNumber[funcIndex] = null; - newFuncTable.columnNumber[funcIndex] = null; - } - - // In both cases, nullify some information in all frames. - frameIndexes.forEach((frameIndex) => { - newFrameTable.line[frameIndex] = null; - newFrameTable.column[frameIndex] = null; - newFrameTable.innerWindowID[frameIndex] = null; - }); - } - - if (resourcesToBeSanitized.size) { - const newResourceTable = (newThread.resourceTable = { - ...resourceTable, - lib: resourceTable.lib.slice(), - name: resourceTable.name.slice(), - host: resourceTable.host.slice(), - }); - const remainingResources = new Set(newFuncTable.resource); - for (const resourceIndex of resourcesToBeSanitized) { - if (!remainingResources.has(resourceIndex)) { - // This resource was used only by sanitized functions. Sanitize it - // as well. - const name = stringTable.indexForString( - `` - ); - newResourceTable.name[resourceIndex] = name; - newResourceTable.lib[resourceIndex] = null; - newResourceTable.host[resourceIndex] = null; - } - } - } - } - + const { samples } = newThread; + if (stackFlags !== null && windowIdFromPrivateBrowsing.size > 0) { // Now we'll remove samples related to the frames const newSamples = (newThread.samples = { ...samples, stack: samples.stack.slice(), }); - // First we'll loop the stack table and populate a typed array with a value - // that is a flag that's inherited by children. This is possible because - // when iterating we visit parents before their children. - // There can be 3 values: - // - 0 is neutral, this means this stack isn't private browsing and isn't - // the tab id we want to keep. - // - 1 means that this stack comes from private browsing. - // They won't be set if the related sanitization isn't set in PIIToBeRemoved. - const stackFlags = new Uint8Array(stackTable.length); - - // Some constants to make it easier to read. - const PRIVATE_BROWSING_STACK = 1; - - for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) { - const prefix = stackTable.prefix[stackIndex]; - if (prefix !== null) { - // Inherit the prefix value - stackFlags[stackIndex] = stackFlags[prefix]; - if (stackFlags[stackIndex] === PRIVATE_BROWSING_STACK) { - // Because private browsing is the strongest value, we can skip - // the rest of the processing. - continue; - } - } - - const frameIndex = stackTable.frame[stackIndex]; - const innerWindowID = frameTable.innerWindowID[frameIndex]; - if (!innerWindowID) { - continue; - } - - const isPrivateBrowsing = windowIdFromPrivateBrowsing.has(innerWindowID); - if (isPrivateBrowsing) { - stackFlags[stackIndex] = PRIVATE_BROWSING_STACK; - } - } - for (let sampleIndex = 0; sampleIndex < samples.length; sampleIndex++) { const stackIndex = samples.stack[sampleIndex]; if (stackIndex === null) { diff --git a/src/profile-logic/symbolication.ts b/src/profile-logic/symbolication.ts index 16188192f5..1c702ed916 100644 --- a/src/profile-logic/symbolication.ts +++ b/src/profile-logic/symbolication.ts @@ -14,7 +14,7 @@ import type { Profile, RawProfileSharedData, RawThread, - ThreadIndex, + RawStackTable, IndexIntoFuncTable, IndexIntoFrameTable, IndexIntoResourceTable, @@ -196,11 +196,10 @@ import { updateRawThreadStacks } from './profile-data'; type LibKey = string; // of the form ${debugName}/${breakpadId} export type SymbolicationStepCallback = ( - threadIndex: ThreadIndex, symbolicationStepInfo: SymbolicationStepInfo ) => void; -type ThreadLibSymbolicationInfo = { +type ProfileLibSymbolicationInfo = { // The resourceIndex for this lib in this thread. resourceIndex: IndexIntoResourceTable; // The libIndex for this lib in this thread. @@ -218,13 +217,13 @@ type ThreadLibSymbolicationInfo = { // This type exists because we symbolicate the profile in steps in order to // provide a profile to the user faster. This type represents a single step. export type SymbolicationStepInfo = { - threadLibSymbolicationInfo: ThreadLibSymbolicationInfo; + libSymbolicationInfo: ProfileLibSymbolicationInfo; resultsForLib: Map; }; export type FuncToFuncsMap = Map; -type ThreadSymbolicationInfo = Map; +type ProfileSymbolicationInfo = Map; /** * Like `new Map(iterableOfEntryPairs)`: Creates a map from an iterable of @@ -264,13 +263,13 @@ function makeConsensusMap( * allows the symbol substitation step at the end to work efficiently. * Returns a map with one entry for each library resource. */ -function getThreadSymbolicationInfo( - thread: RawThread, +function getSymbolicationInfo( + shared: RawProfileSharedData, libs: Lib[] -): ThreadSymbolicationInfo { - const { frameTable, funcTable, nativeSymbols, resourceTable } = thread; +): ProfileSymbolicationInfo { + const { frameTable, funcTable, nativeSymbols, resourceTable } = shared; - const map: ThreadSymbolicationInfo = new Map(); + const map = new Map(); for ( let resourceIndex = 0; resourceIndex < resourceTable.length; @@ -342,19 +341,17 @@ function getThreadSymbolicationInfo( // Go through all the threads to gather up the addresses we need to symbolicate // for each library. function buildLibSymbolicationRequestsForAllThreads( - symbolicationInfo: ThreadSymbolicationInfo[] + symbolicationInfo: ProfileSymbolicationInfo ): LibSymbolicationRequest[] { const libKeyToAddressesMap = new Map>(); - for (const threadSymbolicationInfo of symbolicationInfo) { - for (const [libKey, { frameAddresses }] of threadSymbolicationInfo) { - let addressSet = libKeyToAddressesMap.get(libKey); - if (addressSet === undefined) { - addressSet = new Set(); - libKeyToAddressesMap.set(libKey, addressSet); - } - for (const frameAddress of frameAddresses) { - addressSet.add(frameAddress); - } + for (const [libKey, { frameAddresses }] of symbolicationInfo) { + let addressSet = libKeyToAddressesMap.get(libKey); + if (addressSet === undefined) { + addressSet = new Set(); + libKeyToAddressesMap.set(libKey, addressSet); + } + for (const frameAddress of frameAddresses) { + addressSet.add(frameAddress); } } return Array.from(libKeyToAddressesMap).map(([libKey, addresses]) => { @@ -369,22 +366,17 @@ function buildLibSymbolicationRequestsForAllThreads( // ensure that the symbolication information eventually makes it into the thread. // This function leaves all the actual work to applySymbolicationSteps. function finishSymbolicationForLib( - profile: Profile, - symbolicationInfo: ThreadSymbolicationInfo[], + symbolicationInfo: ProfileSymbolicationInfo, resultsForLib: Map, libKey: string, symbolicationStepCallback: SymbolicationStepCallback ): void { - const { threads } = profile; - for (let threadIndex = 0; threadIndex < threads.length; threadIndex++) { - const threadSymbolicationInfo = symbolicationInfo[threadIndex]; - const threadLibSymbolicationInfo = threadSymbolicationInfo.get(libKey); - if (threadLibSymbolicationInfo === undefined) { - continue; - } - const symbolicationStep = { threadLibSymbolicationInfo, resultsForLib }; - symbolicationStepCallback(threadIndex, symbolicationStep); + const libSymbolicationInfo = symbolicationInfo.get(libKey); + if (libSymbolicationInfo === undefined) { + return; } + const symbolicationStep = { libSymbolicationInfo, resultsForLib }; + symbolicationStepCallback(symbolicationStep); } // Create a new stack table where all stack nodes with frames in @@ -417,18 +409,17 @@ function finishSymbolicationForLib( // - stack E with frame 4 // - stack E' with frame 8 // - stack F with frame 5 -function _computeThreadWithAddedExpansionStacks( - thread: RawThread, +function _computeStackTableWithAddedExpansionStacks( + stackTable: RawStackTable, shouldStacksWithThisOldFrameBeRemoved: Uint8Array, frameIndexToInlineExpansionFrames: Map< IndexIntoFrameTable, IndexIntoFrameTable[] > -): RawThread { +): { newStackTable: RawStackTable; oldStackToNewStack: Int32Array } | null { if (frameIndexToInlineExpansionFrames.size === 0) { - return thread; + return null; } - const { stackTable } = thread; const newStackTable = getEmptyRawStackTable(); const oldStackToNewStack = new Int32Array(stackTable.length); for (let stack = 0; stack < stackTable.length; stack++) { @@ -461,13 +452,7 @@ function _computeThreadWithAddedExpansionStacks( } oldStackToNewStack[stack] = prefix ?? -1; } - return updateRawThreadStacks(thread, newStackTable, (oldStack) => { - if (oldStack === null) { - return null; - } - const newStack = oldStackToNewStack[oldStack]; - return newStack !== -1 ? newStack : null; - }); + return { newStackTable, oldStackToNewStack }; } /** @@ -475,21 +460,24 @@ function _computeThreadWithAddedExpansionStacks( * symbolicationSteps is used to create a new thread with the new symbols. */ export function applySymbolicationSteps( - oldThread: RawThread, - shared: RawProfileSharedData, + oldThreads: RawThread[], + oldShared: RawProfileSharedData, symbolicationSteps: SymbolicationStepInfo[] -): { thread: RawThread; oldFuncToNewFuncsMap: FuncToFuncsMap } { +): { + threads: RawThread[]; + shared: RawProfileSharedData; + oldFuncToNewFuncsMap: FuncToFuncsMap; +} { const oldFuncToNewFuncsMap: FuncToFuncsMap = new Map(); - const frameCount = oldThread.frameTable.length; + const frameCount = oldShared.frameTable.length; const shouldStacksWithThisFrameBeRemoved = new Uint8Array(frameCount); const frameIndexToInlineExpansionFrames = new Map< IndexIntoFrameTable, IndexIntoFrameTable[] >(); - let thread = oldThread; + let shared = oldShared; for (const symbolicationStep of symbolicationSteps) { - thread = _partiallyApplySymbolicationStep( - thread, + shared = _partiallyApplySymbolicationStep( shared, symbolicationStep, oldFuncToNewFuncsMap, @@ -497,13 +485,31 @@ export function applySymbolicationSteps( frameIndexToInlineExpansionFrames ); } - thread = _computeThreadWithAddedExpansionStacks( - thread, + const newStackInfo = _computeStackTableWithAddedExpansionStacks( + shared.stackTable, shouldStacksWithThisFrameBeRemoved, frameIndexToInlineExpansionFrames ); - return { thread, oldFuncToNewFuncsMap }; + if (newStackInfo === null) { + return { threads: oldThreads, shared, oldFuncToNewFuncsMap }; + } + + const { newStackTable, oldStackToNewStack } = newStackInfo; + shared = { + ...shared, + stackTable: newStackTable, + }; + + const threads = updateRawThreadStacks(oldThreads, (oldStack) => { + if (oldStack === null) { + return null; + } + const newStack = oldStackToNewStack[oldStack]; + return newStack !== -1 ? newStack : null; + }); + + return { threads, shared, oldFuncToNewFuncsMap }; } /** @@ -532,7 +538,6 @@ export function applySymbolicationSteps( * steps from multiple libraries have been processed. This can be much faster. */ function _partiallyApplySymbolicationStep( - thread: RawThread, shared: RawProfileSharedData, symbolicationStepInfo: SymbolicationStepInfo, oldFuncToNewFuncsMap: FuncToFuncsMap, @@ -541,22 +546,23 @@ function _partiallyApplySymbolicationStep( IndexIntoFrameTable, IndexIntoFrameTable[] > -): RawThread { - const { stringArray, sources } = shared; +): RawProfileSharedData { const { frameTable: oldFrameTable, funcTable: oldFuncTable, nativeSymbols: oldNativeSymbols, - } = thread; + stringArray, + sources, + } = shared; const stringTable = StringTable.withBackingArray(stringArray); - const { threadLibSymbolicationInfo, resultsForLib } = symbolicationStepInfo; + const { libSymbolicationInfo, resultsForLib } = symbolicationStepInfo; const { resourceIndex, allFramesForThisLib, allFuncsForThisLib, allNativeSymbolsForThisLib, libIndex, - } = threadLibSymbolicationInfo; + } = libSymbolicationInfo; const availableFuncs: Set = new Set(allFuncsForThisLib); const availableNativeSymbols: Set = new Set( @@ -874,8 +880,8 @@ function _partiallyApplySymbolicationStep( ); } - const newThread = { - ...thread, + const newShared = { + ...shared, frameTable, funcTable, nativeSymbols, @@ -883,7 +889,7 @@ function _partiallyApplySymbolicationStep( // We have the finished new frameTable and new funcTable. // The new stackTable will be built by the caller. - return newThread; + return newShared; } /** @@ -898,9 +904,7 @@ export async function symbolicateProfile( symbolicationStepCallback: SymbolicationStepCallback, ignoreCache?: boolean ): Promise { - const symbolicationInfo = profile.threads.map((thread) => - getThreadSymbolicationInfo(thread, profile.libs) - ); + const symbolicationInfo = getSymbolicationInfo(profile.shared, profile.libs); const libSymbolicationRequests = buildLibSymbolicationRequestsForAllThreads(symbolicationInfo); await symbolStore.getSymbols( @@ -909,7 +913,6 @@ export async function symbolicateProfile( const { debugName, breakpadId } = lib; const libKey = `${debugName}/${breakpadId}`; finishSymbolicationForLib( - profile, symbolicationInfo, results, libKey, diff --git a/src/profile-logic/tracks.ts b/src/profile-logic/tracks.ts index d774501c07..b1a1fe5d8f 100644 --- a/src/profile-logic/tracks.ts +++ b/src/profile-logic/tracks.ts @@ -1358,8 +1358,8 @@ function _isFirefoxMediaThreadWhichIsUsuallyIdle(thread: RawThread): boolean { // If the profile has no cpu delta units, the return value is based on the // number of non-idle samples. function _computeThreadSampleScore( - { meta }: Profile, - { samples, stackTable, frameTable }: RawThread, + { meta, shared: { stackTable, frameTable } }: Profile, + { samples }: RawThread, referenceCPUDeltaPerMs: number ): number { if (meta.sampleUnits && samples.threadCPUDelta) { diff --git a/src/reducers/profile-view.ts b/src/reducers/profile-view.ts index 26465510fd..4cda59ab4c 100644 --- a/src/reducers/profile-view.ts +++ b/src/reducers/profile-view.ts @@ -175,26 +175,8 @@ const viewOptionsPerThread: Reducer = ( // The view options are lazily initialized. Reset to the default values. return {}; case 'BULK_SYMBOLICATION': { - const { oldFuncToNewFuncsMaps } = action; - // For each thread, apply oldFuncToNewFuncsMap to that thread's - // selectedCallNodePath and expandedCallNodePaths. - const newState = objectMap(state, (threadViewOptions, threadsKey) => { - // Multiple selected threads are not supported, note that transforming - // the threadKey with multiple threads into a number will result in a NaN. - // This should be fine here, as the oldFuncToNewFuncsMaps only supports - // single thread indexes. - const threadIndex = +threadsKey; - if (Number.isNaN(threadIndex)) { - throw new Error( - 'Bulk symbolication only supports a single thread, and a ThreadsKey with ' + - 'multiple threads was used.' - ); - } - const oldFuncToNewFuncsMap = oldFuncToNewFuncsMaps.get(threadIndex); - if (oldFuncToNewFuncsMap === undefined) { - return threadViewOptions; - } - + const { oldFuncToNewFuncsMap } = action; + const newState = objectMap(state, (threadViewOptions) => { return { ...threadViewOptions, selectedNonInvertedCallNodePath: applyFuncSubstitutionToCallPath( @@ -718,17 +700,10 @@ const rightClickedCallNode: Reducer = ( return null; } - const { oldFuncToNewFuncsMaps } = action; - // This doesn't support a ThreadsKey with multiple threads. - const oldFuncToNewFuncsMap = oldFuncToNewFuncsMaps.get(+state.threadsKey); - if (oldFuncToNewFuncsMap === undefined) { - return state; - } - return { ...state, callNodePath: applyFuncSubstitutionToCallPath( - oldFuncToNewFuncsMap, + action.oldFuncToNewFuncsMap, state.callNodePath ), }; diff --git a/src/selectors/per-thread/composed.ts b/src/selectors/per-thread/composed.ts index 4816f5f56a..c14d3de935 100644 --- a/src/selectors/per-thread/composed.ts +++ b/src/selectors/per-thread/composed.ts @@ -86,7 +86,7 @@ export function getComposedSelectorsPerThread( const { samples, jsAllocations, nativeAllocations } = thread; const hasSamples = [samples, jsAllocations, nativeAllocations].some( - (table) => hasUsefulSamples(table?.stack, thread, shared) + (table) => hasUsefulSamples(table?.stack, shared) ); if (!hasSamples) { visibleTabs = visibleTabs.filter( diff --git a/src/selectors/per-thread/thread.tsx b/src/selectors/per-thread/thread.tsx index fc8b619b89..6f8c041917 100644 --- a/src/selectors/per-thread/thread.tsx +++ b/src/selectors/per-thread/thread.tsx @@ -26,7 +26,6 @@ import type { JsTracerTable, RawSamplesTable, SamplesTable, - StackTable, NativeAllocationsTable, JsAllocationsTable, SamplesLikeTable, @@ -112,12 +111,6 @@ export function getBasicThreadSelectorsPerThread( getRawThread(state), ProfileSelectors.getProfileInterval(state) ); - const getStackTable: Selector = createSelector( - (state: State) => getRawThread(state).stackTable, - (state: State) => getRawThread(state).frameTable, - ProfileSelectors.getDefaultCategory, - ProfileData.computeStackTableFromRawStackTable - ); /** * This selector gets the weight type from the thread.samples table, but @@ -144,7 +137,14 @@ export function getBasicThreadSelectorsPerThread( const getThread: Selector = createSelector( getRawThread, getSamplesTable, - getStackTable, + ProfileSelectors.getStackTable, + (state: State) => + ProfileSelectors.getRawProfileSharedData(state).frameTable, + (state: State) => ProfileSelectors.getRawProfileSharedData(state).funcTable, + (state: State) => + ProfileSelectors.getRawProfileSharedData(state).nativeSymbols, + (state: State) => + ProfileSelectors.getRawProfileSharedData(state).resourceTable, ProfileSelectors.getStringTable, ProfileSelectors.getSourceTable, ProfileData.createThreadFromDerivedTables @@ -286,26 +286,22 @@ export function getBasicThreadSelectorsPerThread( const getHasUsefulTimingSamples: Selector = createSelector( getSamplesTable, - getRawThread, ProfileSelectors.getRawProfileSharedData, - (samples, rawThread, shared) => - ProfileData.hasUsefulSamples(samples.stack, rawThread, shared) + (samples, shared) => ProfileData.hasUsefulSamples(samples.stack, shared) ); const getHasUsefulJsAllocations: Selector = createSelector( getJsAllocations, - getRawThread, ProfileSelectors.getRawProfileSharedData, - (jsAllocations, rawThread, shared) => - ProfileData.hasUsefulSamples(jsAllocations?.stack, rawThread, shared) + (jsAllocations, shared) => + ProfileData.hasUsefulSamples(jsAllocations?.stack, shared) ); const getHasUsefulNativeAllocations: Selector = createSelector( getNativeAllocations, - getRawThread, ProfileSelectors.getRawProfileSharedData, - (nativeAllocations, rawThread, shared) => - ProfileData.hasUsefulSamples(nativeAllocations?.stack, rawThread, shared) + (nativeAllocations, shared) => + ProfileData.hasUsefulSamples(nativeAllocations?.stack, shared) ); /** @@ -337,18 +333,11 @@ export function getBasicThreadSelectorsPerThread( const getExpensiveJsTracerTiming: Selector = createSelector( getJsTracerTable, - getRawThread, - ProfileSelectors.getStringTable, - ProfileSelectors.getSourceTable, - (jsTracerTable, thread, stringTable, sources) => + ProfileSelectors.getRawProfileSharedData, + (jsTracerTable, shared) => jsTracerTable === null ? null - : JsTracer.getJsTracerTiming( - jsTracerTable, - thread, - stringTable, - sources - ) + : JsTracer.getJsTracerTiming(jsTracerTable, shared) ); /** diff --git a/src/selectors/profile.ts b/src/selectors/profile.ts index d761943a3f..50ae8a10c6 100644 --- a/src/selectors/profile.ts +++ b/src/selectors/profile.ts @@ -14,6 +14,7 @@ import { processCounter, getInclusiveSampleIndexRangeForSelection, computeTabToThreadIndexesMap, + computeStackTableFromRawStackTable, } from '../profile-logic/profile-data'; import type { IPCMarkerCorrelations } from '../profile-logic/marker-data'; import { correlateIPCMarkers } from '../profile-logic/marker-data'; @@ -27,6 +28,7 @@ import type { TabSlug } from '../app-logic/tabs-handling'; import type { Profile, RawProfileSharedData, + StackTable, CategoryList, IndexIntoCategoryList, RawThread, @@ -248,6 +250,13 @@ export const getStringTable: Selector = createSelector( (stringArray) => StringTable.withBackingArray(stringArray as string[]) ); +export const getStackTable: Selector = createSelector( + (state: State) => getRawProfileSharedData(state).stackTable, + (state: State) => getRawProfileSharedData(state).frameTable, + getDefaultCategory, + computeStackTableFromRawStackTable +); + export const getSourceTable: Selector = (state: State) => getRawProfileSharedData(state).sources; diff --git a/src/symbolicator-cli/index.ts b/src/symbolicator-cli/index.ts index 2e917c7a52..ad4d8e26d1 100644 --- a/src/symbolicator-cli/index.ts +++ b/src/symbolicator-cli/index.ts @@ -26,7 +26,6 @@ import type { SymbolicationStepInfo } from '../profile-logic/symbolication'; import type { SymbolTableAsTuple } from '../profile-logic/symbol-store-db'; import * as MozillaSymbolicationAPI from '../profile-logic/mozilla-symbolication-api'; import { SymbolsNotFoundError } from '../profile-logic/errors'; -import type { ThreadIndex } from '../types'; /** * Simple 'in-memory' symbol DB that conforms to the same interface as SymbolStoreDB but @@ -133,39 +132,24 @@ export async function run(options: CliOptions) { console.log('Symbolicating...'); - const symbolicationStepsPerThread: Map = - new Map(); + const symbolicationSteps: SymbolicationStepInfo[] = []; await symbolicateProfile( profile, symbolStore, - ( - threadIndex: ThreadIndex, - symbolicationStepInfo: SymbolicationStepInfo - ) => { - let threadSteps = symbolicationStepsPerThread.get(threadIndex); - if (threadSteps === undefined) { - threadSteps = []; - symbolicationStepsPerThread.set(threadIndex, threadSteps); - } - threadSteps.push(symbolicationStepInfo); + (symbolicationStepInfo: SymbolicationStepInfo) => { + symbolicationSteps.push(symbolicationStepInfo); } ); console.log('Applying collected symbolication steps...'); - profile.threads = profile.threads.map((oldThread, threadIndex) => { - const symbolicationSteps = symbolicationStepsPerThread.get(threadIndex); - if (symbolicationSteps === undefined) { - return oldThread; - } - const { thread } = applySymbolicationSteps( - oldThread, - profile.shared, - symbolicationSteps - ); - return thread; - }); - + const { shared, threads } = applySymbolicationSteps( + profile.threads, + profile.shared, + symbolicationSteps + ); + profile.shared = shared; + profile.threads = threads; profile.meta.symbolicated = true; console.log(`Saving profile to ${options.output}`); diff --git a/src/test/components/CallNodeContextMenu.test.tsx b/src/test/components/CallNodeContextMenu.test.tsx index c6c159c0fe..0e032e73cf 100644 --- a/src/test/components/CallNodeContextMenu.test.tsx +++ b/src/test/components/CallNodeContextMenu.test.tsx @@ -62,7 +62,6 @@ describe('calltree/CallNodeContextMenu', function () { A.js B.js `); - const [thread] = profile.threads; const fileNameIndex = stringTable.indexForString( 'https://example.com/script.js' ); @@ -70,15 +69,17 @@ describe('calltree/CallNodeContextMenu', function () { // Create a source entry const sourceIndex = addSourceToTable(profile.shared.sources, fileNameIndex); + const { funcTable } = profile.shared; + const funcIndexA = funcNames.indexOf('A.js'); - thread.funcTable.source[funcIndexA] = sourceIndex; - thread.funcTable.lineNumber[funcIndexA] = 1; - thread.funcTable.columnNumber[funcIndexA] = 111; + funcTable.source[funcIndexA] = sourceIndex; + funcTable.lineNumber[funcIndexA] = 1; + funcTable.columnNumber[funcIndexA] = 111; const funcIndexB = funcNames.indexOf('B.js'); - thread.funcTable.source[funcIndexB] = sourceIndex; - thread.funcTable.lineNumber[funcIndexB] = 2; - thread.funcTable.columnNumber[funcIndexB] = 222; + funcTable.source[funcIndexB] = sourceIndex; + funcTable.lineNumber[funcIndexB] = 2; + funcTable.columnNumber[funcIndexB] = 222; const store = storeWithProfile(profile); store.dispatch(changeRightClickedCallNode(0, [funcIndexA, funcIndexB])); diff --git a/src/test/components/FlameGraph.test.tsx b/src/test/components/FlameGraph.test.tsx index 88bb561d8f..675d692d78 100644 --- a/src/test/components/FlameGraph.test.tsx +++ b/src/test/components/FlameGraph.test.tsx @@ -291,7 +291,7 @@ function setupFlameGraph() { // Add some file and line number to the profile so that tooltips generate // an interesting snapshot. - const { funcTable } = profile.threads[0]; + const { funcTable } = profile.shared; // Create source entries. const defaultFileIndex = stringTable.indexForString('path/to/file'); diff --git a/src/test/components/SampleGraph.test.tsx b/src/test/components/SampleGraph.test.tsx index dd597212eb..90e2c1aba2 100644 --- a/src/test/components/SampleGraph.test.tsx +++ b/src/test/components/SampleGraph.test.tsx @@ -96,7 +96,7 @@ describe('SampleGraph', function () { `Couldn't find the sample graph canvas, with selector .threadSampleGraphCanvas` ) as HTMLElement; const thread = profile.threads[0]; - const { stringArray } = profile.shared; + const { stringArray, funcTable } = profile.shared; // Perform a click on the sample graph. function clickSampleGraph(index: IndexIntoSamplesTable) { @@ -123,7 +123,7 @@ describe('SampleGraph', function () { function getCallNodePath() { return selectedThreadSelectors .getSelectedCallNodePath(getState()) - .map((funcIndex) => stringArray[thread.funcTable.name[funcIndex]]); + .map((funcIndex) => stringArray[funcTable.name[funcIndex]]); } /** diff --git a/src/test/components/TabSelectorMenu.test.tsx b/src/test/components/TabSelectorMenu.test.tsx index b8689d3b41..fe2e5bcc82 100644 --- a/src/test/components/TabSelectorMenu.test.tsx +++ b/src/test/components/TabSelectorMenu.test.tsx @@ -37,16 +37,12 @@ describe('app/TabSelectorMenu', () => { // // Thread 0 will be present in firstTabTabID. // Thread 1 be present in secondTabTabID. - profile.threads[0].frameTable.innerWindowID[0] = - extraPageData.parentInnerWindowIDsWithChildren; - profile.threads[0].frameTable.length++; - + profile.threads[0].usedInnerWindowIDs = [extraPageData.parentInnerWindowIDsWithChildren]; + // TODO: refer to these innerWindowIDs from the frames // Add a threadCPUDelta value for thread activity score. profile.threads[0].samples.threadCPUDelta = [1]; - profile.threads[1].frameTable.innerWindowID[0] = - extraPageData.secondTabInnerWindowIDs[0]; - profile.threads[1].frameTable.length++; + profile.threads[1].usedInnerWindowIDs = [extraPageData.secondTabInnerWindowIDs[0]]; // Add a threadCPUDelta value for thread activity score. This thread // should stay above the first thread. profile.threads[0].samples.threadCPUDelta = [2]; @@ -197,12 +193,8 @@ describe('app/TabSelectorMenu', () => { })); // Attach innerWindowIDs to the samples. - profile.threads[0].frameTable.innerWindowID[0] = - extraPageData.parentInnerWindowIDsWithChildren; - profile.threads[0].frameTable.length++; - profile.threads[0].frameTable.innerWindowID[1] = - extraPageData.secondTabInnerWindowIDs[0]; - profile.threads[0].frameTable.length++; + profile.threads[0].usedInnerWindowIDs = [extraPageData.parentInnerWindowIDsWithChildren, extraPageData.secondTabInnerWindowIDs[0]]; + // TODO: refer to these innerWindowIDs from the frames const store = storeWithProfile(profile); render( diff --git a/src/test/components/ThreadActivityGraph.test.tsx b/src/test/components/ThreadActivityGraph.test.tsx index 48ceb4a170..b879361d3e 100644 --- a/src/test/components/ThreadActivityGraph.test.tsx +++ b/src/test/components/ThreadActivityGraph.test.tsx @@ -96,7 +96,7 @@ describe('ThreadActivityGraph', function () { `Couldn't find the activity graph canvas, with selector .threadActivityGraphCanvas` ) as HTMLElement; const thread = profile.threads[0]; - const { stringArray } = profile.shared; + const { funcTable, stringArray } = profile.shared; // Perform a click on the activity graph. function clickActivityGraph( @@ -113,7 +113,7 @@ describe('ThreadActivityGraph', function () { function getCallNodePath() { return selectedThreadSelectors .getSelectedCallNodePath(getState()) - .map((funcIndex) => stringArray[thread.funcTable.name[funcIndex]]); + .map((funcIndex) => stringArray[funcTable.name[funcIndex]]); } /** diff --git a/src/test/components/TooltipCallnode.test.tsx b/src/test/components/TooltipCallnode.test.tsx index ee2853bbbc..910774ec4a 100644 --- a/src/test/components/TooltipCallnode.test.tsx +++ b/src/test/components/TooltipCallnode.test.tsx @@ -115,8 +115,11 @@ describe('TooltipCallNode', function () { }); } - const { frameTable } = profile.threads[threadIndex]; + const { frameTable } = profile.shared; + // TODO: This loop is now iterating over frames used across all threads, + // whereas in the past it was only touching frames used in thread 1. + // Figure out what to do here. for (let i = 1; i < frameTable.length; i++) { frameTable.innerWindowID[i] = profile.pages[profile.pages.length - 1].innerWindowID; diff --git a/src/test/components/TrackThread.test.tsx b/src/test/components/TrackThread.test.tsx index 8d1f054a55..4e228376b8 100644 --- a/src/test/components/TrackThread.test.tsx +++ b/src/test/components/TrackThread.test.tsx @@ -177,7 +177,7 @@ describe('timeline/TrackThread', function () { .getSelectedCallNodePath(getState()) .map( (funcIndex) => - profile.shared.stringArray[thread.funcTable.name[funcIndex]] + profile.shared.stringArray[profile.shared.funcTable.name[funcIndex]] ); fireFullClick(stackGraphCanvas(), getFillRectCenterByIndex(log, 0)); diff --git a/src/test/fixtures/utils.ts b/src/test/fixtures/utils.ts index 475cf7a3c4..eed1c511ed 100644 --- a/src/test/fixtures/utils.ts +++ b/src/test/fixtures/utils.ts @@ -144,8 +144,8 @@ export function computeThreadFromRawThread( ): Thread { const stringTable = StringTable.withBackingArray(shared.stringArray); const stackTable = computeStackTableFromRawStackTable( - rawThread.stackTable, - rawThread.frameTable, + shared.stackTable, + shared.frameTable, defaultCategory ); const samples = computeSamplesTableFromRawSamplesTable( @@ -157,6 +157,10 @@ export function computeThreadFromRawThread( rawThread, samples, stackTable, + shared.frameTable, + shared.funcTable, + shared.nativeSymbols, + shared.resourceTable, stringTable, shared.sources ); diff --git a/src/types/actions.ts b/src/types/actions.ts index 940b018500..9fa9bcb62a 100644 --- a/src/types/actions.ts +++ b/src/types/actions.ts @@ -5,6 +5,7 @@ import type JSZip from 'jszip'; import type { Profile, + RawProfileSharedData, RawThread, ThreadIndex, Pid, @@ -360,7 +361,8 @@ type ReceiveProfileAction = | { readonly type: 'BULK_SYMBOLICATION'; readonly symbolicatedThreads: RawThread[]; - readonly oldFuncToNewFuncsMaps: Map; + readonly symbolicatedShared: RawProfileSharedData; + readonly oldFuncToNewFuncsMap: FuncToFuncsMap; } | { readonly type: 'DONE_SYMBOLICATING'; diff --git a/src/types/profile.ts b/src/types/profile.ts index 3e5e02466e..2dacf335cf 100644 --- a/src/types/profile.ts +++ b/src/types/profile.ts @@ -654,11 +654,6 @@ export type RawThread = { jsAllocations?: JsAllocationsTable; nativeAllocations?: NativeAllocationsTable; markers: RawMarkerTable; - stackTable: RawStackTable; - frameTable: FrameTable; - funcTable: FuncTable; - resourceTable: ResourceTable; - nativeSymbols: NativeSymbolTable; jsTracer?: JsTracerTable; // If present and true, this thread was launched for a private browsing session only. // When false, it can still contain private browsing data if the profile was @@ -670,6 +665,7 @@ export type RawThread = { // It's absent in Firefox 97 and before, or in Firefox 98+ when this thread // had no extra attribute at all. userContextId?: number; + usedInnerWindowIDs?: InnerWindowID[]; }; export type ExtensionTable = { @@ -941,6 +937,11 @@ export type SourceTable = { }; export type RawProfileSharedData = { + stackTable: RawStackTable; + frameTable: FrameTable; + funcTable: FuncTable; + resourceTable: ResourceTable; + nativeSymbols: NativeSymbolTable; // Strings for profiles are collected into a single table, and are referred to by // their index by other tables. stringArray: string[];