Make stackTable (sub)category derived data.

Stop storing them in the file, compute them at runtime.
This reduces profile sizes.

Author: mstange

src/app-logic/constants.js

@@ -14,7 +14,7 @@ export const GECKO_PROFILE_VERSION = 31;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 51;
+export const PROCESSED_PROFILE_VERSION = 52;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/profile-logic/data-structures.js

@@ -13,6 +13,7 @@ import type {
   RawSamplesTable,
   SamplesTable,
   FrameTable,
+  RawStackTable,
   StackTable,
   FuncTable,
   RawMarkerTable,
@@ -46,6 +47,18 @@ export function getEmptyStackTable(): StackTable {
   };
 }
 
+export function getEmptyRawStackTable(): RawStackTable {
+  return {
+    // Important!
+    // If modifying this structure, please update all callers of this function to ensure
+    // that they are pushing on correctly to the data structure. These pushes may not
+    // be caught by the type system.
+    frame: [],
+    prefix: [],
+    length: 0,
+  };
+}
+
 /**
  * Returns an empty samples table with eventDelay field instead of responsiveness.
  * eventDelay is a new field and it replaced responsiveness. We should still
@@ -370,7 +383,7 @@ export function getEmptyThread(overrides?: $Shape<RawThread>): RawThread {
     // Creating samples with event delay since it's the new samples table.
     samples: getEmptySamplesTableWithEventDelay(),
     markers: getEmptyRawMarkerTable(),
-    stackTable: getEmptyStackTable(),
+    stackTable: getEmptyRawStackTable(),
     frameTable: getEmptyFrameTable(),
     stringArray: [],
     funcTable: getEmptyFuncTable(),

src/profile-logic/import/chrome.js

@@ -4,8 +4,8 @@
 // @flow
 import type {
   Profile,
-  StackTable,
   RawThread,
+  RawStackTable,
   IndexIntoFuncTable,
   IndexIntoStackTable,
   IndexIntoResourceTable,
@@ -688,8 +688,6 @@ async function processTracingEvents(
           frameTable.length = Math.max(frameTable.length, frameIndex + 1);
 
           stackTable.frame.push(frameIndex);
-          stackTable.category.push(category);
-          stackTable.subcategory.push(0);
           stackTable.prefix.push(prefixStackIndex);
           nodeIdToStackId.set(nodeIndex, stackTable.length++);
         }
@@ -909,7 +907,7 @@ function getImageSize(
  * For sanity, check that stacks are ordered where the prefix stack
  * always preceeds the current stack index in the StackTable.
  */
-function assertStackOrdering(stackTable: StackTable) {
+function assertStackOrdering(stackTable: RawStackTable) {
   const visitedStacks = new Set([null]);
   for (let i = 0; i < stackTable.length; i++) {
     if (!visitedStacks.has(stackTable.prefix[i])) {

src/profile-logic/import/dhat.js

@@ -215,8 +215,6 @@ export function attemptToConvertDhat(json: mixed): Profile | null {
   const rootFrameIndex = frameTable.length++;
 
   stackTable.frame.push(rootFrameIndex);
-  stackTable.category.push(otherCategory);
-  stackTable.subcategory.push(otherSubCategory);
   stackTable.prefix.push(null);
   const rootStackIndex = stackTable.length++;
 
@@ -333,8 +331,6 @@ export function attemptToConvertDhat(json: mixed): Profile | null {
       if (stackIndex === stackTable.length) {
         // No stack index was found, add on a new one.
         stackTable.frame.push(frameIndex);
-        stackTable.category.push(otherCategory);
-        stackTable.subcategory.push(otherSubCategory);
         stackTable.prefix.push(prefix);
 
         if (candidateStackTables) {
@@ -399,8 +395,6 @@ export function attemptToConvertDhat(json: mixed): Profile | null {
     thread.frameTable.length = frameTable.length;
 
     thread.stackTable.frame = stackTable.frame.slice();
-    thread.stackTable.category = stackTable.category.slice();
-    thread.stackTable.category = stackTable.category.slice();
     thread.stackTable.prefix = stackTable.prefix.slice();
     thread.stackTable.length = stackTable.length;
 

src/profile-logic/js-tracer.js

@@ -5,7 +5,7 @@
 
 import {
   getEmptyFrameTable,
-  getEmptyStackTable,
+  getEmptyRawStackTable,
   getEmptySamplesTableWithEventDelay,
   getEmptyRawMarkerTable,
 } from './data-structures';
@@ -508,7 +508,7 @@ export function convertJsTracerToThreadWithoutSamples(
   // Create a new thread, with empty information, but preserve some of the existing
   // thread information.
   const frameTable = getEmptyFrameTable();
-  const stackTable = getEmptyStackTable();
+  const stackTable = getEmptyRawStackTable();
   const samples: RawSamplesTable = {
     ...getEmptySamplesTableWithEventDelay(),
     weight: [],
@@ -652,7 +652,6 @@ export function convertJsTracerToThreadWithoutSamples(
     // Each event gets a stack table entry.
     const stackIndex = stackTable.length++;
     stackTable.frame.push(frameIndex);
-    stackTable.category.push(otherCategory);
     stackTable.prefix.push(prefixIndex);
     stackMap.set(tracerEventIndex, stackIndex);
 

src/profile-logic/merge-compare.js

@@ -20,7 +20,7 @@ import {
   getEmptyNativeSymbolTable,
   getEmptyFrameTable,
   getEmptyFuncTable,
-  getEmptyStackTable,
+  getEmptyRawStackTable,
   getEmptyRawMarkerTable,
   getEmptySamplesTableWithEventDelay,
 } from './data-structures';
@@ -55,8 +55,8 @@ import type {
   Lib,
   NativeSymbolTable,
   ResourceTable,
-  StackTable,
   RawSamplesTable,
+  RawStackTable,
   UrlState,
   ImplementationFilter,
   TransformStacksPerThread,
@@ -162,13 +162,6 @@ export function mergeProfilesForDiffing(
 
     // We adjust the categories using the maps computed above.
     // TODO issue #2151: Also adjust subcategories.
-    thread.stackTable = {
-      ...thread.stackTable,
-      category: adjustCategories(
-        thread.stackTable.category,
-        translationMapsForCategories[i]
-      ),
-    };
     thread.frameTable = {
       ...thread.frameTable,
       category: adjustNullableCategories(
@@ -396,27 +389,6 @@ function mergeCategories(categoriesPerThread: Array<CategoryList | void>): {|
   return { categories: newCategories, translationMaps };
 }
 
-/**
- * Adjusts the category indices in a category list using a translation map.
- */
-function adjustCategories(
-  categories: $ReadOnlyArray<IndexIntoCategoryList>,
-  translationMap: TranslationMapForCategories
-): Array<IndexIntoCategoryList> {
-  return categories.map((category) => {
-    const result = translationMap.get(category);
-    if (result === undefined) {
-      throw new Error(
-        stripIndent`
-          Category with index ${category} hasn't been found in the translation map.
-          This shouldn't happen and indicates a bug in the profiler's code.
-        `
-      );
-    }
-    return result;
-  });
-}
-
 /**
  * Adjusts the category indices in a category list using a translation map.
  */
@@ -790,9 +762,9 @@ function combineFrameTables(
 function combineStackTables(
   translationMapsForFrames: TranslationMapForFrames[],
   threads: $ReadOnlyArray<RawThread>
-): { stackTable: StackTable, translationMaps: TranslationMapForStacks[] } {
+): { stackTable: RawStackTable, translationMaps: TranslationMapForStacks[] } {
   const translationMaps = [];
-  const newStackTable = getEmptyStackTable();
+  const newStackTable = getEmptyRawStackTable();
 
   threads.forEach((thread, threadIndex) => {
     const { stackTable } = thread;
@@ -818,8 +790,6 @@ function combineStackTables(
       }
 
       newStackTable.frame.push(newFrameIndex);
-      newStackTable.category.push(stackTable.category[i]);
-      newStackTable.subcategory.push(stackTable.subcategory[i]);
       newStackTable.prefix.push(newPrefix);
 
       translationMap.set(i, newStackTable.length);

src/profile-logic/process-profile.js

@@ -43,10 +43,9 @@ import type {
   RawThread,
   RawCounter,
   ExtensionTable,
-  CategoryList,
   FrameTable,
   RawSamplesTable,
-  StackTable,
+  RawStackTable,
   RawMarkerTable,
   Lib,
   LibMapping,
@@ -616,44 +615,9 @@ function _processFrameTable(
  * Explicitly recreate the stack table here to help enforce our assumptions about types.
  * Also add a category column.
  */
-function _processStackTable(
-  geckoStackTable: GeckoStackStruct,
-  frameTable: FrameTable,
-  categories: CategoryList
-): StackTable {
-  // Compute a non-null category for every stack
-  const defaultCategory = categories.findIndex((c) => c.color === 'grey') || 0;
-  const categoryColumn = new Array(geckoStackTable.length);
-  const subcategoryColumn = new Array(geckoStackTable.length);
-  for (let stackIndex = 0; stackIndex < geckoStackTable.length; stackIndex++) {
-    const frameIndex = geckoStackTable.frame[stackIndex];
-    const frameCategory = frameTable.category[frameIndex];
-    const frameSubcategory = frameTable.subcategory[frameIndex];
-    let stackCategory;
-    let stackSubcategory;
-    if (frameCategory !== null) {
-      stackCategory = frameCategory;
-      stackSubcategory = frameSubcategory || 0;
-    } else {
-      const prefix = geckoStackTable.prefix[stackIndex];
-      if (prefix !== null) {
-        // Because of the structure of the stack table, prefix < stackIndex.
-        // So we've already computed the category for the prefix.
-        stackCategory = categoryColumn[prefix];
-        stackSubcategory = subcategoryColumn[prefix];
-      } else {
-        stackCategory = defaultCategory;
-        stackSubcategory = 0;
-      }
-    }
-    categoryColumn[stackIndex] = stackCategory;
-    subcategoryColumn[stackIndex] = stackSubcategory;
-  }
-
+function _processStackTable(geckoStackTable: GeckoStackStruct): RawStackTable {
   return {
     frame: geckoStackTable.frame,
-    category: categoryColumn,
-    subcategory: subcategoryColumn,
     prefix: geckoStackTable.prefix,
     length: geckoStackTable.length,
   };
@@ -1132,7 +1096,7 @@ function _processThread(
   );
 
   const { libs, pausedRanges, meta } = processProfile;
-  const { categories, shutdownTime } = meta;
+  const { shutdownTime } = meta;
 
   const mutatedStringArray = thread.stringTable.slice();
   const stringTable = StringTable.withBackingArray(mutatedStringArray);
@@ -1151,11 +1115,7 @@ function _processThread(
     frameFuncs,
     frameAddresses
   );
-  const stackTable = _processStackTable(
-    geckoStackTable,
-    frameTable,
-    categories
-  );
+  const stackTable = _processStackTable(geckoStackTable);
   const { markers, jsAllocations, nativeAllocations } =
     _processMarkers(geckoMarkers);
   const samples = _processSamples(geckoSamples);

src/profile-logic/processed-profile-versioning.js

@@ -2299,6 +2299,31 @@ const _upgraders = {
     // marker data with the new field types data, and no modification is needed in the
     // frontend to display older formats.
   },
+  [52]: (profile) => {
+    for (const thread of profile.threads) {
+      const { frameTable, stackTable } = thread;
+
+      // Workaround for Lean profiles missing frameTable.category
+      if (!('category' in frameTable)) {
+        frameTable.category = [];
+        frameTable.subcategory = [];
+        for (let frameIndex = 0; frameIndex < frameTable.length; frameIndex++) {
+          frameTable.category[frameIndex] = null;
+          frameTable.subcategory[frameIndex] = null;
+        }
+        for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
+          const frameIndex = stackTable.frame[stackIndex];
+          frameTable.category[frameIndex] = stackTable.category[stackIndex];
+          frameTable.subcategory[frameIndex] =
+            stackTable.subcategory[stackIndex];
+        }
+      }
+
+      // Remove stackTable.category and stackTable.subcategory.
+      delete stackTable.category;
+      delete stackTable.subcategory;
+    }
+  },
   // If you add a new upgrader here, please document the change in
   // `docs-developer/CHANGELOG-formats.md`.
 };