JSON output (#128)

Summary:
PR changes:
* Adds `--output [text/json]` CLI option which sets the `outputFormat` config flag
* When output is set to JSON, it implies the `--sc` option and directs all logs to `stderr`. This allows to easily capture a clean JSON output using `memlab > result.json`.
* Adds `getJSONifyableObject` to the interfaces of nodes and edges
* Updates `printNodeListInTerminal` and `printReferencesInTerminal` to support JSON output
* Updates `CollectionsHoldingStaleAnalysis` to support JSON output

Open questions:
* Should all analyses support JSON output? I only added the ones I need at the moment.
* The output from `getJSONifyableObject` has inconsistent casing (e.g. snake `self_size` vs. camel `incomingEdgeCount`). Is it a breaking change to change this so it's all the same? Which case is preferrred?

Fixes #127

Pull Request resolved: #128

Reviewed By: twobassdrum

Differential Revision: D61724639

Pulled By: JacksonGL

fbshipit-source-id: 13a056be1c421999ffbd988ee5f85026d66c860d

packages/core/src/lib/Config.ts

@@ -71,6 +71,12 @@ export enum TraceObjectMode {
   SelectedJSObjects = 2,
 }
 
+/** @internal */
+export enum OutputFormat {
+  Text = 1,
+  Json = 2,
+}
+
 /** @internal */
 export enum ErrorHandling {
   Halt = 1,
@@ -90,6 +96,7 @@ export type MuteConfig = {
   muteHighLevel?: boolean;
   muteMidLevel?: boolean;
   muteLowLevel?: boolean;
+  muteOutput?: boolean;
 };
 
 /** @internal */
@@ -260,6 +267,7 @@ export class MemLabConfig {
   skipBrowserCloseWait: boolean;
   simplifyCodeSerialization: boolean;
   heapParserDictFastStoreSize: number;
+  outputFormat: OutputFormat;
 
   constructor(options: ConfigOption = {}) {
     // init properties, they can be configured manually

packages/core/src/lib/Console.ts

@@ -15,7 +15,7 @@ import fs from 'fs';
 import path from 'path';
 import readline from 'readline';
 import stringWidth from 'string-width';
-import type {MemLabConfig} from './Config';
+import {OutputFormat, type MemLabConfig} from './Config';
 import {AnyValue, Nullable, Optional} from './Types';
 
 type Message = {
@@ -39,7 +39,6 @@ type Sections = {
   arr: Section[];
 };
 
-const stdout = process.stdout;
 const TABLE_MAX_WIDTH = 50;
 const LOG_BUFFER_LENGTH = 100;
 const prevLine = '\x1b[F';
@@ -149,6 +148,14 @@ class MemLabConsole {
     return inst;
   }
 
+  private get isTextOutput(): boolean {
+    return this.config.outputFormat === OutputFormat.Text;
+  }
+
+  private get outStream() {
+    return this.isTextOutput ? process.stdout : process.stderr;
+  }
+
   private style(msg: string, name: keyof MemlabConsoleStyles): string {
     if (Object.prototype.hasOwnProperty.call(this.styles, name)) {
       return this.styles[name](msg);
@@ -183,6 +190,10 @@ class MemLabConsole {
         .replace(/\[\d{1,3}m/g, ''),
     );
     this.log.push(...lines);
+    this.tryFlush();
+  }
+
+  private tryFlush(): void {
     if (this.log.length > LOG_BUFFER_LENGTH) {
       this.flushLog({sync: true});
     }
@@ -243,7 +254,7 @@ class MemLabConsole {
       return;
     }
     if (!this.config.muteConsole) {
-      stdout.write(eraseLine);
+      this.outStream.write(eraseLine);
     }
     const msg = section.msgs.pop();
 
@@ -254,11 +265,11 @@ class MemLabConsole {
     const lines = msg.lines;
     while (lines.length > 0) {
       const line = lines.pop() ?? 0;
-      const width = stdout.columns;
+      const width = this.outStream.columns;
       let n = line === 0 ? 1 : Math.ceil(line / width);
       if (!this.config.muteConsole && !this.config.isTest) {
         while (n-- > 0) {
-          stdout.write(prevLine + eraseLine);
+          this.outStream.write(prevLine + eraseLine);
         }
       }
     }
@@ -306,8 +317,23 @@ class MemLabConsole {
       return;
     }
     if (this.config.isContinuousTest || !this.config.muteConsole) {
-      console.log(msg);
+      if (this.isTextOutput) {
+        console.log(msg);
+      } else {
+        this.outStream.write(msg);
+        this.outStream.write('\n');
+      }
+    }
+  }
+
+  public writeOutput(output: string): void {
+    this.log.push(output);
+    if (this.config.muteConfig?.muteOutput) {
+      return;
     }
+    process.stdout.write(output);
+
+    this.tryFlush();
   }
 
   public registerLogFile(logFile: string): void {
@@ -498,7 +524,7 @@ class MemLabConsole {
   public waitForConsole(query: string): Promise<string> {
     const rl = readline.createInterface({
       input: process.stdin,
-      output: process.stdout,
+      output: this.outStream,
     });
     this.pushMsg(query);
     this.logMsg(query);
@@ -515,7 +541,7 @@ class MemLabConsole {
     total: number,
     options: {message?: string} = {},
   ): void {
-    let width = Math.floor(stdout.columns * 0.8);
+    let width = Math.floor(this.outStream.columns * 0.8);
     width = Math.min(width, 80);
     const messageMaxWidth = Math.floor(width * 0.3);
     let message = options.message || '';

packages/core/src/lib/Types.ts

@@ -1592,6 +1592,14 @@ export interface IHeapLocation {
    * get the column number
    */
   column: number;
+  /**
+   * convert to a concise readable object that can be used for serialization
+   * (like calling `JSON.stringify(node, ...args)`).
+   *
+   * This API does not contain all the information
+   * captured by the hosting object.
+   */
+  getJSONifyableObject(): AnyRecord;
   /**
    * convert to a concise readable string output
    * (like calling `JSON.stringify(node, ...args)`).
@@ -1679,6 +1687,14 @@ export interface IHeapEdge extends IHeapEdgeBasic {
    * JS heap object where this reference starts
    */
   fromNode: IHeapNode;
+  /**
+   * convert to a concise readable object that can be used for serialization
+   * (like calling `JSON.stringify(node, ...args)`).
+   *
+   * This API does not contain all the information
+   * captured by the hosting object.
+   */
+  getJSONifyableObject(): AnyRecord;
   /**
    * convert to a concise readable string output
    * (like calling `JSON.stringify(node, ...args)`).
@@ -1904,6 +1920,15 @@ export interface IHeapNode extends IHeapNodeBasic {
    * inside the string node.
    */
   toStringNode(): Nullable<IHeapStringNode>;
+  /**
+   * convert to a concise readable object that can be used for serialization
+   * (like calling `JSON.stringify(node, ...args)`).
+   *
+   * This API does not contain all the information
+   * captured by the hosting object.
+   */
+
+  getJSONifyableObject(): AnyRecord;
   /**
    * convert to a concise readable string output
    * (like calling `JSON.stringify(node, ...args)`).

packages/core/src/trace-cluster/TraceElement.ts

@@ -9,6 +9,7 @@
  */
 
 import type {
+  AnyRecord,
   AnyValue,
   EdgeIterationCallback,
   IHeapEdge,
@@ -175,8 +176,8 @@ export class NodeRecord implements IHeapNode {
     throw new Error('NodeRecord.getReferrerNodes is not implemented');
   }
 
-  toJSONString(...args: Array<AnyValue>): string {
-    const rep = {
+  getJSONifyableObject(): AnyRecord {
+    return {
       id: this.id,
       kind: this.kind,
       name: this.name,
@@ -187,8 +188,10 @@ export class NodeRecord implements IHeapNode {
       incomingEdgeCount: this.numOfReferrers,
       contructorName: this.constructor.name,
     };
+  }
 
-    return JSON.stringify(rep, ...args);
+  toJSONString(...args: Array<AnyValue>): string {
+    return JSON.stringify(this.getJSONifyableObject(), ...args);
   }
 
   constructor(node: IHeapNode) {
@@ -233,16 +236,18 @@ export class EdgeRecord implements IHeapEdge {
     this.to_node = edge.to_node;
   }
 
-  toJSONString(...args: Array<AnyValue>): string {
-    const rep = {
+  getJSONifyableObject(): AnyRecord {
+    return {
       kind: this.kind,
       name_or_index: this.name_or_index,
       type: this.type,
       edgeIndex: this.edgeIndex,
       to_node: this.to_node,
     };
+  }
 
-    return JSON.stringify(rep, ...args);
+  toJSONString(...args: Array<AnyValue>): string {
+    return JSON.stringify(this.getJSONifyableObject(), ...args);
   }
 
   set snapshot(s: IHeapSnapshot) {

packages/heap-analysis/src/PluginUtils.ts

@@ -19,6 +19,8 @@ import {
   MemLabConfig,
   config,
   takeNodeMinimalHeap,
+  OutputFormat,
+  AnyRecord,
 } from '@memlab/core';
 
 import chalk from 'chalk';
@@ -146,10 +148,47 @@ function filterOutDominators(nodeList: IHeapNode[]): IHeapNode[] {
   return nodeList.filter(node => candidateIdSet.has(node.id));
 }
 
+// Note: be cautious when using printRef = true, it may cause infinite loop
+function getNodeRecord(node: IHeapNode, printRef = false): AnyRecord {
+  const refs = node.references.slice(0, MAX_NUM_OF_EDGES_TO_PRINT);
+  return {
+    id: node.id,
+    name: node.name,
+    type: node.type,
+    selfsize: node.self_size,
+    retainedSize: node.retainedSize,
+    traceNodeId: node.trace_node_id,
+    nodeIndex: node.nodeIndex,
+    references: printRef
+      ? refs.map(edge => getEdgeRecord(edge))
+      : refs.map(edge => ({
+          name: edge.name_or_index.toString(),
+          toNode: edge.toNode.id,
+        })),
+    referrers: node.referrers.slice(0, MAX_NUM_OF_EDGES_TO_PRINT).map(edge => ({
+      name: edge.name_or_index.toString(),
+      fromNode: edge.fromNode.id,
+    })),
+  };
+}
+
+function getEdgeRecord(edge: IHeapEdge): AnyRecord {
+  return {
+    nameOrIndex: edge.name_or_index,
+    type: edge.type,
+    edgeIndex: edge.edgeIndex,
+    toNode: getNodeRecord(edge.toNode),
+    fromNode: getNodeRecord(edge.fromNode),
+  };
+}
+
 type PrintNodeOption = {
   indent?: string;
   printReferences?: boolean;
 };
+
+// Note: be cautious when setting printReferences to true,
+// it may cause infinite loop
 function printNodeListInTerminal(
   nodeList: IHeapNode[],
   options: AnyOptions & PrintNodeOption = {},
@@ -162,6 +201,14 @@ function printNodeListInTerminal(
     nodeList = filterOutDominators(nodeList);
   }
 
+  if (config.outputFormat === OutputFormat.Json) {
+    const jsonNodes = nodeList.map(node => getNodeRecord(node, printRef));
+
+    info.writeOutput(JSON.stringify(jsonNodes));
+    info.writeOutput('\n');
+    return;
+  }
+
   for (const node of nodeList) {
     const nodeInfo = getHeapObjectString(node);
     info.topLevel(`${indent}${dot}${nodeInfo}`);
@@ -171,6 +218,12 @@ function printNodeListInTerminal(
   }
 }
 
+function printNodeInTerminal(node: IHeapNode): void {
+  const nodeRecord = getNodeRecord(node);
+  info.writeOutput(JSON.stringify(nodeRecord));
+  info.writeOutput('\n');
+}
+
 function isNumeric(v: number | string): boolean {
   if (typeof v === 'number') {
     return true;
@@ -250,6 +303,13 @@ function printReferencesInTerminal(
   edgeList: IHeapEdge[],
   options: AnyOptions & PrintNodeOption = {},
 ): void {
+  if (config.outputFormat === OutputFormat.Json) {
+    const jsonEdges = edgeList.map(edge => getEdgeRecord(edge));
+
+    info.writeOutput(JSON.stringify(jsonEdges));
+    info.writeOutput('\n');
+  }
+
   const dot = chalk.grey('· ');
   const indent = options.indent || '';
   let n = 0;
@@ -773,6 +833,7 @@ export default {
   printNodeListInTerminal,
   printReferencesInTerminal,
   printReferrersInTerminal,
+  printNodeInTerminal,
   snapshotMapReduce,
   takeNodeFullHeap,
 };

packages/heap-analysis/src/options/HeapAnalysisOutputOption.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall web_perf_infra
+ */
+
+import type {ParsedArgs} from 'minimist';
+import {MemLabConfig, OutputFormat, utils} from '@memlab/core';
+import {BaseOption} from '@memlab/core';
+
+export default class HeapAnalysisOutputOption extends BaseOption {
+  getOptionName(): string {
+    return 'output';
+  }
+
+  getDescription(): string {
+    return 'specify output format of the analysis (defaults to text)';
+  }
+
+  getExampleValues(): string[] {
+    return ['text', 'json'];
+  }
+
+  async parse(config: MemLabConfig, args: ParsedArgs): Promise<void> {
+    const name = this.getOptionName();
+    const format = `${args[name]}` ?? 'text';
+    config.outputFormat = HeapAnalysisOutputOption.parseOutputFormat(format);
+    if (config.outputFormat === OutputFormat.Json) {
+      config.isContinuousTest = true;
+    }
+  }
+
+  private static parseOutputFormat(s: string): OutputFormat {
+    switch (s.toLowerCase()) {
+      case 'text':
+        return OutputFormat.Text;
+      case 'json':
+        return OutputFormat.Json;
+      default:
+        utils.haltOrThrow('Invalid output format, valid output: text, json');
+        return OutputFormat.Text;
+    }
+  }
+}