feat: Refactor feature storages (#1241)

Co-authored-by: ptang-nr <[email protected]>

Author: cwli24

.github/workflows/wdio-single-browser.yml

@@ -16,6 +16,11 @@ on:
         required: false
         type: boolean
         default: false
+      concurrency:
+        description: 'The number of test runner threads to spawn for a run'
+        required: false
+        type: number
+        default: 10
   workflow_call:
     inputs:
       browser-target:

src/common/aggregate/aggregator.js

@@ -2,12 +2,8 @@
  * Copyright 2020 New Relic Corporation. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
-
-import { SharedContext } from '../context/shared-context'
-
-export class Aggregator extends SharedContext {
-  constructor (parent) {
-    super(parent)
+export class Aggregator {
+  constructor () {
     this.aggregatedData = {}
   }
 
@@ -16,13 +12,14 @@ export class Aggregator extends SharedContext {
   // metrics are the numeric values to be aggregated
 
   store (type, name, params, newMetrics, customParams) {
-    var bucket = this.getBucket(type, name, params, customParams)
+    var bucket = this.#getBucket(type, name, params, customParams)
     bucket.metrics = aggregateMetrics(newMetrics, bucket.metrics)
     return bucket
   }
 
-  merge (type, name, metrics, params, customParams) {
-    var bucket = this.getBucket(type, name, params, customParams)
+  merge (type, name, metrics, params, customParams, overwriteParams = false) {
+    var bucket = this.#getBucket(type, name, params, customParams)
+    if (overwriteParams) bucket.params = params // replace current params with incoming params obj
 
     if (!bucket.metrics) {
       bucket.metrics = metrics
@@ -50,32 +47,13 @@ export class Aggregator extends SharedContext {
   }
 
   storeMetric (type, name, params, value) {
-    var bucket = this.getBucket(type, name, params)
+    var bucket = this.#getBucket(type, name, params)
     bucket.stats = updateMetric(value, bucket.stats)
     return bucket
   }
 
-  getBucket (type, name, params, customParams) {
-    if (!this.aggregatedData[type]) this.aggregatedData[type] = {}
-    var bucket = this.aggregatedData[type][name]
-    if (!bucket) {
-      bucket = this.aggregatedData[type][name] = { params: params || {} }
-      if (customParams) {
-        bucket.custom = customParams
-      }
-    }
-    return bucket
-  }
-
-  get (type, name) {
-    // if name is passed, get a single bucket
-    if (name) return this.aggregatedData[type] && this.aggregatedData[type][name]
-    // else, get all buckets of that type
-    return this.aggregatedData[type]
-  }
-
-  // Like get, but for many types and it deletes the retrieved content from the aggregatedData
-  take (types) {
+  // Get all listed types buckets and it deletes the retrieved content from the aggregatedData
+  take (types, deleteWhenRetrieved = true) {
     var results = {}
     var type = ''
     var hasData = false
@@ -84,10 +62,22 @@ export class Aggregator extends SharedContext {
       results[type] = Object.values(this.aggregatedData[type] || {})
 
       if (results[type].length) hasData = true
-      delete this.aggregatedData[type]
+      if (deleteWhenRetrieved) delete this.aggregatedData[type]
     }
     return hasData ? results : null
   }
+
+  #getBucket (type, name, params, customParams) {
+    if (!this.aggregatedData[type]) this.aggregatedData[type] = {}
+    var bucket = this.aggregatedData[type][name]
+    if (!bucket) {
+      bucket = this.aggregatedData[type][name] = { params: params || {} }
+      if (customParams) {
+        bucket.custom = customParams
+      }
+    }
+    return bucket
+  }
 }
 
 function aggregateMetrics (newMetrics, oldMetrics) {

src/common/aggregate/event-aggregator.js

@@ -0,0 +1,76 @@
+import { Aggregator } from './aggregator'
+
+/**
+ * An extension of the Aggregator class that provides an interface similar to that of EventBuffer class.
+ * This typecasting allow features that uses Aggregator as their event handler to share the same AggregateBase.events utilization by those features.
+ */
+export class EventAggregator {
+  #aggregator = new Aggregator()
+  #savedNamesToBuckets = {}
+
+  isEmpty ({ aggregatorTypes }) {
+    if (!aggregatorTypes) return Object.keys(this.#aggregator.aggregatedData).length === 0
+    return aggregatorTypes.every(type => !this.#aggregator.aggregatedData[type]) // no bucket exist for any of the types we're looking for
+  }
+
+  add (type, name, params, newMetrics, customParams) {
+    // Do we need to track byte size here like EventBuffer?
+    this.#aggregator.store(type, name, params, newMetrics, customParams)
+    return true
+  }
+
+  addMetric (type, name, params, value) {
+    this.#aggregator.storeMetric(type, name, params, value)
+    return true
+  }
+
+  save ({ aggregatorTypes }) {
+    const key = aggregatorTypes.toString() // the stringified types serve as the key to each save call, e.g. ['err', 'ierr', 'xhr'] => 'err,ierr,xhr'
+    const backupAggregatedDataSubset = {}
+    aggregatorTypes.forEach(type => (backupAggregatedDataSubset[type] = this.#aggregator.aggregatedData[type])) // make a subset of the aggregatedData for each of the types we want to save
+    this.#savedNamesToBuckets[key] = backupAggregatedDataSubset
+    /*
+    { 'err,ierr,xhr': {
+        'err': {
+            <aggregateHash>: { metrics: { count: 1, time, ... }, params: {}, custom: {} },
+            <otherHashName>: { metrics: { count: 1, ... }, ... }
+        },
+        'ierr': { ... },
+        'xhr': { ... }
+      }
+    }
+    */
+  }
+
+  get (opts) {
+    const aggregatorTypes = Array.isArray(opts) ? opts : opts.aggregatorTypes
+    return this.#aggregator.take(aggregatorTypes, false)
+  }
+
+  clear ({ aggregatorTypes } = {}) {
+    if (!aggregatorTypes) {
+      this.#aggregator.aggregatedData = {}
+      return
+    }
+    aggregatorTypes.forEach(type => delete this.#aggregator.aggregatedData[type])
+  }
+
+  reloadSave ({ aggregatorTypes }) {
+    const key = aggregatorTypes.toString()
+    const backupAggregatedDataSubset = this.#savedNamesToBuckets[key]
+    // Grabs the previously stored subset and merge it back into aggregatedData.
+    aggregatorTypes.forEach(type => {
+      Object.keys(backupAggregatedDataSubset[type] || {}).forEach(name => {
+        const bucket = backupAggregatedDataSubset[type][name]
+        // The older aka saved params take effect over the newer one. This is especially important when merging back for a failed harvest retry if, for example,
+        // the first-ever occurrence of an error is in the retry: it contains the params.stack_trace whereas the newer or current bucket.params would not.
+        this.#aggregator.merge(type, name, bucket.metrics, bucket.params, bucket.custom, true)
+      })
+    })
+  }
+
+  clearSave ({ aggregatorTypes }) {
+    const key = aggregatorTypes.toString()
+    delete this.#savedNamesToBuckets[key]
+  }
+}

src/common/harvest/harvest-scheduler.js

@@ -98,7 +98,7 @@ export class HarvestScheduler extends SharedContext {
     let payload
 
     if (this.opts.getPayload) {
-      // Ajax & PVT & SR features provide a callback function to get data for harvesting
+      // Ajax, PVT, Softnav, Logging, SR & ST features provide a single callback function to get data for harvesting
       submitMethod = submitData.getSubmitMethod({ isFinalHarvest: opts?.unload })
       if (!submitMethod) return false
 

src/common/harvest/harvest.js

@@ -106,11 +106,7 @@ export class Harvest extends SharedContext {
     const gzip = !!qs?.attributes?.includes('gzip')
 
     if (!gzip) {
-      if (endpoint === 'events') {
-        body = body.e
-      } else {
-        body = stringify(body)
-      }
+      if (endpoint !== 'events') body = stringify(body) // all features going to /events/ endpoint should already be serialized & stringified
       /** Warn --once per endpoint-- if the agent tries to send large payloads */
       if (body.length > 750000 && (warnings[endpoint] = (warnings?.[endpoint] || 0) + 1) === 1) warn(28, endpoint)
     }

src/common/harvest/types.js

@@ -12,7 +12,6 @@
  * @typedef {object} HarvestPayload
  * @property {object} qs Map of values that should be sent as part of the request query string.
  * @property {object} body Map of values that should be sent as the body of the request.
- * @property {string} body.e Special case of body used for browser interactions.
  */
 
 /**