feat: add smart sampling for OTel tracing (#2343)

Author: InesNi

packages/artillery-plugin-publish-metrics/lib/open-telemetry/outlier-detection-processor.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');
+const { diag } = require('@opentelemetry/api');
+
+class OutlierDetectionBatchSpanProcessor extends BatchSpanProcessor {
+  constructor(exporter, config) {
+    super(exporter, config);
+    this._traces = new Map();
+  }
+
+  onEnd(span) {
+    if (span.instrumentationLibrary.name === 'artillery-playwright') {
+      super.onEnd(span);
+    } else {
+      const traceId = span.spanContext().traceId;
+
+      // When an outlier span is recognised the whole trace it belongs to is exported, so all the spans that belong to the trace need to be grouped and held until the trace finishes.
+      if (!this._traces.has(traceId)) {
+        this._traces.set(traceId, {
+          spans: [],
+          hasOutlier: false
+        });
+      }
+      const traceData = this._traces.get(traceId);
+      traceData.spans.push(span);
+
+      // Since only request level spans are screened for outliers, the outlier check is performed only if the span is a request level span - has 'http.url' attribute
+      if (span.attributes['http.url'] && this._isOutlier(span)) {
+        traceData.hasOutlier = true;
+      }
+
+      // The trace ends when the root span ends, so we only filter and send data when the span that ended is the root span
+      // The traces that do not have outlier spans are dropped and the rest is sent to buffer/export
+      if (!span.parentSpanId) {
+        if (traceData.hasOutlier) {
+          traceData.spans.forEach(super.onEnd, this);
+        }
+        this._traces.delete(traceId);
+      }
+    }
+  }
+  // Export only outliers on shut down as well for http engine
+  onShutdown() {
+    this._traces.forEach((traceData, traceId) => {
+      if (traceData.hasOutlier) {
+        traceData.spans.forEach(super.onEnd, this);
+      }
+    });
+    this._traces.clear();
+    // By here all the available HTTP engine traces are processed and sent to buffer, the parent onShutDown will cover for possible playwright engine traces and the shutdown process
+    super.onShutdown();
+  }
+
+  // The outlier detection logic based on the provided criteria
+  _isOutlier(span) {
+    return !!span.attributes.outlier;
+  }
+}
+
+module.exports = { OutlierDetectionBatchSpanProcessor };

packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js

@@ -3,6 +3,9 @@
 const debug = require('debug')('plugin:publish-metrics:open-telemetry');
 const grpc = require('@grpc/grpc-js');
 const { traceExporters, validateExporter } = require('../exporters');
+const {
+  OutlierDetectionBatchSpanProcessor
+} = require('../outlier-detection-processor');
 
 const { SemanticAttributes } = require('@opentelemetry/semantic-conventions');
 const {
@@ -57,8 +60,12 @@ class OTelTraceConfig {
       this.exporterOpts
     );
 
+    const Processor = this.config.smartSampling
+      ? OutlierDetectionBatchSpanProcessor
+      : BatchSpanProcessor;
+
     this.tracerProvider.addSpanProcessor(
-      new BatchSpanProcessor(this.exporter, {
+      new Processor(this.exporter, {
         scheduledDelayMillis: 1000
       })
     );

packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/http.js

@@ -15,6 +15,8 @@ const {
 class OTelHTTPTraceReporter extends OTelTraceBase {
   constructor(config, script) {
     super(config, script);
+    this.outlierCriteria = config.smartSampling;
+    this.statusAsErrorThreshold = 400;
   }
   run() {
     this.setTracer('http');
@@ -86,10 +88,14 @@ class OTelHTTPTraceReporter extends OTelTraceBase {
     if (!userContext.vars['__otlpHTTPRequestSpan']) {
       return done();
     }
-
     const span = userContext.vars['__otlpHTTPRequestSpan'];
     let endTime;
 
+    const scenarioSpan = userContext.vars['__httpScenarioSpan'];
+    if (this.config.smartSampling) {
+      this.tagResponseOutliers(span, scenarioSpan, res, this.outlierCriteria);
+    }
+
     if (res.timings && res.timings.phases) {
       span.setAttribute('response.time.ms', res.timings.phases.firstByte);
 
@@ -134,7 +140,7 @@ class OTelHTTPTraceReporter extends OTelTraceBase {
           res.request.options.headers['user-agent']
       });
 
-      if (res.statusCode >= 400) {
+      if (res.statusCode >= this.statusAsErrorThreshold) {
         span.setStatus({
           code: SpanStatusCode.ERROR,
           message: res.statusMessage
@@ -161,6 +167,14 @@ class OTelHTTPTraceReporter extends OTelTraceBase {
         code: SpanStatusCode.ERROR,
         message: err.message || err
       });
+
+      if (this.config.smartSampling) {
+        requestSpan.setAttributes({
+          outlier: 'true',
+          'outlier.type.error': true
+        });
+      }
+
       requestSpan.end();
       this.pendingRequestSpans--;
     } else {
@@ -171,10 +185,50 @@ class OTelHTTPTraceReporter extends OTelTraceBase {
       code: SpanStatusCode.ERROR,
       message: err.message || err
     });
+
+    if (this.config.smartSampling) {
+      scenarioSpan.setAttributes({
+        outlier: 'true',
+        'outlier.type.error': true
+      });
+    }
+
     scenarioSpan.end();
     this.pendingScenarioSpans--;
     return done();
   }
+
+  tagResponseOutliers(span, scenarioSpan, res, criteria) {
+    const types = {};
+    const details = [];
+    if (res.statusCode >= this.statusAsErrorThreshold) {
+      types['outlier.type.status_code'] = true;
+      details.push(`HTTP Status Code >= ${this.statusAsErrorThreshold}`);
+    }
+    if (criteria.thresholds && res.timings?.phases) {
+      Object.entries(criteria.thresholds).forEach(([name, value]) => {
+        if (res.timings.phases[name] >= value) {
+          types[`outlier.type.${name}`] = true;
+          details.push(`'${name}' >= ${value}`);
+        }
+      });
+    }
+
+    if (!details.length) {
+      return;
+    }
+
+    span.setAttributes({
+      outlier: 'true',
+      'outlier.details': details.join(', '),
+      ...types
+    });
+
+    scenarioSpan.setAttributes({
+      outlier: 'true',
+      ...types
+    });
+  }
 }
 
 module.exports = {

packages/types/schema/plugins/publish-metrics.js

@@ -181,7 +181,13 @@ const OpenTelemetryReporterSchema = Joi.object({
       .default('otlp-http'),
     sampleRate: artilleryNumberOrString,
     useRequestNames: artilleryBooleanOrString,
-    attributes: Joi.object().unknown()
+    attributes: Joi.object().unknown(),
+    smartSampling: Joi.object({
+      thresholds: Joi.object({
+        firstByte: artilleryNumberOrString,
+        total: artilleryNumberOrString
+      })
+    })
   })
 })
   .unknown(false)