Merge branch 'main' into bernardobridge/art-1547-allow-beforeafter-validate-script-schema-to-run-arbitrary

Author: bernardobridge

.github/workflows/docker-ecs-worker-image.yml

@@ -5,6 +5,15 @@ on:
   push:
     branches:
       - main
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Branch ref to checkout. Needed for pull_request_target to be able to pull correct ref.'
+        type: string
+    secrets:
+      ECR_WORKER_IMAGE_PUSH_ROLE_ARN:
+        description: 'ARN of the IAM role to assume to push the image to ECR.'
+        required: true
 
 permissions:
   id-token: write
@@ -16,6 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
+          ref: ${{ inputs.ref || null }}
           fetch-depth: 0
 
       - name: Show git ref

.github/workflows/run-aws-tests.yml

@@ -0,0 +1,75 @@
+name: Run AWS tests
+
+on:
+  pull_request_target:
+    branches: [main]
+    #opened, reopened and synchronize will cause the workflow to fail on forks due to permissions
+    #once labeled, that will then be overridden by the is-collaborator job
+    types: [opened, labeled, synchronize, reopened]
+
+jobs:
+  is-collaborator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get User Permission
+        id: checkAccess
+        uses: actions-cool/check-user-permission@cd622002ff25c2311d2e7fb82107c0d24be83f9b
+        with:
+          require: write
+          username: ${{ github.actor }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check User Permission
+        if: steps.checkAccess.outputs.require-result == 'false'
+        run: |
+          echo "${{ github.actor }} does not have permissions on this repo."
+          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
+          exit 1
+
+  publish-branch-image:
+    if: contains( github.event.pull_request.labels.*.name, 'run-aws-tests' )
+    needs: is-collaborator
+    uses: ./.github/workflows/docker-ecs-worker-image.yml
+    permissions:
+      contents: read
+      id-token: write
+    secrets:
+      ECR_WORKER_IMAGE_PUSH_ROLE_ARN: ${{ secrets.ECR_WORKER_IMAGE_PUSH_ROLE_ARN }}
+    with:
+      ref: ${{ github.event.pull_request.head.sha || null }} # this should only be run with this ref if is-collaborator has been run and passed
+
+  run-tests:
+    if: contains( github.event.pull_request.labels.*.name, 'run-aws-tests' )
+    needs: publish-branch-image
+    timeout-minutes: 40
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha || null }} # this should only be run with this ref if is-collaborator has been run and passed
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        env:
+          SHOW_STACK_TRACE: true
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.ARTILLERY_AWS_CLI_ROLE_ARN_TEST1 }}
+          role-session-name: OIDCSession
+          mask-aws-account-id: true
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18.x
+      - run: npm install
+      - run: npm run build
+      - run: npm run test:aws --workspace artillery
+        env:
+          FORCE_COLOR: 1
+          ECR_IMAGE_VERSION: ${{ github.sha }} # the image is published with the sha of the commit within this repo
+          ARTILLERY_CLOUD_ENDPOINT: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
+          ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
+          GITHUB_REPO: ${{ github.repository }}
+          GITHUB_ACTOR: ${{ github.actor }}

packages/artillery/lib/cmds/run.js

@@ -22,7 +22,6 @@ const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
 const os = require('os');
-const esbuild = require('esbuild-wasm');
 const createLauncher = require('../launch-platform');
 const createConsoleReporter = require('../../console-reporter');
 
@@ -406,6 +405,10 @@ function replaceProcessorIfTypescript(script, scriptPath, platform) {
     `${processorFileName}-${Date.now()}.js`
   );
 
+  //TODO: move require to top of file when Lambda bundle size issue is solved
+  //must be conditionally required for now as this package is removed in Lambda for now to avoid bigger package sizes
+  const esbuild = require('esbuild-wasm');
+
   try {
     esbuild.buildSync({
       entryPoints: [actualProcessorPath],
@@ -634,6 +637,30 @@ async function sendTelemetry(script, flags, extraProps) {
       }
     }
 
+    // publish-metrics reporters
+    if (script.config.plugins['publish-metrics']) {
+      const OFFICIAL_REPORTERS = [
+        'datadog',
+        'open-telemetry',
+        'lightstep',
+        'newrelic',
+        'splunk',
+        'dynatrace',
+        'cloudwatch',
+        'honeycomb',
+        'mixpanel',
+        'prometheus'
+      ];
+
+      properties.officialMonitoringReporters = script.config.plugins[
+        'publish-metrics'
+      ].map((reporter) => {
+        if (OFFICIAL_REPORTERS.includes(reporter.type)) {
+          return reporter.type;
+        }
+      });
+    }
+
     // before/after hooks
     if (script.before) {
       properties.beforeHook = true;

packages/artillery/package.json

@@ -46,7 +46,7 @@
     "test:unit": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 --color test/unit/*.test.js",
     "test:acceptance": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --no-coverage --color test/testcases/plugins/*.test.js",
     "test": "npm run test:unit && npm run test:acceptance",
-    "test:cloud": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=300 test/cloud-e2e/*.test.js",
+    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=3600 test/cloud-e2e/**/*.test.js",
     "lint": "eslint --ext \".js,.ts,.tsx\" .",
     "lint-fix": "npm run lint -- --fix"
   },

packages/artillery/test/cli/_helpers.js

@@ -5,6 +5,7 @@ const path = require('path');
 const os = require('os');
 const { getBinPathSync } = require('get-bin-path');
 const a9path = getBinPathSync();
+const { createHash } = require('crypto');
 
 async function execute(args, options) {
   try {
@@ -30,4 +31,27 @@ async function getRootPath(filename) {
   return path.resolve(__dirname, '..', '..', filename);
 }
 
-module.exports = { execute, deleteFile, getRootPath, returnTmpPath };
+function generateTmpReportPath(testName, extension) {
+  return returnTmpPath(
+    `report-${createHash('md5')
+      .update(testName)
+      .digest('hex')}-${Date.now()}.${extension}`
+  );
+}
+
+function getTestTags(additionalTags) {
+  const actorTag = `actor:${process.env.GITHUB_ACTOR || 'localhost'}`;
+  const repoTag = `repo:${process.env.GITHUB_REPO || 'artilleryio/artillery'}`;
+  const ciTag = `ci:${process.env.GITHUB_ACTIONS ? 'true' : 'false'}`;
+
+  return `${repoTag},${actorTag},${ciTag},${additionalTags.join(',')}`;
+}
+
+module.exports = {
+  execute,
+  deleteFile,
+  getRootPath,
+  returnTmpPath,
+  generateTmpReportPath,
+  getTestTags
+};

packages/artillery/test/cli/command-report.test.js

@@ -1,8 +1,8 @@
 const tap = require('tap');
-const { execute, returnTmpPath } = require('../cli/_helpers.js');
+const { execute, generateTmpReportPath } = require('../cli/_helpers.js');
 
 tap.test('If we report specifying output, no browser is opened', async (t) => {
-  const outputFilePath = returnTmpPath('report.html');
+  const outputFilePath = generateTmpReportPath(t.name, 'html');
 
   const [exitCode] = await execute([
     'report',

packages/artillery/test/cli/command-run.test.js

@@ -3,20 +3,16 @@ const {
   execute,
   deleteFile,
   getRootPath,
-  returnTmpPath
+  returnTmpPath,
+  generateTmpReportPath
 } = require('../cli/_helpers.js');
 const fs = require('fs');
 const path = require('path');
 const execa = require('execa');
-const { createHash } = require('crypto');
 
 let reportFilePath;
 tap.beforeEach(async (t) => {
-  reportFilePath = returnTmpPath(
-    `report-${createHash('md5')
-      .update(t.name)
-      .digest('hex')}-${Date.now()}.json`
-  );
+  reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
 tap.test('Run a simple script', async (t) => {

packages/artillery/test/cli/run-typescript.test.js

@@ -1,15 +1,10 @@
 const tap = require('tap');
-const { execute, returnTmpPath } = require('../cli/_helpers.js');
-const { createHash } = require('crypto');
+const { execute, generateTmpReportPath } = require('../cli/_helpers.js');
 const fs = require('fs');
 
 let reportFilePath;
 tap.beforeEach(async (t) => {
-  reportFilePath = returnTmpPath(
-    `report-${createHash('md5')
-      .update(t.name)
-      .digest('hex')}-${Date.now()}.json`
-  );
+  reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
 tap.test('Can run a Typescript processor', async (t) => {

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-exit-conditions/with-expect-ensure.yml

@@ -0,0 +1,18 @@
+config:
+  target: "http://asciiart.artillery.io:8080"
+  plugins:
+    expect: {}
+    ensure:
+      p99: 10000
+      thresholds:
+        - "http.response_time.p95": 1
+  phases:
+    - duration: 5
+      arrivalRate: 1
+scenarios:
+  - name: expect-ensure-exit-condition-test
+    flow:
+      - get:
+          url: "/"
+          expect:
+            - statusCode: 300

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-exit-conditions/with-expect.yml

@@ -0,0 +1,14 @@
+config:
+  target: "http://asciiart.artillery.io:8080"
+  plugins:
+    expect: {}
+  phases:
+    - duration: 5
+      arrivalRate: 1
+scenarios:
+  - name: expect-exit-condition-test
+    flow:
+      - get:
+          url: "/"
+          expect:
+            - statusCode: 300

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-kitchen-sink/kitchen-sink-env

@@ -0,0 +1,2 @@
+DOTENV1=/
+DOTENV2=/dino

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-kitchen-sink/kitchen-sink.yml

@@ -0,0 +1,22 @@
+config:
+  target: http://asciiart.artillery.io:8080
+  plugins:
+    ensure: {}
+  phases:
+    - duration: 20
+      arrivalRate: 1
+  ensure:
+    p99: 10000
+    thresholds:
+      - "http.response_time.p99": 10000
+scenarios:
+  - name: load homepage
+    flow:
+      - get:
+          url: "{{$processEnvironment.SECRET1}}"
+      - get:
+          url: "{{$processEnvironment.SECRET2}}"
+      - get:
+          url: "{{$processEnvironment.DOTENV1}}"
+      - get:
+          url: "{{$processEnvironment.DOTENV2}}"

packages/artillery/test/cloud-e2e/fargate/fixtures/large-output/lots-of-output.yml

@@ -0,0 +1,14 @@
+config:
+  target: http://asciiart.artillery.io:8080
+  phases:
+    - arrivalRate: 1
+      duration: 60
+  defaults:
+    headers:
+      user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"
+  processor: "./processor.js"
+scenarios:
+  - flow:
+    - get:
+        url: "/"
+        afterResponse: logOutput

packages/artillery/test/cloud-e2e/fargate/fixtures/large-output/processor.js

@@ -0,0 +1,19 @@
+function logOutput(req, res, userContext, events, done) {
+  for (let i = 0; i < 10; i++) {
+    events.emit(
+      'counter',
+      `very.very.long.name.for.a.counter.metric.so.that.we.generate.a.lot.of.console.output.${Date.now()}${i}`,
+      1
+    );
+    events.emit(
+      'histogram',
+      `very.very.long.name.for.a.histogram.metric.so.that.we.generate.a.lot.of.console.output.${Date.now()}${i}`,
+      100
+    );
+  }
+  return done();
+}
+
+module.exports = {
+  logOutput
+};

packages/artillery/test/cloud-e2e/fargate/fixtures/memory-hog/memory-hog.yml

@@ -0,0 +1,20 @@
+# 10 VUs, making 10 calls each to a function that allocates 100MB -> ~10GB total
+# This will fail with default configuration, which is:
+# - 4GB RAM given to workers on Fargate
+# - Node.js memory limit of 4GB
+
+config:
+  target: http://asciiart.artillery.io:8080
+  phases:
+    - arrivalRate: 1
+      duration: 10
+  processor: './processor.js'
+
+scenarios:
+  - flow:
+      - loop:
+          - get:
+              url: "/armadillo"
+              beforeRequest: hogSomeRam
+          - think: 1
+        count: 10

packages/artillery/test/cloud-e2e/fargate/fixtures/memory-hog/processor.js

@@ -0,0 +1,18 @@
+let data = [];
+
+console.log('NODE_OPTIONS:');
+console.log(process.env.NODE_OPTIONS);
+
+function hogSomeRam(req, context, events, next) {
+  // Allocate 100MB
+  data.push(Buffer.alloc(1024 * 1024 * 100, 1));
+
+  console.log(new Date(), 'allocated more data');
+  console.log('RSS (MB):', process.memoryUsage().rss / 1024 / 1024);
+
+  return next();
+}
+
+module.exports = {
+  hogSomeRam
+};

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/code/functions.js

@@ -0,0 +1,7 @@
+const bc = require('@babel/core');
+const uuid = require('uuid');
+const client = require('aws-sdk/clients/lambda');
+
+module.exports = {
+  setUrl: require('./set-url')
+};