From d2f584d534918869e85871d3cc5710d5e4edecb2 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 19 Mar 2025 16:59:07 -0400 Subject: [PATCH 1/6] chore: add tags to normalized_throughput --- test/benchmarks/driver_bench/src/main.mts | 43 ++++++++++++++--------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 04e573b7734..f032f5100d7 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -110,6 +110,13 @@ for (const [suite, benchmarks] of Object.entries(tests)) { console.groupEnd(); } +const aMetricInfo = + (testName: string) => + ({ info: { test_name } }: MetricInfo) => + test_name === testName; + +const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; + function calculateCompositeBenchmarks(results: MetricInfo[]) { const composites = { singleBench: ['findOne', 'smallDocInsertOne', 'largeDocInsertOne'], @@ -144,13 +151,6 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { ] }; - const aMetricInfo = - (testName: string) => - ({ info: { test_name } }: MetricInfo) => - test_name === testName; - - const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; - let readBenchResult; let writeBenchResult; @@ -199,31 +199,40 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { } function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { - const baselineBench = results.find(r => r.info.test_name === 'cpuBaseline'); - const pingBench = results.find(r => r.info.test_name === 'ping'); + const baselineBench = results.find(aMetricInfo('cpuBaseline')); + const pingBench = results.find(aMetricInfo('ping')); assert.ok(pingBench, 'ping bench results not found!'); - assert.ok(baselineBench, 'baseline results not found!'); - const pingThroughput = pingBench.metrics[0].value; - const cpuBaseline = baselineBench.metrics[0].value; + assert.ok(baselineBench, 'cpuBaseline results not found!'); + + const cpuBaseline = baselineBench.metrics.find(anMBsMetric); + const pingThroughput = pingBench.metrics.find(anMBsMetric); + + assert.ok(cpuBaseline, 'cpu benchmark does not have a MB/s metric'); + assert.ok(pingThroughput, 'ping does not have a MB/s metric'); for (const bench of results) { if (bench.info.test_name === 'cpuBaseline') continue; + + const currentMetric = bench.metrics.find(anMBsMetric); + assert.ok(currentMetric, `${bench.info.test_name} does not have a MB/s metric`); + if (bench.info.test_name === 'ping') { bench.metrics.push({ name: 'normalized_throughput', - value: bench.metrics[0].value / cpuBaseline, + value: currentMetric.value / cpuBaseline.value, metadata: { + tags: currentMetric.metadata.tags, improvement_direction: 'up' } }); - } - // Compute normalized_throughput of benchmarks against ping bench - else { + } else { + // Compute normalized_throughput of benchmarks against ping bench bench.metrics.push({ name: 'normalized_throughput', - value: bench.metrics[0].value / pingThroughput, + value: currentMetric.value / pingThroughput.value, metadata: { + tags: currentMetric.metadata.tags, improvement_direction: 'up' } }); From e8f0e433b972b882c96bc6b09ea5484bc412e5b3 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 19 Mar 2025 17:19:28 -0400 Subject: [PATCH 2/6] naming --- test/benchmarks/driver_bench/src/main.mts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index f032f5100d7..90e58e584c7 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -110,12 +110,12 @@ for (const [suite, benchmarks] of Object.entries(tests)) { console.groupEnd(); } -const aMetricInfo = +const metricInfoFilterByName = (testName: string) => ({ info: { test_name } }: MetricInfo) => test_name === testName; -const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; +const isMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; function calculateCompositeBenchmarks(results: MetricInfo[]) { const composites = { @@ -162,10 +162,10 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { let sum = 0; for (const testName of compositeTests) { - const testScore = results.find(aMetricInfo(testName)); + const testScore = results.find(metricInfoFilterByName(testName)); assert.ok(testScore, `${compositeName} suite requires ${testName} for composite score`); - const metric = testScore.metrics.find(anMBsMetric); + const metric = testScore.metrics.find(isMBsMetric); assert.ok(metric, `${testName} is missing a megabytes_per_second metric`); sum += metric.value; @@ -199,14 +199,14 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { } function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { - const baselineBench = results.find(aMetricInfo('cpuBaseline')); - const pingBench = results.find(aMetricInfo('ping')); + const baselineBench = results.find(metricInfoFilterByName('cpuBaseline')); + const pingBench = results.find(metricInfoFilterByName('ping')); assert.ok(pingBench, 'ping bench results not found!'); assert.ok(baselineBench, 'cpuBaseline results not found!'); - const cpuBaseline = baselineBench.metrics.find(anMBsMetric); - const pingThroughput = pingBench.metrics.find(anMBsMetric); + const cpuBaseline = baselineBench.metrics.find(isMBsMetric); + const pingThroughput = pingBench.metrics.find(isMBsMetric); assert.ok(cpuBaseline, 'cpu benchmark does not have a MB/s metric'); assert.ok(pingThroughput, 'ping does not have a MB/s metric'); @@ -214,7 +214,7 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { for (const bench of results) { if (bench.info.test_name === 'cpuBaseline') continue; - const currentMetric = bench.metrics.find(anMBsMetric); + const currentMetric = bench.metrics.find(isMBsMetric); assert.ok(currentMetric, `${bench.info.test_name} does not have a MB/s metric`); if (bench.info.test_name === 'ping') { From 3a3f061f04877d7f20c3c1e766ad8c2346c9dc22 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 20 Mar 2025 10:26:28 -0400 Subject: [PATCH 3/6] fix: logging results preview --- .evergreen/perf-send.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/perf-send.sh b/.evergreen/perf-send.sh index a3c7f234b82..8e52855af48 100644 --- a/.evergreen/perf-send.sh +++ b/.evergreen/perf-send.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash -set -euox pipefail +set -euo pipefail source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh TARGET_FILE=$(realpath "${TARGET_FILE:-./test/benchmarks/driver_bench/results.json}") +set -o xtrace + node ./.evergreen/perf_send.mjs $TARGET_FILE From e8ed43a8ed385c5b3e13bde09e350f553a2c504f Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 20 Mar 2025 10:59:11 -0400 Subject: [PATCH 4/6] upload the json --- .evergreen/config.in.yml | 4 ++++ .evergreen/config.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 1a3adee09d2..d3a8022e966 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -109,6 +109,10 @@ functions: - .evergreen/run-tests.sh "perf send": + - command: attach.artifacts + params: + files: + - ./test/benchmarks/driver_bench/results.json - command: subprocess.exec params: working_dir: src diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d9f3b3a1f73..d533d2699cd 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -81,6 +81,10 @@ functions: args: - .evergreen/run-tests.sh perf send: + - command: attach.artifacts + params: + files: + - ./test/benchmarks/driver_bench/results.json - command: subprocess.exec params: working_dir: src From d68cfd5002e9255b65f3300055585f4101cf02f3 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 20 Mar 2025 13:02:05 -0400 Subject: [PATCH 5/6] set working_dir --- .evergreen/config.in.yml | 1 + .evergreen/config.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index d3a8022e966..431f6b4f191 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -111,6 +111,7 @@ functions: "perf send": - command: attach.artifacts params: + working_dir: src files: - ./test/benchmarks/driver_bench/results.json - command: subprocess.exec diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d533d2699cd..b1f7e6e7f06 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -83,6 +83,7 @@ functions: perf send: - command: attach.artifacts params: + working_dir: src files: - ./test/benchmarks/driver_bench/results.json - command: subprocess.exec From 91161997753b33d9666f347d0d369bb6d9dd6911 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 20 Mar 2025 14:07:24 -0400 Subject: [PATCH 6/6] chore: use s3 --- .evergreen/config.in.yml | 15 +++++++++++---- .evergreen/config.yml | 14 ++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 431f6b4f191..750ef3d81a1 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -109,11 +109,18 @@ functions: - .evergreen/run-tests.sh "perf send": - - command: attach.artifacts + - command: s3.put params: - working_dir: src - files: - - ./test/benchmarks/driver_bench/results.json + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/test/benchmarks/driver_bench/results.json + optional: true + # TODO NODE-4707 - change upload directory to ${UPLOAD_BUCKET} + remote_file: mongo-node-driver/${revision}/${version_id}/results.${task_name}.json + bucket: mciuploads + permissions: public-read + content_type: application/json + display_name: "Performance Results" - command: subprocess.exec params: working_dir: src diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b1f7e6e7f06..f74052f2a7b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -81,11 +81,17 @@ functions: args: - .evergreen/run-tests.sh perf send: - - command: attach.artifacts + - command: s3.put params: - working_dir: src - files: - - ./test/benchmarks/driver_bench/results.json + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/test/benchmarks/driver_bench/results.json + optional: true + remote_file: mongo-node-driver/${revision}/${version_id}/results.${task_name}.json + bucket: mciuploads + permissions: public-read + content_type: application/json + display_name: Performance Results - command: subprocess.exec params: working_dir: src