Skip to content

add ability to use benchmark.ini file to customize redline tests #887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

add ability to use benchmark.ini file to customize redline tests #887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
48c9dee
add ability to use benchmark.ini file to customize redline tests
OVI3D0 Jun 17, 2025
6fce991
add default values for config obj
OVI3D0 Jul 7, 2025
1fd3aac
fix lint errors
OVI3D0 Jul 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions osbenchmark/benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
from osbenchmark.workload_generator import workload_generator
from osbenchmark.utils import io, convert, process, console, net, opts, versions
from osbenchmark import aggregator
from osbenchmark.worker_coordinator.worker_coordinator import ConfigureFeedbackScaling

def create_arg_parser():
def positive_number(v):
Expand Down Expand Up @@ -672,19 +671,19 @@ def add_workload_source(subparser):
"--redline-scale-step",
type=int,
help="How many clients to add while scaling up during redline testing (default: 5).",
default=ConfigureFeedbackScaling.DEFAULT_SCALE_STEP
default=None
)
test_execution_parser.add_argument(
"--redline-scaledown-percentage",
type=float,
help="What percentage of clients to remove when errors occur (default: 10%%).",
default=ConfigureFeedbackScaling.DEFAULT_SCALEDOWN_PCT
default=None
)
test_execution_parser.add_argument(
"--redline-post-scaledown-sleep",
type=int,
help="How many seconds to wait before scaling up again after a scale down (default: 30).",
default=ConfigureFeedbackScaling.DEFAULT_SLEEP_SECONDS
default=None
)
test_execution_parser.add_argument(
"--redline-max-clients",
Expand All @@ -702,13 +701,13 @@ def add_workload_source(subparser):
"--redline-cpu-window-seconds",
type=int,
help="How many seconds the window for average CPU load should be in seconds during CPU-based redline testing. (Default: 30)",
default=ConfigureFeedbackScaling.DEFAULT_CPU_WINDOW_SECONDS
default=None
)
test_execution_parser.add_argument(
"--redline-cpu-check-interval",
type=int,
help="How many seconds between CPU checks there should be during CPU-based redline testing. (Default: 30)",
default=ConfigureFeedbackScaling.DEFAULT_CPU_CHECK_INTERVAL
default=None
)

###############################################################################
Expand Down
6 changes: 6 additions & 0 deletions osbenchmark/resources/benchmark.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,9 @@ preserve_benchmark_candidate = false
[distributions]
release.cache = true

[redline]
scale_step = 5
scaledown_percentage = 0.10
post_scaledown_sleep = 30
cpu_window_seconds = 30
cpu_check_interval = 30
53 changes: 43 additions & 10 deletions osbenchmark/worker_coordinator/worker_coordinator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@
import asyncio
import collections
import concurrent.futures
import configparser
import datetime
import itertools
import json
import logging
import math
import multiprocessing
import os
import queue
import random
import sys
Expand Down Expand Up @@ -204,6 +206,33 @@ def __init__(self, metrics, next_task_scheduled_in):
self.metrics = metrics
self.next_task_scheduled_in = next_task_scheduled_in

def load_redline_config():
config = configparser.ConfigParser()
benchmark_home = os.environ.get('BENCHMARK_HOME') or os.environ['HOME']
benchmark_ini = benchmark_home + '/.benchmark/benchmark.ini'
if not os.path.isfile(benchmark_ini):
print(f"WARNING: redline config file {benchmark_ini} not found. Proceeding with default values.")
return {}

config.read(benchmark_ini)
config_object = {}

if "redline" in config:
redline = config["redline"]
for key in [
"scale_step",
"scaledown_percentage",
"post_scaledown_sleep",
"max_cpu_usage",
"cpu_window_seconds",
"cpu_check_interval",
"max_clients"
]:
if key in redline:
config_object[key] = redline[key]

return config_object

class ConfigureFeedbackScaling:
DEFAULT_SLEEP_SECONDS = 30
DEFAULT_SCALE_STEP = 5
Expand All @@ -213,11 +242,15 @@ class ConfigureFeedbackScaling:

def __init__(self, scale_step=None, scale_down_pct=None, sleep_seconds=None, max_clients=None, cpu_max=None,
cpu_window_seconds=None, cpu_check_interval=None, metrics_index=None, test_execution_id=None, cfg=None):
self.scale_step = scale_step if scale_step is not None else self.DEFAULT_SCALE_STEP
self.scale_down_pct = scale_down_pct if scale_down_pct is not None else self.DEFAULT_SCALEDOWN_PCT
self.sleep_seconds = sleep_seconds if sleep_seconds is not None else self.DEFAULT_SLEEP_SECONDS
self.cpu_window_seconds = cpu_window_seconds if cpu_window_seconds is not None else self.DEFAULT_CPU_WINDOW_SECONDS
self.cpu_check_interval = cpu_check_interval if cpu_check_interval is not None else self.DEFAULT_CPU_CHECK_INTERVAL

config_object = load_redline_config()

# priority: command flags -> config object -> default values
self.scale_step = int(scale_step if scale_step is not None else config_object.get("scale_step", self.DEFAULT_SCALE_STEP))
self.scale_down_pct = float(scale_down_pct if scale_down_pct is not None else config_object.get("scaledown_percentage", self.DEFAULT_SCALEDOWN_PCT))
self.sleep_seconds = int(sleep_seconds if sleep_seconds is not None else config_object.get("post_scaledown_sleep", self.DEFAULT_SLEEP_SECONDS))
self.cpu_window_seconds = int(cpu_window_seconds if cpu_window_seconds is not None else config_object.get("cpu_window_seconds", self.DEFAULT_CPU_WINDOW_SECONDS))
self.cpu_check_interval = int(cpu_check_interval if cpu_check_interval is not None else config_object.get("cpu_check_interval", self.DEFAULT_CPU_CHECK_INTERVAL))
self.max_clients = max_clients
self.cpu_max=cpu_max
self.cfg=cfg
Expand Down Expand Up @@ -1110,11 +1143,11 @@ def start_benchmark(self):
metrics_index = self.metrics_store.index
test_execution_id = self.metrics_store.test_execution_id

scale_step = self.config.opts("workload", "redline.scale_step", default_value=5)
scale_down_pct = self.config.opts("workload", "redline.scale_down_pct", default_value=0.10)
sleep_seconds = self.config.opts("workload", "redline.sleep_seconds", default_value=30)
cpu_window_seconds = self.config.opts("workload", "redline.cpu_window_seconds", default_value=30)
cpu_check_interval = self.config.opts("workload", "redline.cpu_check_interval", default_value=30)
scale_step = self.config.opts("workload", "redline.scale_step", default_value=0)
scale_down_pct = self.config.opts("workload", "redline.scale_down_pct", default_value=0)
sleep_seconds = self.config.opts("workload", "redline.sleep_seconds", default_value=0)
cpu_window_seconds = self.config.opts("workload", "redline.cpu_window_seconds", default_value=0)
cpu_check_interval = self.config.opts("workload", "redline.cpu_check_interval", default_value=0)

self.target.send(self.target.feedback_actor, ConfigureFeedbackScaling(
scale_step=scale_step,
Expand Down
Loading