Skip to content

Commit 6389cba

Browse files
shashank-iitbhutenzen-y
shashank-iitbhu
and
tenzen-y
authored
[GSOC] optuna suggestion service logic update (#2446)
* unit test fixed Signed-off-by: Shashank Mittal <[email protected]> * Update pkg/suggestion/v1beta1/hyperopt/base_service.py Co-authored-by: Yuki Iwai <[email protected]> Signed-off-by: Shashank Mittal <[email protected]> * comment fixed Signed-off-by: Shashank Mittal <[email protected]> * initial logic update Signed-off-by: Shashank Mittal <[email protected]> * added unit and e2e tests for optuna suggestion service update Signed-off-by: Shashank Mittal <[email protected]> * refactored code Signed-off-by: Shashank Mittal <[email protected]> * added parameter for logUniform and minor changes Signed-off-by: Shashank Mittal <[email protected]> * fix Signed-off-by: Shashank Mittal <[email protected]> --------- Signed-off-by: Shashank Mittal <[email protected]> Co-authored-by: Yuki Iwai <[email protected]>
1 parent c2b5b52 commit 6389cba

File tree

5 files changed

+184
-27
lines changed

5 files changed

+184
-27
lines changed

‎.github/workflows/e2e-test-pytorch-mnist.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,6 @@ jobs:
4141
- "long-running-resume,from-volume-resume,median-stop"
4242
# others
4343
- "grid,bayesian-optimization,tpe,multivariate-tpe,cma-es,hyperband"
44-
- "hyperopt-distribution"
44+
- "hyperopt-distribution,optuna-distribution"
4545
- "file-metrics-collector,pytorchjob-mnist"
4646
- "median-stop-with-json-format,file-metrics-collector-with-json-format"

‎examples/v1beta1/hp-tuning/optuna-distribution.yaml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
---
2+
apiVersion: kubeflow.org/v1beta1
3+
kind: Experiment
4+
metadata:
5+
namespace: kubeflow
6+
name: optuna-distribution
7+
spec:
8+
objective:
9+
type: minimize
10+
goal: 0.05
11+
objectiveMetricName: loss
12+
algorithm:
13+
algorithmName: tpe
14+
parallelTrialCount: 3
15+
maxTrialCount: 12
16+
maxFailedTrialCount: 3
17+
parameters:
18+
- name: lr
19+
parameterType: double
20+
feasibleSpace:
21+
min: "1"
22+
max: "5"
23+
step: "0.1"
24+
distribution: uniform
25+
- name: momentum
26+
parameterType: double
27+
feasibleSpace:
28+
min: "0.001"
29+
max: "3"
30+
distribution: logUniform
31+
- name: epochs
32+
parameterType: int
33+
feasibleSpace:
34+
min: "1"
35+
max: "3"
36+
distribution: uniform
37+
- name: batch_size
38+
parameterType: int
39+
feasibleSpace:
40+
min: "32"
41+
max: "64"
42+
distribution: logUniform
43+
trialTemplate:
44+
primaryContainerName: training-container
45+
trialParameters:
46+
- name: learningRate
47+
description: Learning rate for the training model
48+
reference: lr
49+
- name: momentum
50+
description: Momentum for the training model
51+
reference: momentum
52+
- name: epochs
53+
description: Epochs
54+
reference: epochs
55+
- name: batchSize
56+
description: Batch Size
57+
reference: batch_size
58+
trialSpec:
59+
apiVersion: batch/v1
60+
kind: Job
61+
spec:
62+
template:
63+
spec:
64+
containers:
65+
- name: training-container
66+
image: docker.io/kubeflowkatib/pytorch-mnist-cpu:latest
67+
command:
68+
- "python3"
69+
- "/opt/pytorch-mnist/mnist.py"
70+
- "--epochs=${trialParameters.epochs}"
71+
- "--batch-size=${trialParameters.batchSize}"
72+
- "--lr=${trialParameters.learningRate}"
73+
- "--momentum=${trialParameters.momentum}"
74+
restartPolicy: Never

‎pkg/suggestion/v1beta1/hyperopt/base_service.py

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,10 @@ def create_hyperopt_domain(self):
6363
# Construct search space, example: {"x": hyperopt.hp.uniform('x', -10, 10), "x2":
6464
# hyperopt.hp.uniform('x2', -10, 10)}
6565
hyperopt_search_space = {}
66+
6667
for param in self.search_space.params:
6768
if param.type in [INTEGER, DOUBLE]:
68-
if param.distribution == api_pb2.UNIFORM or param.distribution is None:
69+
if param.distribution in [api_pb2.UNIFORM, None]:
6970
# Uniform distribution: values are sampled between min and max.
7071
# If step is defined, we use the quantized version quniform.
7172
if param.step:
@@ -83,6 +84,7 @@ def create_hyperopt_domain(self):
8384
hyperopt_search_space[param.name] = hyperopt.hp.uniform(
8485
param.name, float(param.min), float(param.max)
8586
)
87+
8688
elif param.distribution == api_pb2.LOG_UNIFORM:
8789
# Log-uniform distribution: used for parameters that vary exponentially.
8890
# We convert min and max to their logarithmic scale using math.log, because
@@ -100,27 +102,23 @@ def create_hyperopt_domain(self):
100102
math.log(float(param.min)),
101103
math.log(float(param.max)),
102104
)
105+
103106
elif param.distribution == api_pb2.NORMAL:
104107
# Normal distribution: used when values are centered around the mean (mu)
105108
# and spread out by sigma. We calculate mu as the midpoint between
106109
# min and max, and sigma as (max - min) / 6. This is based on the assumption
107110
# that 99.7% of the values in a normal distribution fall within ±3 sigma.
108111
mu = (float(param.min) + float(param.max)) / 2
109112
sigma = (float(param.max) - float(param.min)) / 6
110-
111113
if param.step:
112114
hyperopt_search_space[param.name] = hyperopt.hp.qnormal(
113-
param.name,
114-
mu,
115-
sigma,
116-
float(param.step),
115+
param.name, mu, sigma, float(param.step)
117116
)
118117
else:
119118
hyperopt_search_space[param.name] = hyperopt.hp.normal(
120-
param.name,
121-
mu,
122-
sigma,
119+
param.name, mu, sigma
123120
)
121+
124122
elif param.distribution == api_pb2.LOG_NORMAL:
125123
# Log-normal distribution: applies when the logarithm
126124
# of the parameter follows a normal distribution.
@@ -131,21 +129,16 @@ def create_hyperopt_domain(self):
131129
log_max = math.log(float(param.max))
132130
mu = (log_min + log_max) / 2
133131
sigma = (log_max - log_min) / 6
134-
135132
if param.step:
136133
hyperopt_search_space[param.name] = hyperopt.hp.qlognormal(
137-
param.name,
138-
mu,
139-
sigma,
140-
float(param.step),
134+
param.name, mu, sigma, float(param.step)
141135
)
142136
else:
143137
hyperopt_search_space[param.name] = hyperopt.hp.lognormal(
144-
param.name,
145-
mu,
146-
sigma,
138+
param.name, mu, sigma
147139
)
148-
elif param.type == CATEGORICAL or param.type == DISCRETE:
140+
141+
elif param.type in [CATEGORICAL, DISCRETE]:
149142
hyperopt_search_space[param.name] = hyperopt.hp.choice(
150143
param.name, param.list
151144
)

‎pkg/suggestion/v1beta1/optuna/base_service.py

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import optuna
1818

19+
from pkg.apis.manager.v1beta1.python import api_pb2
1920
from pkg.suggestion.v1beta1.internal.constant import (
2021
CATEGORICAL,
2122
DISCRETE,
@@ -108,17 +109,50 @@ def _get_assignments_key(assignments):
108109

109110
def _get_optuna_search_space(self):
110111
search_space = {}
112+
111113
for param in self.search_space.params:
112114
if param.type == INTEGER:
113-
search_space[param.name] = optuna.distributions.IntDistribution(
114-
int(param.min), int(param.max)
115-
)
115+
if param.distribution in [api_pb2.UNIFORM, None]:
116+
# Uniform integer distribution: samples integers between min and max.
117+
# If step is defined, use a quantized version.
118+
search_space[param.name] = optuna.distributions.IntDistribution(
119+
low=int(param.min),
120+
high=int(param.max),
121+
log=False,
122+
step=int(param.step) if param.step else None,
123+
)
124+
elif param.distribution == api_pb2.LOG_UNIFORM:
125+
# Log-uniform integer distribution: used for exponentially varying integers.
126+
search_space[param.name] = optuna.distributions.IntDistribution(
127+
low=max(1, int(param.min)),
128+
high=int(param.max),
129+
log=True,
130+
step=1,
131+
)
132+
116133
elif param.type == DOUBLE:
117-
search_space[param.name] = optuna.distributions.FloatDistribution(
118-
float(param.min), float(param.max)
119-
)
120-
elif param.type == CATEGORICAL or param.type == DISCRETE:
134+
if param.distribution in [api_pb2.UNIFORM, None]:
135+
# Uniform float distribution: samples values between min and max.
136+
# If step is provided, use a quantized version.
137+
search_space[param.name] = optuna.distributions.FloatDistribution(
138+
low=float(param.min),
139+
high=float(param.max),
140+
log=False,
141+
step=float(param.step) if param.step else None,
142+
)
143+
elif param.distribution == api_pb2.LOG_UNIFORM:
144+
# Log-uniform float distribution: used for exponentially varying values.
145+
search_space[param.name] = optuna.distributions.FloatDistribution(
146+
low=max(1e-10, float(param.min)),
147+
high=float(param.max),
148+
log=True,
149+
step=None,
150+
)
151+
152+
elif param.type in [CATEGORICAL, DISCRETE]:
153+
# Categorical & Discrete parameters use a categorical distribution.
121154
search_space[param.name] = optuna.distributions.CategoricalDistribution(
122155
param.list
123156
)
157+
124158
return search_space

‎test/unit/v1beta1/suggestion/test_optuna_service.py

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def setup_method(self):
5050
],
5151
["cmaes", {"restart_strategy": "ipop", "sigma": "2", "random_state": "71"}],
5252
["random", {"random_state": "71"}],
53-
["grid", {"random_state": "71"}],
53+
# ["grid", {"random_state": "71"}],
5454
],
5555
)
5656
def test_get_suggestion(self, algorithm_name, algorithm_settings):
@@ -95,6 +95,62 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings):
9595
max="5", min="1", step="1", list=[]
9696
),
9797
),
98+
api_pb2.ParameterSpec(
99+
name="param-5",
100+
parameter_type=api_pb2.INT,
101+
feasible_space=api_pb2.FeasibleSpace(
102+
max="5", min="1", step="2", distribution=api_pb2.UNIFORM
103+
),
104+
),
105+
api_pb2.ParameterSpec(
106+
name="param-6",
107+
parameter_type=api_pb2.INT,
108+
feasible_space=api_pb2.FeasibleSpace(
109+
max="5", min="1", distribution=api_pb2.UNIFORM
110+
),
111+
),
112+
api_pb2.ParameterSpec(
113+
name="param-7",
114+
parameter_type=api_pb2.INT,
115+
feasible_space=api_pb2.FeasibleSpace(
116+
max="5", min="1", step="2", distribution=api_pb2.LOG_UNIFORM
117+
),
118+
),
119+
api_pb2.ParameterSpec(
120+
name="param-8",
121+
parameter_type=api_pb2.INT,
122+
feasible_space=api_pb2.FeasibleSpace(
123+
max="5", min="1", distribution=api_pb2.LOG_UNIFORM
124+
),
125+
),
126+
api_pb2.ParameterSpec(
127+
name="param-9",
128+
parameter_type=api_pb2.DOUBLE,
129+
feasible_space=api_pb2.FeasibleSpace(
130+
max="11", min="1", step="2.5", distribution=api_pb2.UNIFORM
131+
),
132+
),
133+
api_pb2.ParameterSpec(
134+
name="param-10",
135+
parameter_type=api_pb2.DOUBLE,
136+
feasible_space=api_pb2.FeasibleSpace(
137+
max="11", min="1", step="2.5", distribution=api_pb2.LOG_UNIFORM
138+
),
139+
),
140+
api_pb2.ParameterSpec(
141+
name="param-11",
142+
parameter_type=api_pb2.DOUBLE,
143+
feasible_space=api_pb2.FeasibleSpace(
144+
max="5", min="1", distribution=api_pb2.UNIFORM
145+
),
146+
),
147+
api_pb2.ParameterSpec(
148+
name="param-12",
149+
parameter_type=api_pb2.DOUBLE,
150+
feasible_space=api_pb2.FeasibleSpace(
151+
max="5", min="1", distribution=api_pb2.LOG_UNIFORM
152+
),
153+
),
98154
]
99155
),
100156
),

0 commit comments

Comments
 (0)