Skip to content

Commit 2a0b12e

Browse files
tenzen-yandreyvelich
tenzen-y
and
andreyvelich
authored
Bump Python version to 3.9 (#1731)
* bump Python to 3.9 * modify script to build container image * fix example for enas * update scripts to modify image name in ci * review: change docker build command * review: use new tf-mnist-with-example in Ci for tfjob * review: refactor tf-mnist-with-summaries * review: remove Dockerfile.ppc64le for new-ui * review: update docs related tf-mnist-with-summaries * TFEventMetricsCollector supports TF>=2.0 and stop supporting TF <=1.x * review: add help command to scripts/v1beta1/build.sh Co-authored-by: Andrey Velichkevich <[email protected]> * fix unit test for tfevent-metricscollector * review: generate tf event files on CI * add test command to Makefile * update publish-trial-images * update update-images.sh * reduce batch size Co-authored-by: Andrey Velichkevich <[email protected]>
1 parent 7be1f0a commit 2a0b12e

File tree

48 files changed

+610
-339
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+610
-339
lines changed

‎.github/workflows/publish-trial-images.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ jobs:
3232
dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
3333
- trial-name: pytorch-mnist
3434
dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile
35+
- trial-name: tf-mnist-with-summaries
36+
dockerfile: examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
3537
- trial-name: enas-cnn-cifar10-gpu
3638
dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
3739
- trial-name: enas-cnn-cifar10-cpu

‎.github/workflows/test-python.yaml

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,7 @@ jobs:
1616
- name: Setup Python
1717
uses: actions/setup-python@v2
1818
with:
19-
python-version: 3.7
20-
21-
- name: Install Packages
22-
run: |
23-
pip install -r test/unit/v1beta1/requirements.txt
24-
25-
pip install -r cmd/suggestion/chocolate/v1beta1/requirements.txt
26-
pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
27-
pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
28-
pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
29-
pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
30-
pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
31-
pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
32-
33-
pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
19+
python-version: 3.9
3420

3521
- name: Run Python test
36-
run: |
37-
export PYTHONPATH=$(pwd):$(pwd)/pkg/apis/manager/v1beta1/python:$(pwd)/pkg/apis/manager/health/python
38-
pytest ./test/unit/v1beta1/suggestion
39-
pytest ./test/unit/v1beta1/earlystopping
22+
run: make pytest

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ __pycache__/
66
*.egg-info
77
build/
88
*.charm
9+
test/unit/v1beta1/metricscollector/testdata
910

1011
# SDK generator JAR file
1112
hack/gen-python-sdk/openapi-generator-cli.jar

‎Makefile

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
HAS_LINT := $(shell command -v golangci-lint;)
22
COMMIT := v1beta1-$(shell git rev-parse --short=7 HEAD)
33
KATIB_REGISTRY := docker.io/kubeflowkatib
4+
CPU_ARCH ?= amd64
5+
6+
# for pytest
7+
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/apis/manager/v1beta1/python:$(CURDIR)/pkg/apis/manager/health/python
8+
PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/metricscollector/v1beta1/common:$(CURDIR)/pkg/metricscollector/v1beta1/tfevent-metricscollector
9+
TEST_TENSORFLOW_EVENT_FILE_PATH ?= $(CURDIR)/test/unit/v1beta1/metricscollector/testdata/tfevent-metricscollector/logs
410

511
# Run tests
612
.PHONY: test
@@ -49,10 +55,10 @@ endif
4955

5056
# Build images for the Katib v1beta1 components.
5157
build: generate
52-
ifeq ($(and $(REGISTRY),$(TAG)),)
53-
$(error REGISTRY and TAG must be set. Usage: make build REGISTRY=<registry> TAG=<tag>)
58+
ifeq ($(and $(REGISTRY),$(TAG),$(CPU_ARCH)),)
59+
$(error REGISTRY and TAG must be set. Usage: make build REGISTRY=<registry> TAG=<tag> CPU_ARCH=<cpu-architecture>)
5460
endif
55-
bash scripts/v1beta1/build.sh $(REGISTRY) $(TAG)
61+
bash scripts/v1beta1/build.sh $(REGISTRY) $(TAG) $(CPU_ARCH)
5662

5763
# Build and push Katib images from the latest master commit.
5864
push-latest: generate
@@ -94,3 +100,25 @@ prettier-check:
94100
# Update boilerplate for the source code.
95101
update-boilerplate:
96102
./hack/boilerplate/update-boilerplate.sh
103+
104+
prepare-pytest:
105+
pip install -r test/unit/v1beta1/requirements.txt
106+
pip install -r cmd/suggestion/chocolate/v1beta1/requirements.txt
107+
pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
108+
pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
109+
pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
110+
pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
111+
pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
112+
pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
113+
pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
114+
pip install -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
115+
116+
prepare-pytest-testdata:
117+
ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
118+
python examples/v1beta1/trial-images/tf-mnist-with-summaries/mnist.py --epochs 5 --batch-size 200 --log-path $(TEST_TENSORFLOW_EVENT_FILE_PATH)
119+
endif
120+
121+
pytest: prepare-pytest prepare-pytest-testdata
122+
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion
123+
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/earlystopping
124+
PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/metricscollector

‎cmd/earlystopping/medianstop/v1beta1/Dockerfile

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1-
FROM python:3.6
1+
FROM python:3.9
22

33
ENV TARGET_DIR /opt/katib
44
ENV EARLY_STOPPING_DIR cmd/earlystopping/medianstop/v1beta1
55

66
RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
7-
apt-get -y update && \
8-
apt-get -y install gfortran libopenblas-dev liblapack-dev && \
9-
pip install cython; \
7+
apt-get -y update && \
8+
apt-get -y install gfortran libopenblas-dev liblapack-dev && \
9+
apt-get clean && \
10+
rm -rf /var/lib/apt/lists/*; \
1011
fi
1112

1213
ADD ./pkg/ ${TARGET_DIR}/pkg/

‎cmd/earlystopping/medianstop/v1beta1/requirements.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
grpcio==1.23.0
2-
protobuf==3.9.1
1+
grpcio==1.41.1
2+
protobuf==3.19.1
33
googleapis-common-protos==1.6.0
44
kubernetes==11.0.0
5+
cython>=0.29.24

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,25 @@
1-
FROM tensorflow/tensorflow:1.11.0
2-
RUN pip install rfc3339 grpcio googleapis-common-protos
3-
ADD . /usr/src/app/github.com/kubeflow/katib
4-
WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
1+
FROM python:3.9
2+
3+
ENV TARGET_DIR /opt/katib
4+
ENV METRICS_COLLECTOR_DIR cmd/metricscollector/v1beta1/tfevent-metricscollector
5+
# tensorflow community build for aarch64
6+
# https://github.com/tensorflow/build#tensorflow-builds
7+
ENV PIP_EXTRA_INDEX_URL https://snapshots.linaro.org/ldcg/python-cache/
8+
9+
ADD ./pkg/ ${TARGET_DIR}/pkg/
10+
ADD ./${METRICS_COLLECTOR_DIR}/ ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}/
11+
WORKDIR ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}
12+
13+
RUN if [ "$(uname -m)" = "aarch64" ]; then \
14+
pip install tensorflow-aarch64==2.7.0; \
15+
else \
16+
pip install tensorflow==2.7.0; \
17+
fi;
518
RUN pip install --no-cache-dir -r requirements.txt
6-
ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/apis/manager/v1beta1/python:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/tfevent-metricscollector/:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/common/
19+
20+
RUN chgrp -R 0 ${TARGET_DIR} \
21+
&& chmod -R g+rwX ${TARGET_DIR}
22+
23+
ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/metricscollector/v1beta1/tfevent-metricscollector/::${TARGET_DIR}/pkg/metricscollector/v1beta1/common/
24+
725
ENTRYPOINT ["python", "main.py"]

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.aarch64

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
FROM ibmcom/tensorflow-ppc64le:1.14.0-py3
2-
RUN pip install rfc3339 grpcio googleapis-common-protos
1+
FROM ibmcom/tensorflow-ppc64le:2.2.0-py3
32
ADD . /usr/src/app/github.com/kubeflow/katib
43
WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
54
RUN pip install --no-cache-dir -r requirements.txt

‎cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
1-
psutil==5.6.6
1+
psutil==5.8.0
2+
rfc3339>=6.2
3+
grpcio==1.41.1
4+
googleapis-common-protos==1.6.0

0 commit comments

Comments
 (0)