Skip to content

Commit 9e34003

Browse files
jlevequejleveque
authored
[sonic-config-engine] Clean up dependencies, pin versions; install Python 3 package in Buster container (#5656)
To clean up the image build procedure, and let setuptools/pip[3] implicitly install Python dependencies. Also use ipaddress package instead of ipaddr.
1 parent 7bee509 commit 9e34003

File tree

15 files changed

+202
-154
lines changed

15 files changed

+202
-154
lines changed

‎dockers/docker-config-engine-buster/Dockerfile.j2

+17-13
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@ ENV DEBIAN_FRONTEND=noninteractive
66

77
RUN apt-get update && \
88
apt-get install -y \
9-
# Dependencies for sonic-cfggen
10-
python-lxml \
11-
python-yaml \
12-
python-bitarray \
9+
build-essential \
1310
python-pip \
11+
python3-pip \
1412
python-dev \
13+
python3-dev \
1514
apt-utils \
16-
python-setuptools
15+
python-setuptools \
16+
python3-setuptools
1717

18-
RUN pip install --upgrade pip
18+
RUN pip2 install --upgrade pip
19+
RUN pip3 install --upgrade pip
20+
RUN apt-get purge -y python-pip python3-pip
1921

20-
RUN pip install \
21-
netaddr \
22-
ipaddr \
23-
jinja2 \
24-
pyangbind==0.6.0
22+
# For sonic-config-engine Python 3 package
23+
# Explicitly install pyangbind here, as pyangbind causes enum34 to be installed.
24+
# enum34 causes Python 're' package to not work properly as it redefines an incompatible enum.py module
25+
# https://github.com/robshakir/pyangbind/issues/232
26+
RUN pip3 install pyangbind==0.8.1
27+
RUN pip3 uninstall -y enum34
2528

2629
{% if docker_config_engine_buster_debs.strip() %}
2730
# Copy locally-built Debian package dependencies
@@ -44,8 +47,9 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
4447

4548
## Clean up
4649
RUN apt-get purge -y \
47-
python-pip \
48-
python-dev && \
50+
python-dev \
51+
python3-dev \
52+
build-essential && \
4953
apt-get clean -y && \
5054
apt-get autoclean -y && \
5155
apt-get autoremove -y && \

‎dockers/docker-config-engine-stretch/Dockerfile.j2

+2-9
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,13 @@ ENV DEBIAN_FRONTEND=noninteractive
77
RUN apt-get update && \
88
apt-get install -y \
99
# Dependencies for sonic-cfggen
10-
python-lxml \
11-
python-yaml \
12-
python-bitarray \
10+
build-essential \
1311
python-pip \
1412
python-dev \
1513
python-setuptools
1614

1715
RUN pip install --upgrade pip
1816

19-
RUN pip install \
20-
netaddr \
21-
ipaddr \
22-
jinja2 \
23-
pyangbind==0.5.10
24-
2517
{% if docker_config_engine_stretch_debs.strip() %}
2618
# Copy locally-built Debian package dependencies
2719
{{ copy_files("debs/", docker_config_engine_stretch_debs.split(' '), "/debs/") }}
@@ -43,6 +35,7 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
4335

4436
## Clean up
4537
RUN apt-get purge -y \
38+
build-essential \
4639
python-pip \
4740
python-dev && \
4841
apt-get clean -y && \

‎dockers/docker-config-engine/Dockerfile.j2

+3-3
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ ENV DEBIAN_FRONTEND=noninteractive
66
RUN apt-get update
77

88
# Dependencies for sonic-cfggen
9-
RUN apt-get install -y python-lxml python-yaml python-bitarray python-pip python-dev
9+
RUN apt-get install -y build-essential python-pip python-dev
1010

1111
RUN pip install --upgrade pip
1212

13-
RUN pip install netaddr ipaddr jinja2 pyangbind==0.5.10
13+
RUN apt-get purge -y python-pip
1414

1515
{% if docker_config_engine_debs.strip() %}
1616
COPY \
@@ -46,5 +46,5 @@ python-wheels/{{ whl }}{{' '}}
4646
COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
4747

4848
## Clean up
49-
RUN apt-get purge -y python-pip python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
49+
RUN apt-get purge -y build-essential python-dev; apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
5050
RUN rm -rf /debs /python-wheels

‎dockers/docker-orchagent/Dockerfile.j2

+16-8
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ FROM docker-config-engine-buster
44
ARG docker_container_name
55
RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
66

7-
## Make apt-get non-interactive
7+
# Make apt-get non-interactive
88
ENV DEBIAN_FRONTEND=noninteractive
99

1010
RUN apt-get update && \
@@ -18,10 +18,14 @@ RUN apt-get update && \
1818
libmnl0 \
1919
bridge-utils \
2020
conntrack \
21-
ndppd
21+
ndppd \
22+
# Needed for installing netifaces Python package
23+
build-essential \
24+
python-dev \
25+
python3-dev
2226

2327
{% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %}
24-
## Fix for gcc/python not found in arm docker
28+
# Fix for gcc/python not found in arm docker
2529
RUN apt-get install -f -y python2.7 python2.7-dev
2630
RUN apt-get install -y gcc-6
2731
{% endif %}
@@ -53,10 +57,14 @@ RUN apt-get remove -y gcc-6
5357
{{ install_debian_packages(docker_orchagent_debs.split(' ')) }}
5458
{%- endif %}
5559

56-
## Clean up
57-
RUN apt-get clean -y && \
58-
apt-get autoclean -y && \
59-
apt-get autoremove -y && \
60+
# Clean up
61+
RUN apt-get purge -y \
62+
build-essential \
63+
python-dev \
64+
python3-dev && \
65+
apt-get clean -y && \
66+
apt-get autoclean -y && \
67+
apt-get autoremove -y && \
6068
rm -rf /debs
6169

6270
COPY ["files/arp_update", "/usr/bin"]
@@ -68,7 +76,7 @@ COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
6876
COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
6977
COPY ["critical_processes", "/etc/supervisor/"]
7078

71-
## Copy all Jinja2 template files into the templates folder
79+
# Copy all Jinja2 template files into the templates folder
7280
COPY ["*.j2", "/usr/share/sonic/templates/"]
7381

7482
ENTRYPOINT ["/usr/bin/docker-init.sh"]

‎dockers/docker-sonic-mgmt-framework/Dockerfile.j2

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s
77
ENV DEBIAN_FRONTEND=noninteractive
88

99
RUN apt-get update && \
10-
apt-get install -y g++ python-dev
10+
apt-get install -y g++ python-dev libxml2
1111

1212
RUN pip install connexion==1.1.15 \
1313
setuptools==21.0.0 \

‎dockers/dockerfile-macros.j2

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
66
{%- endmacro %}
77

88
{% macro install_python2_wheels(packages) -%}
9-
RUN cd /python-wheels/ && pip install {{ packages | join(' ') }}
9+
RUN cd /python-wheels/ && pip2 install {{ packages | join(' ') }}
1010
{%- endmacro %}
1111

1212
{% macro install_python3_wheels(packages) -%}

‎files/build_templates/sonic_debian_extension.j2

+1-4
Original file line numberDiff line numberDiff line change
@@ -93,10 +93,7 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/iptables_*.deb || \
9393
# Install dependencies for SONiC config engine
9494
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
9595
python-dev \
96-
python3-dev \
97-
python-lxml \
98-
python-yaml \
99-
python-bitarray
96+
python3-dev
10097

10198
# Install Python client for Redis
10299
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==3.5.3"

‎platform/vs/docker-sonic-vs/Dockerfile.j2

+12-12
Original file line numberDiff line numberDiff line change
@@ -61,20 +61,20 @@ RUN curl -o redis-server_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.co
6161
RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb || apt-get install -f
6262
RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb
6363

64-
RUN pip install setuptools
64+
RUN pip2 install setuptools
6565
RUN pip3 install setuptools
66-
RUN pip install py2_ipaddress
67-
RUN pip install six
68-
RUN pip install pyroute2==0.5.3 netifaces==0.10.7
69-
RUN pip install monotonic==1.5
70-
RUN pip install urllib3
71-
RUN pip install requests
72-
RUN pip install crontab
66+
RUN pip2 install py2_ipaddress
67+
RUN pip2 install six
68+
RUN pip2 install pyroute2==0.5.3 netifaces==0.10.7
69+
RUN pip2 install monotonic==1.5
70+
RUN pip2 install urllib3
71+
RUN pip2 install requests
72+
RUN pip2 install crontab
7373

7474
# Install dependencies for Dynamic Port Breakout
75-
RUN pip install xmltodict==0.12.0
76-
RUN pip install jsondiff==1.2.0
77-
RUN pip install ijson==2.6.1
75+
RUN pip2 install xmltodict==0.12.0
76+
RUN pip2 install jsondiff==1.2.0
77+
RUN pip2 install ijson==2.6.1
7878

7979
{% if docker_sonic_vs_debs.strip() -%}
8080
# Copy locally-built Debian package dependencies
@@ -109,7 +109,7 @@ COPY python-wheels/{{ whl }} python-wheels/
109109
# install PKGs after copying all PKGs to avoid dependency failure
110110
# use py3 to find python3 package, which is forced by wheel as of now
111111
{%- for whl in docker_sonic_vs_whls.split(' ') %}
112-
RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
112+
RUN pip{% if 'py3' in whl %}3{% else %}2{% endif %} install python-wheels/{{ whl }}
113113
{%- endfor %}
114114
{% endif %}
115115

‎rules/docker-config-engine-buster.mk

+3
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz
44
$(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster
55

66
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2)
7+
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3)
78
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
9+
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
810
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
11+
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
912
$(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
1013
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)
1114

‎rules/sonic-config.mk

+3-1
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@ $(SONIC_CONFIG_ENGINE_PY2)_DEPENDS += $(SWSSSDK_PY2) $(SONIC_PY_COMMON_PY2)
66
$(SONIC_CONFIG_ENGINE_PY2)_PYTHON_VERSION = 2
77
SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
88

9-
109
# SONIC_CONFIG_ENGINE_PY3 package
1110

1211
SONIC_CONFIG_ENGINE_PY3 = sonic_config_engine-1.0-py3-none-any.whl
1312
$(SONIC_CONFIG_ENGINE_PY3)_SRC_PATH = $(SRC_PATH)/sonic-config-engine
1413
$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SWSSSDK_PY3) $(SONIC_PY_COMMON_PY3)
14+
# Synthetic dependency to avoid building the Python 2 and 3 packages
15+
# simultaneously and any potential conflicts which may arise
16+
$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY2)
1517
$(SONIC_CONFIG_ENGINE_PY3)_PYTHON_VERSION = 3
1618
SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)

‎sonic-slave-buster/Dockerfile.j2

+36-28
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ RUN apt-get update && apt-get install -y \
5757
libtinyxml2-dev \
5858
python \
5959
python-pip \
60+
python3-pip \
6061
libncurses5-dev \
6162
texinfo \
6263
dh-autoreconf \
63-
python3-pip \
6464
doxygen \
6565
devscripts \
6666
git-buildpackage \
@@ -245,13 +245,6 @@ RUN apt-get update && apt-get install -y \
245245
python3-sphinx \
246246
# For sonic config engine testing
247247
python-dev \
248-
python-lxml \
249-
python3-lxml \
250-
python-jinja2 \
251-
python-netaddr \
252-
python-ipaddr \
253-
python-yaml \
254-
python3-yaml \
255248
# For lockfile
256249
procmail \
257250
# For gtest
@@ -305,7 +298,14 @@ RUN apt-get update && apt-get install -y \
305298
# For SWI Tools
306299
python-m2crypto \
307300
# For build dtb
308-
device-tree-compiler
301+
device-tree-compiler \
302+
# For sonic-mgmt-framework
303+
autoconf \
304+
m4 \
305+
libxml2-utils \
306+
xsltproc \
307+
python-lxml \
308+
libexpat1-dev
309309

310310
## Config dpkg
311311
## install the configuration file if it’s currently missing
@@ -334,29 +334,33 @@ RUN export VERSION=1.14.2 \
334334
&& echo 'export PATH=$PATH:$GOROOT/bin' >> /etc/bash.bashrc \
335335
&& rm go$VERSION.linux-*.tar.gz
336336

337+
RUN pip2 install --upgrade pip
338+
RUN pip3 install --upgrade pip
339+
RUN apt-get purge -y python-pip python3-pip
340+
337341
# For building Python packages
338-
RUN pip install setuptools==40.8.0
342+
RUN pip2 install setuptools==40.8.0
339343
RUN pip3 install setuptools==49.6.00
340344

341345
# For building sonic-utilities
342-
RUN pip install fastentrypoints
346+
RUN pip2 install fastentrypoints
343347
RUN pip3 install fastentrypoints
344348

345349
# For running Python unit tests
346-
RUN pip install pytest-runner==4.4
350+
RUN pip2 install pytest-runner==4.4
347351
RUN pip3 install pytest-runner==5.2
348-
RUN pip install mockredispy==2.9.3
352+
RUN pip2 install mockredispy==2.9.3
349353
RUN pip3 install mockredispy==2.9.3
350354

351355
# For Python 2 unit tests, we need 'mock'. The last version of 'mock'
352356
# which supports Python 2 is 3.0.5. In Python 3, 'mock' is part of 'unittest'
353357
# in the standard library
354-
RUN pip install mock==3.0.5
358+
RUN pip2 install mock==3.0.5
355359

356360
# For p4 build
357-
RUN pip install \
358-
ctypesgen==1.0.2 \
359-
crc16
361+
RUN pip2 install \
362+
ctypesgen==1.0.2 \
363+
crc16
360364

361365
# For sonic config engine testing
362366
# Install pyangbind here, outside sonic-config-engine dependencies, as pyangbind causes enum34 to be installed.
@@ -366,31 +370,35 @@ RUN pip3 install pyangbind==0.8.1
366370
RUN pip3 uninstall -y enum34
367371

368372
# For templating
369-
RUN pip install j2cli==0.3.10
373+
RUN pip2 install j2cli==0.3.10
370374

371-
RUN pip3 install "PyYAML>=5.1"
375+
# For sonic-mgmt-framework
376+
RUN pip2 install "PyYAML==5.3.1"
377+
RUN pip3 install "PyYAML==5.3.1"
378+
RUN pip2 install "lxml==4.6.1"
379+
RUN pip3 install "lxml==4.6.1"
372380

373381
# For sonic-platform-common testing
374382
RUN pip3 install redis
375383

376384
# For supervisor build
377-
RUN pip install meld3
385+
RUN apt-get install python-meld3
378386

379387
# For vs image build
380-
RUN pip install pexpect==4.6.0
388+
RUN pip2 install pexpect==4.6.0
381389

382390
# For sonic-swss-common testing
383-
RUN pip install Pympler==0.8
391+
RUN pip2 install Pympler==0.8
384392

385393
# For sonic_yang_mgmt build
386-
RUN pip install ijson==2.6.1
394+
RUN pip2 install ijson==2.6.1
387395
RUN pip3 install ijson==2.6.1
388-
RUN pip install jsondiff==1.2.0
389-
RUN pip install xmltodict==0.12.0
390-
RUN pip install pyang==2.1.1
396+
RUN pip2 install jsondiff==1.2.0
397+
RUN pip2 install xmltodict==0.12.0
398+
RUN pip2 install pyang==2.1.1
391399

392400
# For mgmt-framework build
393-
RUN pip install mmh3
401+
RUN pip2 install mmh3
394402
RUN apt-get install -y xsltproc
395403

396404
# Install dependencies for isc-dhcp-relay build
@@ -434,4 +442,4 @@ RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs\"" >> /etc/default/d
434442
RUN update-alternatives --set iptables /usr/sbin/iptables-legacy
435443

436444
# Install swi tools
437-
RUN python -m pip install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00
445+
RUN pip2 install git+https://github.com/aristanetworks/swi-tools.git@d51761ec0bb93c73039233f3c01ed48235ffad00

0 commit comments

Comments
 (0)