Skip to content

Commit f78aa4f

Browse files
renukamanavalanCarl Keene
authored and
Carl Keene
committed
Revert "Revert "[Kubernetes]: The kube server could be used as http-proxy for docker (sonic-net#7469)" (sonic-net#8023)" (sonic-net#8158)
This reverts commit 7236fa9. Restore original PR sonic-net#7469
1 parent b3bbad5 commit f78aa4f

11 files changed

+371
-11
lines changed

build_debian.sh

-1
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,6 @@ then
237237
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${KUBERNETES_VERSION}-00
238238
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${KUBERNETES_VERSION}-00
239239
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${KUBERNETES_VERSION}-00
240-
# kubeadm package auto install kubelet & kubectl
241240
else
242241
echo '[INFO] Skipping Install kubernetes'
243242
fi

files/build_templates/sonic_debian_extension.j2

+23
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-
448448
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchdog==0.10.3
449449

450450
{% if include_kubernetes == "y" %}
451+
# Point to kubelet to /etc/resolv.conf
452+
#
453+
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
454+
451455
# Copy Flannel conf file into sonic-templates
452456
#
453457
sudo cp $BUILD_TEMPLATES/kube_cni.10-flannel.conflist $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
@@ -468,6 +472,25 @@ sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCR
468472
sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py
469473

470474
# Config file used by container mgmt scripts/service
475+
fl="${files_path}/remote_ctr.config.json"
476+
use_k8s_as_http_proxy=$(python3 -c 'import json
477+
with open("'${fl}'", "r") as s:
478+
d=json.load(s);print(d.get("use_k8s_as_http_proxy", ""))
479+
')
480+
if [ "${use_k8s_as_http_proxy}" == "y" ]; then
481+
# create proxy files for docker using private IP which will
482+
# be later directed to k8s master upon config
483+
PROXY_INFO="http://172.16.1.1:3128/"
484+
cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/http_proxy.conf > /dev/null
485+
[Service]
486+
Environment="HTTP_PROXY=${PROXY_INFO}"
487+
EOT
488+
cat <<EOT | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/docker.service.d/https_proxy.conf > /dev/null
489+
[Service]
490+
Environment="HTTPS_PROXY=${PROXY_INFO}"
491+
EOT
492+
fi
493+
471494
sudo cp ${files_path}/remote_ctr.config.json ${FILESYSTEM_ROOT_ETC_SONIC}/
472495

473496
# Remote container management service files

rules/config

+4-3
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ INCLUDE_DHCP_RELAY = y
145145
# TELEMETRY_WRITABLE - Enable write/config operations via the gNMI interface.
146146
# Uncomment to enable:
147147
# TELEMETRY_WRITABLE = y
148+
148149
# INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to
149150
# run as worker node in kubernetes cluster.
150151
INCLUDE_KUBERNETES = n
@@ -157,9 +158,9 @@ INCLUDE_MACSEC = y
157158
# These are Used *only* when INCLUDE_KUBERNETES=y
158159
# NOTE: As a worker node it has to run version compatible to kubernetes master.
159160
#
160-
KUBERNETES_VERSION = 1.18.6
161-
KUBERNETES_CNI_VERSION = 0.8.6
162-
K8s_GCR_IO_PAUSE_VERSION = 3.2
161+
KUBERNETES_VERSION = 1.21.1
162+
KUBERNETES_CNI_VERSION = 0.8.7
163+
K8s_GCR_IO_PAUSE_VERSION = 3.4.1
163164

164165
# SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
165166
# To not use the auto-generated self-signed certificate, the required files to sign the image as below:
+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/usr/bin/env python3
2+
3+
import ipaddress
4+
import os
5+
import re
6+
import socket
7+
import subprocess
8+
import syslog
9+
10+
UNIT_TESTING = 0
11+
12+
# NOTE:
13+
# Unable to use python-iptables as that does not create rules per ip-tables default
14+
# which is nf_tables. So rules added via iptc package will not be listed under
15+
# "sudo iptables -t nat -L -n". But available in kernel. To list, we need to
16+
# use legacy mode as "sudo iptables-legacy -t nat -L -n".
17+
# As we can't use two modes and using non-default could make any debugging effort
18+
# very tough.
19+
20+
21+
from urllib.parse import urlparse
22+
23+
DST_FILE = "/etc/systemd/system/docker.service.d/http_proxy.conf"
24+
DST_IP = None
25+
DST_PORT = None
26+
SQUID_PORT = "3128"
27+
28+
def _get_ip(ip_str):
29+
ret = ""
30+
if ip_str:
31+
try:
32+
ipaddress.ip_address(ip_str)
33+
ret = ip_str
34+
except ValueError:
35+
pass
36+
37+
if not ret:
38+
try:
39+
ret = socket.gethostbyname(ip_str)
40+
except (OSError, socket.error):
41+
pass
42+
if not ret:
43+
syslog.syslog(syslog.LOG_ERR, "{} is neither IP nor resolves to IP".
44+
format(ip_str))
45+
return ret
46+
47+
48+
def _get_dst_info():
49+
global DST_IP, DST_PORT
50+
DST_IP = None
51+
DST_PORT = None
52+
print("DST_FILE={}".format(DST_FILE))
53+
if os.path.exists(DST_FILE):
54+
with open(DST_FILE, "r") as s:
55+
for line in s.readlines():
56+
url_match = re.search('^Environment=.HTTP_PROXY=(.+?)"', line)
57+
if url_match:
58+
url = urlparse(url_match.group(1))
59+
DST_IP = _get_ip(url.hostname)
60+
DST_PORT = url.port
61+
break
62+
else:
63+
print("{} not available".format(DST_FILE))
64+
print("DST_IP={}".format(DST_IP))
65+
66+
67+
def _is_rule_match(rule):
68+
expect = "DNAT tcp -- 0.0.0.0/0 {} tcp dpt:{} to:".format(
69+
DST_IP, DST_PORT)
70+
71+
# Remove duplicate spaces
72+
rule = " ".join(rule.split()).strip()
73+
74+
if rule.startswith(expect):
75+
return rule[len(expect):]
76+
else:
77+
return ""
78+
79+
80+
def check_proc(proc):
81+
if proc.returncode:
82+
syslog.syslog(syslog.LOG_ERR, "Failed to run: cmd: {}".format(proc.args))
83+
syslog.syslog(syslog.LOG_ERR, "Failed to run: stdout: {}".format(proc.stdout))
84+
syslog.syslog(syslog.LOG_ERR, "Failed to run: stderr: {}".format(proc.stderr))
85+
if not UNIT_TESTING:
86+
assert False
87+
88+
89+
def iptable_proxy_rule_upd(ip_str, port = SQUID_PORT):
90+
_get_dst_info()
91+
if not DST_IP:
92+
# There is no proxy in use. Bail out.
93+
return ""
94+
95+
destination = ""
96+
if ip_str:
97+
upd_ip = _get_ip(ip_str)
98+
if not upd_ip:
99+
return ""
100+
destination = "{}:{}".format(upd_ip, port)
101+
102+
found = False
103+
num = 0
104+
105+
while True:
106+
num += 1
107+
108+
cmd = "sudo iptables -t nat -n -L OUTPUT {}".format(num)
109+
proc = subprocess.run(cmd, shell=True, capture_output=True)
110+
check_proc(proc)
111+
112+
if not proc.stdout:
113+
# No more rule
114+
break
115+
116+
rule_dest = _is_rule_match(proc.stdout.decode("utf-8").strip())
117+
if rule_dest:
118+
if not found and destination and (rule_dest == destination):
119+
found = True
120+
else:
121+
# Duplicate or different IP - delete it
122+
cmd = "sudo iptables -t nat -D OUTPUT {}".format(num)
123+
proc = subprocess.run(cmd, shell=True, capture_output=True)
124+
check_proc(proc)
125+
# Decrement number to accommodate deleted rule
126+
num -= 1
127+
128+
if destination and not found:
129+
cmd = "sudo iptables -t nat -A OUTPUT -p tcp -d {} --dport {} -j DNAT --to-destination {}".format(
130+
DST_IP, DST_PORT, destination)
131+
proc = subprocess.run(cmd, shell=True, capture_output=True)
132+
133+
check_proc(proc)
134+
135+
return destination

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

+11-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import syslog
99

1010
from collections import defaultdict
11+
from ctrmgr.ctrmgr_iptables import iptable_proxy_rule_upd
1112

1213
from swsscommon import swsscommon
1314
from sonic_py_common import device_info
@@ -87,11 +88,13 @@
8788
JOIN_LATENCY = "join_latency_on_boot_seconds"
8889
JOIN_RETRY = "retry_join_interval_seconds"
8990
LABEL_RETRY = "retry_labels_update_seconds"
91+
USE_K8S_PROXY = "use_k8s_as_http_proxy"
9092

9193
remote_ctr_config = {
9294
JOIN_LATENCY: 10,
9395
JOIN_RETRY: 10,
94-
LABEL_RETRY: 2
96+
LABEL_RETRY: 2,
97+
USE_K8S_PROXY: ""
9598
}
9699

97100
def log_debug(m):
@@ -309,6 +312,9 @@ def __init__(self, server):
309312

310313
self.start_time = datetime.datetime.now()
311314

315+
if remote_ctr_config[USE_K8S_PROXY] == "y":
316+
iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
317+
312318
if not self.st_server[ST_FEAT_UPDATE_TS]:
313319
# This is upon system start. Sleep 10m before join
314320
self.start_time += datetime.timedelta(
@@ -336,6 +342,9 @@ def on_config_update(self, key, op, data):
336342
log_debug("Received config update: {}".format(str(data)))
337343
self.cfg_server = cfg_data
338344

345+
if remote_ctr_config[USE_K8S_PROXY] == "y":
346+
iptable_proxy_rule_upd(self.cfg_server[CFG_SER_IP])
347+
339348
if self.pending:
340349
tnow = datetime.datetime.now()
341350
if tnow < self.start_time:
@@ -359,7 +368,7 @@ def handle_update(self):
359368

360369
ip = self.cfg_server[CFG_SER_IP]
361370
disable = self.cfg_server[CFG_SER_DISABLE] != "false"
362-
371+
363372
pre_state = dict(self.st_server)
364373
log_debug("server: handle_update: disable={} ip={}".format(disable, ip))
365374
if disable or not ip:
+5-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
[Unit]
22
Description=Container Manager watcher daemon
3-
Requires=updategraph.service
4-
After=updategraph.service
5-
3+
Requires=caclmgrd.service
4+
After=caclmgrd.service
5+
BindsTo=sonic.target
6+
After=sonic.target
67

78
[Service]
89
Type=simple
@@ -11,4 +12,4 @@ Restart=always
1112
RestartSec=30
1213

1314
[Install]
14-
WantedBy=multi-user.target
15+
WantedBy=sonic.target

src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"join_latency_on_boot_seconds": 300,
33
"retry_join_interval_seconds": 30,
44
"retry_labels_update_seconds": 5,
5-
"revert_to_local_on_wait_seconds": 60
5+
"revert_to_local_on_wait_seconds": 60,
6+
"use_k8s_as_http_proxy": "y"
67
}
78

0 commit comments

Comments
 (0)