Skip to content

Commit a1b50ca

Browse files
lixiaoyunerlixiaoyuner
authored
Make client indentity by AME cert (#11946)
* Make client indentity by AME cert * Join k8s cluster by ipv6 * Change join test cases * Test case bug fix * Improve read node label func * Configure kubelet and change test cases * For kubernetes version 1.22.2 * Fix undefine issue Signed-off-by: Yun Li <[email protected]>
1 parent 23de13f commit a1b50ca

File tree

5 files changed

+148
-27
lines changed

5 files changed

+148
-27
lines changed

‎files/build_templates/sonic_debian_extension.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd
455455
{% if include_kubernetes == "y" %}
456456
# Point to kubelet to /etc/resolv.conf
457457
#
458-
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
458+
echo 'KUBELET_EXTRA_ARGS="--resolv-conf=/etc/resolv.conf --cgroup-driver=cgroupfs --node-ip=::"' | sudo tee -a $FILESYSTEM_ROOT/etc/default/kubelet
459459

460460
# Copy Flannel conf file into sonic-templates
461461
#

‎src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,12 @@ def log_debug(m):
104104

105105

106106
def log_error(m):
107+
msg = "{}: {}".format(inspect.stack()[1][3], m)
107108
syslog.syslog(syslog.LOG_ERR, msg)
108109

109110

110111
def log_info(m):
112+
msg = "{}: {}".format(inspect.stack()[1][3], m)
111113
syslog.syslog(syslog.LOG_INFO, msg)
112114

113115

‎src/sonic-ctrmgrd/ctrmgr/kube_commands.py

Lines changed: 76 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,24 @@
1313
import syslog
1414
import tempfile
1515
import urllib.request
16+
import base64
1617
from urllib.parse import urlparse
1718

1819
import yaml
20+
import requests
1921
from sonic_py_common import device_info
22+
from jinja2 import Template
23+
from swsscommon import swsscommon
2024

2125
KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf"
2226
KUBELET_YAML = "/var/lib/kubelet/config.yaml"
2327
SERVER_ADMIN_URL = "https://{}/admin.conf"
2428
LOCK_FILE = "/var/lock/kube_join.lock"
2529
FLANNEL_CONF_FILE = "/usr/share/sonic/templates/kube_cni.10-flannel.conflist"
2630
CNI_DIR = "/etc/cni/net.d"
31+
K8S_CA_URL = "https://{}:{}/api/v1/namespaces/default/configmaps/kube-root-ca.crt"
32+
AME_CRT = "/etc/sonic/credentials/restapiserver.crt"
33+
AME_KEY = "/etc/sonic/credentials/restapiserver.key"
2734

2835
def log_debug(m):
2936
msg = "{}: {}".format(inspect.stack()[1][3], m)
@@ -77,8 +84,7 @@ def _run_command(cmd, timeout=5):
7784

7885
def kube_read_labels():
7986
""" Read current labels on node and return as dict. """
80-
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes --show-labels |\
81-
grep {} | tr -s ' ' | cut -f6 -d' '"
87+
KUBECTL_GET_CMD = "kubectl --kubeconfig {} get nodes {} --show-labels |tr -s ' ' | cut -f6 -d' '"
8288

8389
labels = {}
8490
ret, out, _ = _run_command(KUBECTL_GET_CMD.format(
@@ -211,6 +217,68 @@ def _download_file(server, port, insecure):
211217
log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
212218

213219

220+
def _gen_cli_kubeconf(server, port, insecure):
221+
"""generate identity which can help authenticate and
222+
authorization to k8s cluster
223+
"""
224+
client_kubeconfig_template = """
225+
apiVersion: v1
226+
clusters:
227+
- cluster:
228+
certificate-authority-data: {{ k8s_ca }}
229+
server: https://{{ vip }}:{{ port }}
230+
name: kubernetes
231+
contexts:
232+
- context:
233+
cluster: kubernetes
234+
user: user
235+
name: user@kubernetes
236+
current-context: user@kubernetes
237+
kind: Config
238+
preferences: {}
239+
users:
240+
- name: user
241+
user:
242+
client-certificate-data: {{ ame_crt }}
243+
client-key-data: {{ ame_key }}
244+
"""
245+
if insecure:
246+
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY), verify=False)
247+
else:
248+
r = requests.get(K8S_CA_URL.format(server, port), cert=(AME_CRT, AME_KEY))
249+
if not r.ok:
250+
raise requests.RequestException("Something wrong with AME cert or something wrong about sonic role in k8s cluster")
251+
k8s_ca = r.json()["data"]["ca.crt"]
252+
k8s_ca_b64 = base64.b64encode(k8s_ca.encode("utf-8")).decode("utf-8")
253+
ame_crt_raw = open(AME_CRT, "rb")
254+
ame_crt_b64 = base64.b64encode(ame_crt_raw.read()).decode("utf-8")
255+
ame_key_raw = open(AME_KEY, "rb")
256+
ame_key_b64 = base64.b64encode(ame_key_raw.read()).decode("utf-8")
257+
client_kubeconfig_template_j2 = Template(client_kubeconfig_template)
258+
client_kubeconfig = client_kubeconfig_template_j2.render(
259+
k8s_ca=k8s_ca_b64, vip=server, port=port, ame_crt=ame_crt_b64, ame_key=ame_key_b64)
260+
(h, fname) = tempfile.mkstemp(suffix="_kube_join")
261+
os.write(h, client_kubeconfig.encode("utf-8"))
262+
os.close(h)
263+
log_debug("Downloaded = {}".format(fname))
264+
265+
shutil.copyfile(fname, KUBE_ADMIN_CONF)
266+
267+
log_debug("{} downloaded".format(KUBE_ADMIN_CONF))
268+
269+
270+
def _get_local_ipv6():
271+
try:
272+
config_db = swsscommon.DBConnector("CONFIG_DB", 0)
273+
mgmt_ip_data = swsscommon.Table(config_db, 'MGMT_INTERFACE')
274+
for key in mgmt_ip_data.getKeys():
275+
if key.find(":") >= 0:
276+
return key.split("|")[1].split("/")[0]
277+
raise IOError("IPV6 not find from MGMT_INTERFACE table")
278+
except Exception as e:
279+
raise IOError(str(e))
280+
281+
214282
def _troubleshoot_tips():
215283
""" log troubleshoot tips which could be handy,
216284
when in trouble with join
@@ -264,12 +332,14 @@ def _do_reset(pending_join = False):
264332

265333

266334
def _do_join(server, port, insecure):
267-
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {}"
335+
KUBEADM_JOIN_CMD = "kubeadm join --discovery-file {} --node-name {} --apiserver-advertise-address {}"
268336
err = ""
269337
out = ""
270338
ret = 0
271339
try:
272-
_download_file(server, port, insecure)
340+
local_ipv6 = _get_local_ipv6()
341+
#_download_file(server, port, insecure)
342+
_gen_cli_kubeconf(server, port, insecure)
273343
_do_reset(True)
274344
_run_command("modprobe br_netfilter")
275345
# Copy flannel.conf
@@ -279,11 +349,11 @@ def _do_join(server, port, insecure):
279349

280350
if ret == 0:
281351
(ret, out, err) = _run_command(KUBEADM_JOIN_CMD.format(
282-
KUBE_ADMIN_CONF, get_device_name()), timeout=60)
352+
KUBE_ADMIN_CONF, get_device_name(), local_ipv6), timeout=60)
283353
log_debug("ret = {}".format(ret))
284354

285355
except IOError as e:
286-
err = "Download failed: {}".format(str(e))
356+
err = "Join failed: {}".format(str(e))
287357
ret = -1
288358
out = ""
289359

‎src/sonic-ctrmgrd/tests/common_test.py

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
CONFIG_DB_NO = 4
1010
STATE_DB_NO = 6
1111
FEATURE_TABLE = "FEATURE"
12+
MGMT_INTERFACE_TABLE = "MGMT_INTERFACE"
1213
KUBE_LABEL_TABLE = "KUBE_LABELS"
1314
KUBE_LABEL_SET_KEY = "SET"
1415

@@ -41,6 +42,7 @@
4142
IMAGE_TAG = "image_tag"
4243
FAIL_LOCK = "fail_lock"
4344
DO_JOIN = "do_join"
45+
REQ = "req"
4446

4547
# subproc key words
4648

@@ -643,8 +645,27 @@ def mock_subproc_side_effect(cmd, shell=False, stdout=None, stderr=None):
643645
return mock_proc(cmd, index)
644646

645647

646-
def set_kube_mock(mock_subproc):
648+
class mock_reqget:
649+
def __init__(self):
650+
self.ok = True
651+
652+
def json(self):
653+
return current_test_data.get(REQ, "")
654+
655+
656+
def mock_reqget_side_effect(url, cert, verify=True):
657+
return mock_reqget()
658+
659+
660+
def set_kube_mock(mock_subproc, mock_table=None, mock_conn=None, mock_reqget=None):
647661
mock_subproc.side_effect = mock_subproc_side_effect
662+
if mock_table != None:
663+
mock_table.side_effect = table_side_effect
664+
if mock_conn != None:
665+
mock_conn.side_effect = conn_side_effect
666+
if mock_reqget != None:
667+
mock_reqget.side_effect = mock_reqget_side_effect
668+
648669

649670
def create_remote_ctr_config_json():
650671
str_conf = '\

‎src/sonic-ctrmgrd/tests/kube_commands_test.py

Lines changed: 47 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
KUBE_ADMIN_CONF = "/tmp/kube_admin.conf"
1616
FLANNEL_CONF_FILE = "/tmp/flannel.conf"
1717
CNI_DIR = "/tmp/cni/net.d"
18+
AME_CRT = "/tmp/restapiserver.crt"
19+
AME_KEY = "/tmp/restapiserver.key"
1820

1921
# kube_commands test cases
2022
# NOTE: Ensure state-db entry is complete in PRE as we need to
@@ -25,8 +27,7 @@
2527
common_test.DESCR: "read labels",
2628
common_test.RETVAL: 0,
2729
common_test.PROC_CMD: ["\
28-
kubectl --kubeconfig {} get nodes --show-labels |\
29-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
30+
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
3031
common_test.PROC_OUT: ["foo=bar,hello=world"],
3132
common_test.POST: {
3233
"foo": "bar",
@@ -39,8 +40,7 @@
3940
common_test.TRIGGER_THROW: True,
4041
common_test.RETVAL: -1,
4142
common_test.PROC_CMD: ["\
42-
kubectl --kubeconfig {} get nodes --show-labels |\
43-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
43+
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
4444
common_test.POST: {
4545
},
4646
common_test.PROC_KILLED: 1
@@ -49,8 +49,7 @@
4949
common_test.DESCR: "read labels fail",
5050
common_test.RETVAL: -1,
5151
common_test.PROC_CMD: ["\
52-
kubectl --kubeconfig {} get nodes --show-labels |\
53-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
52+
kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)],
5453
common_test.PROC_OUT: [""],
5554
common_test.PROC_ERR: ["command failed"],
5655
common_test.POST: {
@@ -65,8 +64,7 @@
6564
common_test.RETVAL: 0,
6665
common_test.ARGS: { "foo": "bar", "hello": "World!", "test": "ok" },
6766
common_test.PROC_CMD: [
68-
"kubectl --kubeconfig {} get nodes --show-labels |\
69-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
67+
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF),
7068
"kubectl --kubeconfig {} label --overwrite nodes none hello-".format(
7169
KUBE_ADMIN_CONF),
7270
"kubectl --kubeconfig {} label --overwrite nodes none hello=World! test=ok".format(
@@ -79,8 +77,7 @@
7977
common_test.RETVAL: 0,
8078
common_test.ARGS: { "foo": "bar", "hello": "world" },
8179
common_test.PROC_CMD: [
82-
"kubectl --kubeconfig {} get nodes --show-labels |\
83-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
80+
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
8481
],
8582
common_test.PROC_OUT: ["foo=bar,hello=world"]
8683
},
@@ -90,8 +87,7 @@
9087
common_test.ARGS: { "any": "thing" },
9188
common_test.RETVAL: -1,
9289
common_test.PROC_CMD: [
93-
"kubectl --kubeconfig {} get nodes --show-labels |\
94-
grep none | tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
90+
"kubectl --kubeconfig {} get nodes none --show-labels |tr -s ' ' | cut -f6 -d' '".format(KUBE_ADMIN_CONF)
9591
],
9692
common_test.PROC_ERR: ["read failed"]
9793
}
@@ -114,10 +110,22 @@
114110
"mkdir -p {}".format(CNI_DIR),
115111
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
116112
"systemctl start kubelet",
117-
"kubeadm join --discovery-file {} --node-name none".format(
113+
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
118114
KUBE_ADMIN_CONF)
119115
],
120-
common_test.PROC_RUN: [True, True]
116+
common_test.PROC_RUN: [True, True],
117+
common_test.PRE: {
118+
common_test.CONFIG_DB_NO: {
119+
common_test.MGMT_INTERFACE_TABLE: {
120+
"eth0|FC00:2::32/64": {
121+
"gwaddr": "fc00:2::1"
122+
}
123+
}
124+
}
125+
},
126+
common_test.REQ: {
127+
"data": {"ca.crt": "test"}
128+
}
121129
},
122130
1: {
123131
common_test.DESCR: "Regular secure join",
@@ -135,10 +143,22 @@
135143
"mkdir -p {}".format(CNI_DIR),
136144
"cp {} {}".format(FLANNEL_CONF_FILE, CNI_DIR),
137145
"systemctl start kubelet",
138-
"kubeadm join --discovery-file {} --node-name none".format(
146+
"kubeadm join --discovery-file {} --node-name none --apiserver-advertise-address FC00:2::32".format(
139147
KUBE_ADMIN_CONF)
140148
],
141-
common_test.PROC_RUN: [True, True]
149+
common_test.PROC_RUN: [True, True],
150+
common_test.PRE: {
151+
common_test.CONFIG_DB_NO: {
152+
common_test.MGMT_INTERFACE_TABLE: {
153+
"eth0|FC00:2::32/64": {
154+
"gwaddr": "fc00:2::1"
155+
}
156+
}
157+
}
158+
},
159+
common_test.REQ: {
160+
"data": {"ca.crt": "test"}
161+
}
142162
},
143163
2: {
144164
common_test.DESCR: "Skip join as already connected",
@@ -228,11 +248,17 @@ def init(self):
228248
s.close()
229249
with open(FLANNEL_CONF_FILE, "w") as s:
230250
s.close()
251+
with open(AME_CRT, "w") as s:
252+
s.close()
253+
with open(AME_KEY, "w") as s:
254+
s.close()
231255
kube_commands.KUBELET_YAML = kubelet_yaml
232256
kube_commands.CNI_DIR = CNI_DIR
233257
kube_commands.FLANNEL_CONF_FILE = FLANNEL_CONF_FILE
234258
kube_commands.SERVER_ADMIN_URL = "file://{}".format(self.admin_conf_file)
235259
kube_commands.KUBE_ADMIN_CONF = KUBE_ADMIN_CONF
260+
kube_commands.AME_CRT = AME_CRT
261+
kube_commands.AME_KEY = AME_KEY
236262

237263

238264
@patch("kube_commands.subprocess.Popen")
@@ -295,11 +321,13 @@ def test_write_labels(self, mock_subproc):
295321
json.dumps(labels, indent=4)))
296322
assert False
297323

298-
324+
@patch("kube_commands.requests.get")
325+
@patch("kube_commands.swsscommon.DBConnector")
326+
@patch("kube_commands.swsscommon.Table")
299327
@patch("kube_commands.subprocess.Popen")
300-
def test_join(self, mock_subproc):
328+
def test_join(self, mock_subproc, mock_table, mock_conn, mock_reqget):
301329
self.init()
302-
common_test.set_kube_mock(mock_subproc)
330+
common_test.set_kube_mock(mock_subproc, mock_table, mock_conn, mock_reqget)
303331

304332
for (i, ct_data) in join_test_data.items():
305333
lock_file = ""

0 commit comments

Comments
 (0)