Skip to content

Commit cc9a811

Browse files
TrekkieCoderTrekkieCoder
committed
cicd: added k3s-multi-master scenario with service LB and proxy support
1 parent 145c68d commit cc9a811

26 files changed

+856
-0
lines changed

‎cicd/k3s-multi-master-service-proxy/EPconfig.txt

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"Attr":[
3+
{
4+
"hostName":"192.168.80.10",
5+
"name":"192.168.80.10_tcp_6443",
6+
"inactiveReTries":2,
7+
"probeType":"tcp",
8+
"probeReq":"",
9+
"probeResp":"",
10+
"probeDuration":5,
11+
"probePort":6443
12+
},
13+
{
14+
"hostName":"192.168.80.11",
15+
"name":"192.168.80.11_tcp_6443",
16+
"inactiveReTries":2,
17+
"probeType":"tcp",
18+
"probeReq":"",
19+
"probeResp":"",
20+
"probeDuration":5,
21+
"probePort":6443
22+
},
23+
{
24+
"hostName":"192.168.80.12",
25+
"name":"192.168.80.12_tcp_6443",
26+
"inactiveReTries":2,
27+
"probeType":"tcp",
28+
"probeReq":"",
29+
"probeResp":"",
30+
"probeDuration":5,
31+
"probePort":6443
32+
}
33+
]
34+
}

‎cicd/k3s-multi-master-service-proxy/README

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
## Test Case Description
2+
3+
This scenario will have K3s(2 Master Nodes & 2 Worker Nodes) cluster with flannel CNI. LoxiLB will be running in the in-cluster Active-Backup High Availabity mode(in both the master nodes) but without State Syncronization. Workloads will be spawned in all the cluster nodes.
4+
5+
Client will be connected directly to the cluster with L2 network. Service CIDR will also be a Virtual IP from the K3s cluster network.
6+
7+
In in-cluster scenarios, it is advised to create LB services in either one-arm or fullnat mode for ease of connectivity.

‎cicd/k3s-multi-master-service-proxy/Vagrantfile

+85
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# -*- mode: ruby -*-
2+
# vi: set ft=ruby :
3+
4+
workers = (ENV['WORKERS'] || "2").to_i
5+
box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
6+
box_version = "0.7.1"
7+
Vagrant.configure("2") do |config|
8+
config.vm.box = "#{box_name}"
9+
config.vm.box_version = "#{box_version}"
10+
11+
if Vagrant.has_plugin?("vagrant-vbguest")
12+
config.vbguest.auto_update = false
13+
end
14+
15+
config.vm.define "host" do |host|
16+
host.vm.hostname = 'host1'
17+
host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
18+
host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
19+
host.vm.provision :shell, :path => "host.sh"
20+
host.vm.provider :virtualbox do |vbox|
21+
vbox.memory = "2048"
22+
vbox.cpus = "2"
23+
vbox.default_nic_type = "virtio"
24+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
25+
vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
26+
end
27+
end
28+
29+
config.vm.define "master1" do |master|
30+
master.vm.hostname = 'master1'
31+
master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0"
32+
master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0"
33+
master.vm.provision :shell, :path => "master1.sh"
34+
master.vm.provider :virtualbox do |vbox|
35+
vbox.memory = "8192"
36+
vbox.cpus = "3"
37+
vbox.default_nic_type = "virtio"
38+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
39+
vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
40+
end
41+
end
42+
43+
config.vm.define "master2" do |master|
44+
master.vm.hostname = 'master2'
45+
master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0"
46+
master.vm.network :private_network, ip: "192.168.80.11", :netmask => "255.255.255.0"
47+
master.vm.provision :shell, :path => "master2.sh"
48+
master.vm.provider :virtualbox do |vbox|
49+
vbox.memory = "8192"
50+
vbox.cpus = "3"
51+
vbox.default_nic_type = "virtio"
52+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
53+
vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
54+
end
55+
end
56+
57+
config.vm.define "master3" do |master|
58+
master.vm.hostname = 'master3'
59+
master.vm.network :private_network, ip: "192.168.90.12", :netmask => "255.255.255.0"
60+
master.vm.network :private_network, ip: "192.168.80.12", :netmask => "255.255.255.0"
61+
master.vm.provision :shell, :path => "master3.sh"
62+
master.vm.provider :virtualbox do |vbox|
63+
vbox.memory = "8192"
64+
vbox.cpus = "3"
65+
vbox.default_nic_type = "virtio"
66+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
67+
vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
68+
end
69+
end
70+
71+
(1..workers).each do |node_number|
72+
config.vm.define "worker#{node_number}" do |worker|
73+
worker.vm.hostname = "worker#{node_number}"
74+
ip = node_number + 100
75+
worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
76+
worker.vm.provision :shell, :path => "worker.sh"
77+
worker.vm.provider :virtualbox do |vbox|
78+
vbox.memory = "4096"
79+
vbox.cpus = "3"
80+
vbox.default_nic_type = "virtio"
81+
vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
82+
end
83+
end
84+
end
85+
end

‎cicd/k3s-multi-master-service-proxy/config.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
3+
vagrant up
4+
#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/tcp-onearm-ds.yml'
5+
#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/udp-onearm-ds.yml'
6+
#vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/sctp-onearm-ds.yml'

‎cicd/k3s-multi-master-service-proxy/host.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
sudo apt-get install -y lksctp-tools iperf iperf3
2+
sysctl net.core.netdev_max_backlog=10000
3+
echo "Host is up"

‎cicd/k3s-multi-master-service-proxy/host_validation.sh

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
extIP=$(cat /vagrant/extIP)
3+
4+
mode="onearm"
5+
tcp_port=55001
6+
udp_port=55002
7+
sctp_port=55003
8+
9+
code=0
10+
echo Service IP: $extIP
11+
12+
ip route list match $extIP | grep $extIP -A 2
13+
14+
echo -e "\n*********************************************"
15+
echo "Testing Service"
16+
echo "*********************************************"
17+
for((i=0;i<20;i++))
18+
do
19+
20+
out=$(curl -s --connect-timeout 10 http://$extIP:$tcp_port)
21+
if [[ ${out} == *"Welcome to nginx"* ]]; then
22+
echo -e "K3s-flannel-incluster-l2 TCP\t($mode)\t[OK]"
23+
else
24+
echo -e "K3s-flannel-incluster-l2 TCP\t($mode)\t[FAILED]"
25+
code=1
26+
fi
27+
28+
out=$(timeout 5 /vagrant/udp_client $extIP $udp_port)
29+
if [[ ${out} == *"Client"* ]]; then
30+
echo -e "K3s-flannel-incluster-l2 UDP\t($mode)\t[OK]"
31+
else
32+
echo -e "K3s-flannel-incluster-l2 UDP\t($mode)\t[FAILED]"
33+
code=1
34+
fi
35+
36+
sctp_darn -H 192.168.80.9 -h $extIP -p $sctp_port -s < /vagrant/input > output
37+
#sleep 2
38+
exp="New connection, peer addresses
39+
192.168.80.200:55003"
40+
41+
res=`cat output | grep -A 1 "New connection, peer addresses"`
42+
sudo rm -rf output
43+
if [[ "$res" == "$exp" ]]; then
44+
#echo $res
45+
echo -e "K3s-flannel-incluster-l2 SCTP\t($mode)\t[OK]"
46+
else
47+
echo -e "K3s-flannel-incluster-l2 SCTP\t($mode)\t[FAILED]"
48+
code=1
49+
fi
50+
51+
52+
done
53+
exit $code

‎cicd/k3s-multi-master-service-proxy/input

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
3+
4+
5+
6+

‎cicd/k3s-multi-master-service-proxy/iperf-onearm-ds.yml

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: iperf-service
5+
annotations:
6+
loxilb.io/lbmode: "onearm"
7+
spec:
8+
externalTrafficPolicy: Local
9+
loadBalancerClass: loxilb.io/loxilb
10+
selector:
11+
app: iperf-app
12+
ports:
13+
- port: 55001
14+
targetPort: 5001
15+
type: LoadBalancer
16+
---
17+
apiVersion: apps/v1
18+
kind: DaemonSet
19+
metadata:
20+
name: iperf-daemonset
21+
labels:
22+
app: iperf-app
23+
spec:
24+
selector:
25+
matchLabels:
26+
app: iperf-app
27+
template:
28+
metadata:
29+
labels:
30+
app: iperf-app
31+
spec:
32+
dnsPolicy: ClusterFirstWithHostNet
33+
#affinity:
34+
# nodeAffinity:
35+
# requiredDuringSchedulingIgnoredDuringExecution:
36+
# nodeSelectorTerms:
37+
# - matchExpressions:
38+
# - key: "node-role.kubernetes.io/master"
39+
# operator: DoesNotExist
40+
# - key: "node-role.kubernetes.io/control-plane"
41+
# operator: DoesNotExist
42+
containers:
43+
- name: iperf
44+
image: eyes852/ubuntu-iperf-test:0.5
45+
command: [ "iperf", "-s"]
46+
ports:
47+
- containerPort: 5001

‎cicd/k3s-multi-master-service-proxy/k3s.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: v1
2+
clusters:
3+
- cluster:
4+
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTU29Zc0RSbkVnNGdpeFVDcEY3aWZpUWVWVU5QMDJidmc5bUJ6VmUwbUcKRDZmZjVWV1h2Yk1JTXM4UWdzUDlkMVMrUWNab3JyUGEvZHo3NXJDRnZrM2tvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVZLSEQwQ0prd2k3SFQ3ejJPUUNSCmZZTC83Y1F3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnWmp1aENUek5sYzMyc1NTa3hTVndEaUxEQ2hHenpaRUkKaGxGSFJtNFIrbllDSVFDaHJja0kzY3F6djB6UTN2VFNjSHM5MzRYSkF6S0dnb0hQZmx4b0tkbVM4QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
5+
server: https://192.168.80.80:6443
6+
name: default
7+
contexts:
8+
- context:
9+
cluster: default
10+
user: default
11+
name: default
12+
current-context: default
13+
kind: Config
14+
preferences: {}
15+
users:
16+
- name: default
17+
user:
18+
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJYUR3NFdwek1wRmd3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekUwTXpnMk5EZzVNQjRYRFRJME1EUXlPVEV3TWpnd09Wb1hEVEkxTURReQpPVEV3TWpnd09Wb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJDdUdLQlZNSUJNYVVrZEcKQlNCalBGNzNWMmVaNHZCaGl0ejUrYWtqRUprNnBWSW1yQzdPTkNVVytjOWowZnFJS0xZVGJyblI3cWZiT0FiMgp2UU9GWTZ1alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCU21QOXBVR0FoNlpKOFZONUhnUzdHMEd6Y092akFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBaEVNcWs3TUtPTXd6TmM4aEc3L04wR3RZZzIzQVgvdmFsdmNEOFAycitqZ0lnZHJIbDZZdmNTbWFoVG1GYgowQTJwWkpxK2hvNkg2ZThBVCtRRTJGSHVKZXM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRRek9EWTBPRGt3SGhjTk1qUXdOREk1TVRBeU9EQTVXaGNOTXpRd05ESTNNVEF5T0RBNQpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRRek9EWTBPRGt3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTNVZyeHFpaE4yMktlcCtwSTFWalBna21hV09tUXBBek9wY0VrQzNrRVAKTDB3cS9CalVGaVo2ckNxZklLSndzckhnbGgxemtJQzhuRjdhbzU0SjdzbS9vMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVXBqL2FWQmdJZW1TZkZUZVI0RXV4CnRCczNEcjR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUlnSFVIM3UzR2NsWmxPQWFNMEpNOVRrMTVISUhYUVJ2VVIKcmUzNksvSkNmTWNDSVFEdk5pWmdFaGVmRytrZkdoOHBHK080REFHVVA1ZG5QTXNMU2JvZVlWelhTUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
19+
client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUJNL1lLR0svWkkzc3IrUmZjNURQWkF3YzBtY3gwZFBlNFkwdmxpdGhMS3JvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFSzRZb0ZVd2dFeHBTUjBZRklHTThYdmRYWjVuaThHR0szUG41cVNNUW1UcWxVaWFzTHM0MApKUmI1ejJQUitvZ290aE51dWRIdXA5czRCdmE5QTRWanF3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=

0 commit comments

Comments
 (0)