Skip to content

Commit 70c7161

Browse files
TrekkieCoderTrekkieCoder
authored
Merge pull request #857 from TrekkieCoder/main
gh-87: added initial cicd for service sharding
2 parents cb901ec + d9878cd commit 70c7161

20 files changed

+692
-0
lines changed

‎cicd/k3s-sharding/EPconfig.txt

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"Attr":[
3+
{
4+
"hostName":"192.168.80.10",
5+
"name":"192.168.80.10_tcp_6443",
6+
"inactiveReTries":2,
7+
"probeType":"tcp",
8+
"probeReq":"",
9+
"probeResp":"",
10+
"probeDuration":10,
11+
"probePort":6443
12+
},
13+
{
14+
"hostName":"192.168.80.11",
15+
"name":"192.168.80.11_tcp_6443",
16+
"inactiveReTries":2,
17+
"probeType":"tcp",
18+
"probeReq":"",
19+
"probeResp":"",
20+
"probeDuration":10,
21+
"probePort":6443
22+
},
23+
{
24+
"hostName":"192.168.80.12",
25+
"name":"192.168.80.12_tcp_6443",
26+
"inactiveReTries":2,
27+
"probeType":"tcp",
28+
"probeReq":"",
29+
"probeResp":"",
30+
"probeDuration":10,
31+
"probePort":6443
32+
}
33+
]
34+
}

‎cicd/k3s-sharding/Vagrantfile

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# -*- mode: ruby -*-
2+
# vi: set ft=ruby :
3+
4+
workers = (ENV['WORKERS'] || "2").to_i
5+
box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
6+
box_version = "0.7.1"
7+
Vagrant.configure("2") do |config|
8+
config.vm.box = "#{box_name}"
9+
config.vm.box_version = "#{box_version}"
10+
11+
if Vagrant.has_plugin?("vagrant-vbguest")
12+
config.vbguest.auto_update = false
13+
end
14+
15+
config.vm.define "host" do |host|
16+
host.vm.hostname = 'host1'
17+
host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
18+
host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
19+
host.vm.provision :shell, :path => "host.sh"
20+
host.vm.provider :virtualbox do |vbox|
21+
vbox.customize ["modifyvm", :id, "--memory", 2048]
22+
vbox.customize ["modifyvm", :id, "--cpus", 1]
23+
end
24+
end
25+
26+
config.vm.define "master1" do |master|
27+
master.vm.hostname = 'master1'
28+
master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0"
29+
master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0"
30+
master.vm.provision :shell, :path => "master1.sh"
31+
master.vm.provider :virtualbox do |vbox|
32+
vbox.customize ["modifyvm", :id, "--memory", 8192]
33+
vbox.customize ["modifyvm", :id, "--cpus", 4]
34+
end
35+
end
36+
37+
config.vm.define "master2" do |master|
38+
master.vm.hostname = 'master2'
39+
master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0"
40+
master.vm.network :private_network, ip: "192.168.80.11", :netmask => "255.255.255.0"
41+
master.vm.provision :shell, :path => "master2.sh"
42+
master.vm.provider :virtualbox do |vbox|
43+
vbox.customize ["modifyvm", :id, "--memory", 8192]
44+
vbox.customize ["modifyvm", :id, "--cpus", 4]
45+
end
46+
end
47+
48+
config.vm.define "master3" do |master|
49+
master.vm.hostname = 'master3'
50+
master.vm.network :private_network, ip: "192.168.90.12", :netmask => "255.255.255.0"
51+
master.vm.network :private_network, ip: "192.168.80.12", :netmask => "255.255.255.0"
52+
master.vm.provision :shell, :path => "master3.sh"
53+
master.vm.provider :virtualbox do |vbox|
54+
vbox.customize ["modifyvm", :id, "--memory", 8192]
55+
vbox.customize ["modifyvm", :id, "--cpus", 4]
56+
end
57+
end
58+
59+
60+
(1..workers).each do |node_number|
61+
config.vm.define "worker#{node_number}" do |worker|
62+
worker.vm.hostname = "worker#{node_number}"
63+
ip = node_number + 100
64+
worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0"
65+
worker.vm.provision :shell, :path => "worker.sh"
66+
worker.vm.provider :virtualbox do |vbox|
67+
vbox.customize ["modifyvm", :id, "--memory", 4096]
68+
vbox.customize ["modifyvm", :id, "--cpus", 2]
69+
end
70+
end
71+
end
72+
end

‎cicd/k3s-sharding/config.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
3+
vagrant up
4+
#sudo ip route add 123.123.123.1 via 192.168.90.10 || true
5+
vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/tcp-onearm-ds.yml'
6+
vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/udp-onearm-ds.yml'
7+
vagrant ssh master1 -c 'sudo kubectl create -f /vagrant/sctp-onearm-ds.yml'

‎cicd/k3s-sharding/host.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
sudo apt-get install -y lksctp-tools
2+
sudo ip route add 123.123.123.0/24 via 192.168.90.10
3+
sysctl net.ipv4.conf.eth1.arp_accept=1
4+
sysctl net.ipv4.conf.eth2.arp_accept=1
5+
sysctl net.ipv4.conf.default.arp_accept=1
6+
echo "Host is up"

‎cicd/k3s-sharding/host_validation.sh

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
extIP=$(cat /vagrant/extIP)
3+
extIP1=$(cat /vagrant/extIP1)
4+
extIP2=$(cat /vagrant/extIP2)
5+
6+
mode="onearm"
7+
tcp_port=55001
8+
udp_port=55002
9+
sctp_port=55003
10+
11+
code=0
12+
echo TCP Service IP: $extIP
13+
14+
ip route list match $extIP | grep $extIP -A 2
15+
16+
echo -e "\n*********************************************"
17+
echo "Testing Service"
18+
echo "*********************************************"
19+
for((i=0;i<20;i++))
20+
do
21+
22+
out=$(curl -s --connect-timeout 10 http://$extIP:$tcp_port)
23+
if [[ ${out} == *"Welcome to nginx"* ]]; then
24+
echo -e "K3s-sharding TCP\t($mode)\t[OK]"
25+
else
26+
echo -e "K3s-sharding TCP\t($mode)\t[FAILED]"
27+
code=1
28+
fi
29+
30+
echo UDP Service IP: $extIP1
31+
32+
out=$(timeout 5 /vagrant/udp_client $extIP1 $udp_port)
33+
if [[ ${out} == *"Client"* ]]; then
34+
echo -e "K3s-sharding UDP\t($mode)\t[OK]"
35+
else
36+
echo -e "K3s-sharding UDP\t($mode)\t[FAILED]"
37+
code=1
38+
fi
39+
40+
echo SCTP Service IP: $extIP2
41+
42+
sctp_darn -H 192.168.80.9 -h $extIP2 -p $sctp_port -s < /vagrant/input > output
43+
#sleep 2
44+
exp="New connection, peer addresses
45+
192.168.80.202:55003"
46+
47+
res=`cat output | grep -A 1 "New connection, peer addresses"`
48+
sudo rm -rf output
49+
if [[ "$res" == "$exp" ]]; then
50+
#echo $res
51+
echo -e "K3s-sharding SCTP\t($mode)\t[OK]"
52+
else
53+
echo -e "K3s-sharding SCTP\t($mode)\t[FAILED]"
54+
code=1
55+
fi
56+
57+
58+
done
59+
exit $code

‎cicd/k3s-sharding/input

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
3+
4+
5+
6+

‎cicd/k3s-sharding/kube-loxilb.yml

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: kube-loxilb
6+
namespace: kube-system
7+
---
8+
kind: ClusterRole
9+
apiVersion: rbac.authorization.k8s.io/v1
10+
metadata:
11+
name: kube-loxilb
12+
rules:
13+
- apiGroups:
14+
- ""
15+
resources:
16+
- nodes
17+
verbs:
18+
- get
19+
- watch
20+
- list
21+
- patch
22+
- apiGroups:
23+
- ""
24+
resources:
25+
- pods
26+
verbs:
27+
- get
28+
- watch
29+
- list
30+
- patch
31+
- apiGroups:
32+
- ""
33+
resources:
34+
- endpoints
35+
- services
36+
- namespaces
37+
- services/status
38+
verbs:
39+
- get
40+
- watch
41+
- list
42+
- patch
43+
- update
44+
- apiGroups:
45+
- discovery.k8s.io
46+
resources:
47+
- endpointslices
48+
verbs:
49+
- get
50+
- watch
51+
- list
52+
- apiGroups:
53+
- authentication.k8s.io
54+
resources:
55+
- tokenreviews
56+
verbs:
57+
- create
58+
- apiGroups:
59+
- authorization.k8s.io
60+
resources:
61+
- subjectaccessreviews
62+
verbs:
63+
- create
64+
---
65+
kind: ClusterRoleBinding
66+
apiVersion: rbac.authorization.k8s.io/v1
67+
metadata:
68+
name: kube-loxilb
69+
roleRef:
70+
apiGroup: rbac.authorization.k8s.io
71+
kind: ClusterRole
72+
name: kube-loxilb
73+
subjects:
74+
- kind: ServiceAccount
75+
name: kube-loxilb
76+
namespace: kube-system
77+
---
78+
apiVersion: apps/v1
79+
kind: Deployment
80+
metadata:
81+
name: kube-loxilb
82+
namespace: kube-system
83+
labels:
84+
app: loxilb
85+
spec:
86+
replicas: 1
87+
selector:
88+
matchLabels:
89+
app: loxilb
90+
template:
91+
metadata:
92+
labels:
93+
app: loxilb
94+
spec:
95+
hostNetwork: true
96+
dnsPolicy: ClusterFirstWithHostNet
97+
tolerations:
98+
- effect: NoSchedule
99+
operator: Exists
100+
# Mark the pod as a critical add-on for rescheduling.
101+
- key: CriticalAddonsOnly
102+
operator: Exists
103+
- effect: NoExecute
104+
operator: Exists
105+
- key: "node-role.kubernetes.io/master"
106+
operator: Exists
107+
- key: "node-role.kubernetes.io/control-plane"
108+
operator: Exists
109+
affinity:
110+
nodeAffinity:
111+
requiredDuringSchedulingIgnoredDuringExecution:
112+
nodeSelectorTerms:
113+
- matchExpressions:
114+
- key: "node-role.kubernetes.io/master"
115+
operator: Exists
116+
- key: "node-role.kubernetes.io/control-plane"
117+
operator: Exists
118+
priorityClassName: system-node-critical
119+
serviceAccountName: kube-loxilb
120+
terminationGracePeriodSeconds: 0
121+
containers:
122+
- name: kube-loxilb
123+
image: ghcr.io/loxilb-io/kube-loxilb:latest
124+
imagePullPolicy: Always
125+
command:
126+
- /bin/kube-loxilb
127+
args:
128+
#- --loxiURL=http://192.168.80.10:11111
129+
- --cidrPools=defaultPool=192.168.80.200/24
130+
#- --setBGP=64512
131+
- --setRoles=0.0.0.0
132+
- --setUniqueIP
133+
- --numZoneInstances=3
134+
#- --monitor
135+
#- --setBGP
136+
#- --setLBMode=1
137+
#- --config=/opt/loxilb/agent/kube-loxilb.conf
138+
resources:
139+
requests:
140+
cpu: "100m"
141+
memory: "50Mi"
142+
limits:
143+
cpu: "100m"
144+
memory: "50Mi"
145+
securityContext:
146+
privileged: true
147+
capabilities:
148+
add: ["NET_ADMIN", "NET_RAW"]

‎cicd/k3s-sharding/lbconfig.txt

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"lbAttr":[
3+
{
4+
"serviceArguments":{
5+
"externalIP":"192.168.80.80",
6+
"port":6443,
7+
"protocol":"tcp",
8+
"sel":0,
9+
"mode":2,
10+
"BGP":false,
11+
"Monitor":true,
12+
"inactiveTimeOut":240,
13+
"block":0
14+
},
15+
"secondaryIPs":null,
16+
"endpoints":[
17+
{
18+
"endpointIP":"192.168.80.10",
19+
"targetPort":6443,
20+
"weight":1,
21+
"state":"active",
22+
"counter":""
23+
},
24+
{
25+
"endpointIP":"192.168.80.11",
26+
"targetPort":6443,
27+
"weight":1,
28+
"state":"active",
29+
"counter":""
30+
},
31+
{
32+
"endpointIP":"192.168.80.12",
33+
"targetPort":6443,
34+
"weight":1,
35+
"state":"active",
36+
"counter":""
37+
}
38+
]
39+
}
40+
]
41+
}

0 commit comments

Comments
 (0)