Skip to content

Commit ec7eca1

Browse files
nathanklicknathanklick
authored
feat: add support for multi-cluster testing via KinD (#1392)
Signed-off-by: Nathan Klick <[email protected]>
1 parent 22ed9ae commit ec7eca1

16 files changed

+352
-23
lines changed

‎.github/workflows/flow-gcs-test.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@ jobs:
8989
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
9090
with:
9191
install_only: true
92-
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
93-
version: v0.21.0
94-
kubectl_version: v1.28.6
92+
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
93+
version: v0.26.0
94+
kubectl_version: v1.31.4
9595
verbosity: 3
9696
wait: 120s
9797

‎.github/workflows/flow-pull-request-checks.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ jobs:
9191
- { name: "Node Delete - Separate commands", npm-test-script: "test-${{ needs.env-vars.outputs.e2e-node-delete-separate-commands-test-subdir }}", coverage-subdirectory: "${{ needs.env-vars.outputs.e2e-node-delete-separate-commands-test-subdir }}", coverage-report-name: "${{ needs.env-vars.outputs.e2e-node-delete-separate-commands-coverage-report }}", cluster-name: "${{ needs.env-vars.outputs.e2e-node-delete-separate-commands-test-subdir }}-${{ github.run_id }}-${{ github.run_attempt }}" }
9292
- { name: "Node Add - Separate commands", npm-test-script: "test-${{ needs.env-vars.outputs.e2e-node-add-separate-commands-test-subdir }}", coverage-subdirectory: "${{ needs.env-vars.outputs.e2e-node-add-separate-commands-test-subdir }}", coverage-report-name: "${{ needs.env-vars.outputs.e2e-node-add-separate-commands-coverage-report }}", cluster-name: "${{ needs.env-vars.outputs.e2e-node-add-separate-commands-test-subdir }}-${{ github.run_id }}-${{ github.run_attempt }}" }
9393
- { name: "Node Upgrade - Separate commands", npm-test-script: "test-${{ needs.env-vars.outputs.e2e-node-upgrade-separate-commands-test-subdir }}", coverage-subdirectory: "${{ needs.env-vars.outputs.e2e-node-upgrade-separate-commands-test-subdir }}", coverage-report-name: "${{ needs.env-vars.outputs.e2e-node-upgrade-separate-commands-coverage-report }}", cluster-name: "${{ needs.env-vars.outputs.e2e-node-upgrade-separate-commands-test-subdir }}-${{ github.run_id }}-${{ github.run_attempt }}" }
94-
max-parallel: 3
94+
max-parallel: 5
9595
with:
9696
custom-job-label: ${{ matrix.e2e-test-type.name }}
9797
npm-test-script: ${{ matrix.e2e-test-type.npm-test-script }}

‎.github/workflows/flow-task-test.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,9 @@ jobs:
6767
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
6868
with:
6969
install_only: true
70-
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
71-
version: v0.21.0
72-
kubectl_version: v1.28.6
70+
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
71+
version: v0.26.0
72+
kubectl_version: v1.31.4
7373
verbosity: 3
7474
wait: 120s
7575

‎.github/workflows/script/solo_smoke_test.sh

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,17 @@ function start_sdk_test ()
8989
function check_monitor_log()
9090
{
9191
# get the logs of mirror-monitor
92-
kubectl get pods -n solo-e2e | grep mirror-monitor | awk '{print $1}' | xargs kubectl logs -n solo-e2e > mirror-monitor.log
92+
kubectl get pods -n solo-e2e | grep mirror-monitor | awk '{print $1}' | xargs -IPOD kubectl logs -n solo-e2e POD > mirror-monitor.log
9393

9494
if grep -q "ERROR" mirror-monitor.log; then
9595
echo "mirror-monitor.log contains ERROR"
96+
97+
echo "------- BEGIN LOG DUMP -------"
98+
echo
99+
cat mirror-monitor.log
100+
echo
101+
echo "------- END LOG DUMP -------"
102+
96103
exit 1
97104
fi
98105

@@ -109,9 +116,16 @@ function check_monitor_log()
109116

110117
function check_importer_log()
111118
{
112-
kubectl get pods -n solo-e2e | grep mirror-importer | awk '{print $1}' | xargs kubectl logs -n solo-e2e > mirror-importer.log
119+
kubectl get pods -n solo-e2e | grep mirror-importer | awk '{print $1}' | xargs -IPOD kubectl logs -n solo-e2e POD > mirror-importer.log
113120
if grep -q "ERROR" mirror-importer.log; then
114121
echo "mirror-importer.log contains ERROR"
122+
123+
echo "------- BEGIN LOG DUMP -------"
124+
echo
125+
cat mirror-importer.log
126+
echo
127+
echo "------- END LOG DUMP -------"
128+
115129
exit 1
116130
fi
117131
}

‎.github/workflows/script/write_diag_script.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
set -eo pipefail
3+
4+
SCRIPT_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
5+
readonly SCRIPT_PATH
6+
7+
readonly DIAG_SCRIPT_PATH="${HOME}/diagnostics.sh"
8+
9+
cat <<EOF >"${DIAG_SCRIPT_PATH}"
10+
#!/usr/bin/env bash
11+
set -eo pipefail
12+
export PATH="${PATH}"
13+
export KUBECONFIG="${KUBECONFIG}"
14+
export GITHUB_WORKSPACE="${GITHUB_WORKSPACE}"
15+
export JAVA_HOME="${JAVA_HOME}"
16+
cd "${HOME}"
17+
18+
# Install the necessary tools
19+
wget https://github.com/derailed/k9s/releases/download/v0.32.7/k9s_linux_amd64.deb
20+
apt update
21+
apt install -y ./k9s_linux_amd64.deb
22+
rm -f ./k9s_linux_amd64.deb
23+
24+
EOF
25+
26+
sudo chmod +x "${DIAG_SCRIPT_PATH}"
27+
echo "Wrote diagnostics script: ${DIAG_SCRIPT_PATH}"

‎.github/workflows/zxc-e2e-test.yaml

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ permissions:
6969

7070
env:
7171
#CG_EXEC: export R_UID=$(id -u); CGROUP_LOGLEVEL=DEBUG cgexec -g cpu,memory:user.slice/user-${R_UID}.slice/user@${R_UID}.service/e2e-${{ github.run_id }} --sticky ionice -c 2 -n 2 nice -n 19
72-
CG_EXEC: ""
72+
CG_EXEC: "ionice -c 2 -n 2 nice -n 19"
73+
SOLO_CLUSTER_DUALITY: 1
7374

7475
jobs:
7576
e2e-test:
@@ -132,19 +133,26 @@ jobs:
132133

133134
- name: Install wget
134135
run: |
135-
sudo apt-get update
136-
sudo apt-get install -y wget
136+
if ! command -v wget >/dev/null 2>&1; then
137+
sudo apt-get update
138+
sudo apt-get install -y wget
139+
fi
137140
138141
- name: Setup Kind
139142
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
140143
with:
141144
install_only: true
142-
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
143-
version: v0.21.0
144-
kubectl_version: v1.28.6
145+
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
146+
version: v0.26.0
147+
kubectl_version: v1.31.4
145148
verbosity: 3
146149
wait: 120s
147150

151+
- name: Setup Helm
152+
uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
153+
with:
154+
version: "v3.17.1" # helm version
155+
148156
- name: Install Dependencies
149157
id: npm-deps
150158
run: npm ci
@@ -153,7 +161,7 @@ jobs:
153161
run: npm run build
154162

155163
- name: Pull Kind Docker Image
156-
run: docker image pull kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
164+
run: docker image pull kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
157165

158166
- name: Setup Java
159167
if: ${{ runner.os == 'linux' && (inputs.npm-test-script == 'test-e2e-node-local-hedera' || inputs.npm-test-script == 'test-e2e-node-local-ptt' || inputs.npm-test-script == 'test-e2e-node-add-local') && !cancelled() && !failure() }}
@@ -168,6 +176,13 @@ jobs:
168176
with:
169177
cache-read-only: false
170178

179+
- name: Create Diagnostic Script
180+
env:
181+
WRITE_DIAG_SCRIPT: ${{ github.workspace }}/.github/workflows/script/write_diag_script.sh
182+
run: |
183+
sudo chmod +x "${WRITE_DIAG_SCRIPT}"
184+
${WRITE_DIAG_SCRIPT}
185+
171186
- name: Build Hedera code locally
172187
if: ${{ runner.os == 'linux' && (inputs.npm-test-script == 'test-e2e-node-local-hedera' || inputs.npm-test-script == 'test-e2e-node-local-ptt' || inputs.npm-test-script == 'test-e2e-node-add-local') && !cancelled() && !failure() }}
173188
run: |
@@ -182,12 +197,16 @@ jobs:
182197
run: |
183198
npm link
184199
echo "SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}"
185-
${{ env.CG_EXEC }} SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ./test/e2e/setup-e2e.sh
200+
SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ${{ env.CG_EXEC }} ./test/e2e/dual-cluster/setup-dual-e2e.sh
186201
187202
- name: Run E2E Tests
188203
run: |
189204
echo "SOLO_TEST_CLUSTER=${{ inputs.cluster-name }}"
190-
${{ env.CG_EXEC }} SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} npm run ${{ inputs.npm-test-script }}
205+
SOLO_TEST_CLUSTER=${{ inputs.cluster-name }} ${{ env.CG_EXEC }} npm run ${{ inputs.npm-test-script }}
206+
207+
# - name: Sleep on Failure
208+
# if: ${{ failure() }}
209+
# run: sleep 3600
191210

192211
- name: Upload E2E Logs to GitHub
193212
if: ${{ !cancelled() }}

‎.github/workflows/zxc-update-readme.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ jobs:
8383
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
8484
with:
8585
install_only: true
86-
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
87-
version: v0.21.0
88-
kubectl_version: v1.28.6
86+
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
87+
version: v0.26.0
88+
kubectl_version: v1.31.4
8989
verbosity: 3
9090
wait: 120s
9191

‎test/e2e/dual-cluster/diagnostics/cluster/deploy.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
set -eo pipefail
3+
4+
SCRIPT_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
5+
readonly SCRIPT_PATH
6+
7+
kubectl apply -f "${SCRIPT_PATH}/manifest.yaml"

‎test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
---
2+
apiVersion: v1
3+
kind: Namespace
4+
metadata:
5+
name: cluster-diagnostics
6+
---
7+
apiVersion: v1
8+
kind: ConfigMap
9+
metadata:
10+
name: cluster-diagnostics-cm
11+
namespace: cluster-diagnostics
12+
data:
13+
entrypoint.sh: |
14+
#!/usr/bin/env bash
15+
set -eo pipefail
16+
17+
uid="$(id -u)"
18+
19+
SUDO=""
20+
if [[ "${uid}" -ne 0 ]]; then
21+
if ! command -v sudo >/dev/null 2>&1; then
22+
echo "FATAL: sudo is required to run this script as a non-root user"
23+
exit 1
24+
fi
25+
26+
SUDO="$(command -v sudo)"
27+
fi
28+
29+
export DEBIAN_FRONTEND=noninteractive
30+
31+
${SUDO} apt update
32+
${SUDO} apt upgrade -y
33+
${SUDO} apt install -y curl ca-certificates jq netcat-traditional \
34+
dnsutils iperf3 iputils-ping iproute2 tcpdump iputils-tracepath socat
35+
36+
[[ -d /app ]] || ${SUDO} mkdir -p /app
37+
38+
${SUDO} iperf3 -p 8081 -s --timestamps &
39+
${SUDO} socat -lh -lu -v TCP4-LISTEN:8080,fork EXEC:cat &
40+
exec sleep infinity
41+
---
42+
apiVersion: v1
43+
kind: Service
44+
metadata:
45+
name: cluster-diagnostics-svc
46+
namespace: cluster-diagnostics
47+
annotations:
48+
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
49+
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
50+
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
51+
networking.gke.io/load-balancer-type: "Internal"
52+
spec:
53+
selector:
54+
app: cluster-diagnostics
55+
type: LoadBalancer
56+
externalTrafficPolicy: Cluster
57+
internalTrafficPolicy: Cluster
58+
ports:
59+
- name: socat
60+
port: 8080
61+
targetPort: 8080
62+
protocol: TCP
63+
- name: iperf3
64+
port: 8081
65+
targetPort: 8081
66+
protocol: TCP
67+
---
68+
apiVersion: apps/v1
69+
kind: Deployment
70+
metadata:
71+
name: cluster-diagnostics
72+
namespace: cluster-diagnostics
73+
spec:
74+
replicas: 1
75+
selector:
76+
matchLabels:
77+
app: cluster-diagnostics
78+
template:
79+
metadata:
80+
labels:
81+
app: cluster-diagnostics
82+
spec:
83+
containers:
84+
- name: cluster-diagnostics
85+
image: ubuntu:noble
86+
command: ["/bin/bash", "/app/entrypoint.sh"]
87+
volumeMounts:
88+
- name: entrypoint
89+
mountPath: /app/entrypoint.sh
90+
subPath: entrypoint.sh
91+
ports:
92+
- containerPort: 8080
93+
name: socat
94+
protocol: TCP
95+
- containerPort: 8081
96+
name: iperf3
97+
protocol: TCP
98+
volumes:
99+
- name: entrypoint
100+
configMap:
101+
name: cluster-diagnostics-cm

‎test/e2e/dual-cluster/kind-cluster-1.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
nodes:
4+
- role: control-plane
5+
kubeadmConfigPatches:
6+
- |
7+
kind: ClusterConfiguration
8+
networking:
9+
dnsDomain: "cluster.local"
10+
networking:
11+
# apiServerAddress: "172.19.0.2"
12+
# apiServerPort: 6443
13+
podSubnet: "10.10.0.0/16"
14+
serviceSubnet: "10.20.0.0/16"
15+
containerdConfigPatches:
16+
- |-
17+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
18+
endpoint = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]

‎test/e2e/dual-cluster/kind-cluster-2.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
nodes:
4+
- role: control-plane
5+
kubeadmConfigPatches:
6+
- |
7+
kind: ClusterConfiguration
8+
networking:
9+
dnsDomain: "cluster.local"
10+
networking:
11+
# apiServerAddress: "172.19.0.3"
12+
# apiServerPort: 6443
13+
podSubnet: "10.30.0.0/16"
14+
serviceSubnet: "10.40.0.0/16"
15+
containerdConfigPatches:
16+
- |-
17+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
18+
endpoint = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]

‎test/e2e/dual-cluster/metallb-cluster-1.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: metallb.io/v1beta1
3+
kind: IPAddressPool
4+
metadata:
5+
name: local
6+
namespace: metallb-system
7+
spec:
8+
addresses:
9+
- 172.19.1.0/24
10+
---
11+
apiVersion: metallb.io/v1beta1
12+
kind: L2Advertisement
13+
metadata:
14+
name: local
15+
namespace: metallb-system
16+
spec:
17+
ipAddressPools:
18+
- local
19+
nodeSelectors:
20+
- matchLabels:
21+
kubernetes.io/os: linux
22+

‎test/e2e/dual-cluster/metallb-cluster-2.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: metallb.io/v1beta1
3+
kind: IPAddressPool
4+
metadata:
5+
name: local
6+
namespace: metallb-system
7+
spec:
8+
addresses:
9+
- 172.19.2.0/24
10+
---
11+
apiVersion: metallb.io/v1beta1
12+
kind: L2Advertisement
13+
metadata:
14+
name: local
15+
namespace: metallb-system
16+
spec:
17+
ipAddressPools:
18+
- local
19+
nodeSelectors:
20+
- matchLabels:
21+
kubernetes.io/os: linux
22+

0 commit comments

Comments
 (0)