Skip to content

feat: add support for multi-cluster testing via KinD #1392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 14, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/flow-gcs-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ jobs:
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
with:
install_only: true
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
version: v0.21.0
kubectl_version: v1.28.6
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
version: v0.26.0
kubectl_version: v1.31.4
verbosity: 3
wait: 120s

Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/flow-task-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ jobs:
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
with:
install_only: true
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
version: v0.21.0
kubectl_version: v1.28.6
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
version: v0.26.0
kubectl_version: v1.31.4
verbosity: 3
wait: 120s

Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/zxc-e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,9 @@ jobs:
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
with:
install_only: true
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
version: v0.21.0
kubectl_version: v1.28.6
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
version: v0.26.0
kubectl_version: v1.31.4
verbosity: 3
wait: 120s

Expand All @@ -153,7 +153,7 @@ jobs:
run: npm run build

- name: Pull Kind Docker Image
run: docker image pull kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
run: docker image pull kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30

- name: Setup Java
if: ${{ runner.os == 'linux' && (inputs.npm-test-script == 'test-e2e-node-local-hedera' || inputs.npm-test-script == 'test-e2e-node-local-ptt' || inputs.npm-test-script == 'test-e2e-node-add-local') && !cancelled() && !failure() }}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/zxc-update-readme.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,9 @@ jobs:
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
with:
install_only: true
node_image: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
version: v0.21.0
kubectl_version: v1.28.6
node_image: kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30
version: v0.26.0
kubectl_version: v1.31.4
verbosity: 3
wait: 120s

Expand Down
7 changes: 7 additions & 0 deletions test/e2e/dual-cluster/diagnostics/cluster/deploy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -eo pipefail

SCRIPT_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
readonly SCRIPT_PATH

kubectl apply -f "${SCRIPT_PATH}/manifest.yaml"
101 changes: 101 additions & 0 deletions test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: cluster-diagnostics
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cluster-diagnostics-cm
namespace: cluster-diagnostics
data:
entrypoint.sh: |
#!/usr/bin/env bash
set -eo pipefail

uid="$(id -u)"

SUDO=""
if [[ "${uid}" -ne 0 ]]; then
if ! command -v sudo >/dev/null 2>&1; then
echo "FATAL: sudo is required to run this script as a non-root user"
exit 1
fi

SUDO="$(command -v sudo)"
fi

export DEBIAN_FRONTEND=noninteractive

${SUDO} apt update
${SUDO} apt upgrade -y
${SUDO} apt install -y curl ca-certificates jq netcat-traditional \
dnsutils iperf3 iputils-ping iproute2 tcpdump iputils-tracepath socat

[[ -d /app ]] || ${SUDO} mkdir -p /app

${SUDO} iperf3 -p 8081 -s --timestamps &
${SUDO} socat -lh -lu -v TCP4-LISTEN:8080,fork EXEC:cat &
exec sleep infinity
---
apiVersion: v1
kind: Service
metadata:
name: cluster-diagnostics-svc
namespace: cluster-diagnostics
annotations:
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
networking.gke.io/load-balancer-type: "Internal"
spec:
selector:
app: cluster-diagnostics
type: LoadBalancer
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ports:
- name: socat
port: 8080
targetPort: 8080
protocol: TCP
- name: iperf3
port: 8081
targetPort: 8081
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cluster-diagnostics
namespace: cluster-diagnostics
spec:
replicas: 1
selector:
matchLabels:
app: cluster-diagnostics
template:
metadata:
labels:
app: cluster-diagnostics
spec:

Check notice on line 82 in test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml#L82

When running containers in Kubernetes, it's important to ensure that they are properly secured to prevent privilege escalation attacks.
containers:
- name: cluster-diagnostics

Check warning on line 84 in test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

test/e2e/dual-cluster/diagnostics/cluster/manifest.yaml#L84

In Kubernetes, each pod runs in its own isolated environment with its own set of security policies.
image: ubuntu:noble
command: ["/bin/bash", "/app/entrypoint.sh"]
volumeMounts:
- name: entrypoint
mountPath: /app/entrypoint.sh
subPath: entrypoint.sh
ports:
- containerPort: 8080
name: socat
protocol: TCP
- containerPort: 8081
name: iperf3
protocol: TCP
volumes:
- name: entrypoint
configMap:
name: cluster-diagnostics-cm
18 changes: 18 additions & 0 deletions test/e2e/dual-cluster/kind-cluster-1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
networking:
dnsDomain: "cluster1.local"
networking:
# apiServerAddress: "172.19.0.2"
# apiServerPort: 6443
podSubnet: "10.10.0.0/16"
serviceSubnet: "10.20.0.0/16"
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]
18 changes: 18 additions & 0 deletions test/e2e/dual-cluster/kind-cluster-2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
networking:
dnsDomain: "cluster2.local"
networking:
# apiServerAddress: "172.19.0.3"
# apiServerPort: 6443
podSubnet: "10.30.0.0/16"
serviceSubnet: "10.40.0.0/16"
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]
22 changes: 22 additions & 0 deletions test/e2e/dual-cluster/metallb-cluster-1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: local
namespace: metallb-system
spec:
addresses:
- 172.19.1.0/24
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: local
namespace: metallb-system
spec:
ipAddressPools:
- local
nodeSelectors:
- matchLabels:
kubernetes.io/os: linux

22 changes: 22 additions & 0 deletions test/e2e/dual-cluster/metallb-cluster-2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: local
namespace: metallb-system
spec:
addresses:
- 172.19.2.0/24
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: local
namespace: metallb-system
spec:
ipAddressPools:
- local
nodeSelectors:
- matchLabels:
kubernetes.io/os: linux

60 changes: 60 additions & 0 deletions test/e2e/dual-cluster/setup-dual-e2e.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
set -eo pipefail

##### Setup Environment #####
SCRIPT_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
readonly SCRIPT_PATH

readonly CLUSTER_DIAGNOSTICS_PATH="${SCRIPT_PATH}/diagnostics/cluster"
readonly KIND_IMAGE="kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30"

echo "SOLO_CHARTS_DIR: ${SOLO_CHARTS_DIR}"
export PATH=${PATH}:~/.solo/bin

if [[ -n "${SOLO_TEST_CLUSTER}" ]]; then
SOLO_CLUSTER_NAME="${SOLO_TEST_CLUSTER}"
elif [[ -z "${SOLO_CLUSTER_NAME}" ]]; then
SOLO_CLUSTER_NAME="solo-e2e"
fi

for i in {1..2}; do
kind delete cluster -n "${SOLO_CLUSTER_NAME}-c${i}" || true
done

docker network rm kind
docker network create kind --scope local --subnet 172.19.0.0/16 --driver bridge

# Setup Helm Repos
helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
helm repo add metallb https://metallb.github.io/metallb

for i in {1..2}; do
kind create cluster -n "${SOLO_CLUSTER_NAME}-c${i}" --image "${KIND_IMAGE}" --config "${SCRIPT_PATH}/kind-cluster-${i}.yaml" || exit 1
helm upgrade --install metrics-server metrics-server/metrics-server \
--namespace kube-system --wait --atomic \
--set "args[0]=--kubelet-insecure-tls"

helm upgrade --install metallb metallb/metallb \
--namespace metallb-system --create-namespace --wait --atomic \
--set speaker.frr.enabled=true

kubectl apply -f "${SCRIPT_PATH}/metallb-cluster-${i}.yaml"

"${CLUSTER_DIAGNOSTICS_PATH}"/deploy.sh
done

# **********************************************************************************************************************
# Warm up the cluster
# **********************************************************************************************************************
# source test/data/warmup-cluster.sh; download_images; load_images

# **********************************************************************************************************************
# Init and deploy a network for e2e tests in (test/e2e/core)
# -d ${SOLO_CHARTS_DIR} is optional, if you want to use a local chart, it will be ignored if not set
# **********************************************************************************************************************
# SOLO_CLUSTER_SETUP_NAMESPACE=solo-setup
# npm run build
#npm run solo -- init || exit 1 # cache args for subsequent commands
#npm run solo -- cluster setup -s "${SOLO_CLUSTER_SETUP_NAMESPACE}" || exit 1
#helm list --all-namespaces
#sleep 10 # give time for solo-setup to finish deploying
2 changes: 1 addition & 1 deletion test/e2e/setup-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ SCRIPT_PATH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
readonly SCRIPT_PATH

readonly KIND_CONFIG_FILE="${SCRIPT_PATH}/kind-cluster.yaml"
readonly KIND_IMAGE="kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72"
readonly KIND_IMAGE="kindest/node:v1.31.4@sha256:2cb39f7295fe7eafee0842b1052a599a4fb0f8bcf3f83d96c7f4864c357c6c30"
echo "SOLO_CHARTS_DIR: ${SOLO_CHARTS_DIR}"
export PATH=${PATH}:~/.solo/bin

Expand Down
Loading