ROX-27073: github action for multicluster test (#2158)

* implemented ginkgo e2e test for cluster migration

* small bugfixes for test implementation, tested against infra

* chore: upgrade stackrox go.mod ref to 4.6.1 (#2142)

upgrade stackrox go.mod ref to 4.6.1

* add GHA definition for multicluster test workflow

* adjust setup and test scripts to make e2e test setup succeed

* PR feedback

Author: johannes94

.github/workflows/multicluster-e2e.yaml

@@ -0,0 +1,117 @@
+name: Multicluster E2E tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'fleetshard/pkg/runtime/**'
+      - 'fleetshard/pkg/reconciler/**'
+      - '.github/workflows/multicluster-e2e.yaml'
+      - 'scripts/ci/**'
+      - 'scripts/lib/**'
+      - 'internal/pkg/handlers/admin_dinosaur.go'
+      - 'internal/dinosaur/pkg/services/dinosaur.go'
+
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - 'fleetshard/pkg/runtime/**'
+      - 'fleetshard/pkg/reconciler/**'
+      - '.github/workflows/multicluster-e2e.yaml'
+      - 'scripts/ci/**'
+      - 'scripts/lib/**'
+      - 'internal/pkg/handlers/admin_dinosaur.go'
+      - 'internal/dinosaur/pkg/services/dinosaur.go'
+
+jobs:
+  create-cluster:
+    name: "Create Test Infra Clusters"
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }} # do not run for PRs from forks and drafts
+    environment: development
+    strategy:
+      matrix:
+       name: [acscs1, acscs2]
+    steps:
+      - name: Create cluster
+        uses: stackrox/actions/infra/create-cluster@v1
+        with:
+          token: ${{ secrets.INFRA_TOKEN }}
+          flavor: osd-on-aws
+          name: ${{ matrix.name }}-${{ github.run_id }}${{ github.run_attempt }}
+          lifespan: 3h
+          args: nodes=3,machine-type=m5.2xlarge
+          wait: true
+
+  e2e-test:
+    name: "Multicluster e2e tests"
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }} # do not run for PRs from forks and drafts
+    needs: [create-cluster]
+    environment: development
+    env:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+      AWS_AUTH_HELPER: "none"
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+      - name: Install oc
+        uses: redhat-actions/oc-installer@v1
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Set cluster credentials
+        run: |
+          set -eo pipefail
+          mkdir kube
+          cluster1Conf="$(pwd)/kube/cluster1"
+          url=$(infractl artifacts "acscs1-${{ github.run_id }}${{ github.run_attempt }}" --json | jq '.Artifacts[] | select(.Name=="kubeconfig") | .URL' -r)
+          wget -O "$cluster1Conf" "$url"
+
+          cluster2Conf="$(pwd)/kube/cluster2"
+          url=$(infractl artifacts "acscs2-${{ github.run_id }}${{ github.run_attempt }}" --json | jq '.Artifacts[] | select(.Name=="kubeconfig") | .URL' -r)
+          wget -O "$cluster2Conf" "$url"
+
+          echo "CLUSTER_1_KUBECONFIG=$cluster1Conf" >> "$GITHUB_ENV"
+          echo "CLUSTER_2_KUBECONFIG=$cluster2Conf" >> "$GITHUB_ENV"
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github
+      - name: Set registry.redhat.io credentials
+        run: |
+          set -eo pipefail
+
+          KUBECONFIG=$CLUSTER_1_KUBECONFIG oc get secret/pull-secret -n openshift-config --template='{{index .data ".dockerconfigjson" | base64decode}}' > dockercfg
+          creds=$(jq '.auths."registry.redhat.io".auth' -r < dockercfg | base64 -d)
+          user=$(echo "$creds" | cut -d':' -f1)
+          pw=$(echo "$creds" | cut -d':' -f2)
+          echo "RH_REGISTRY_USER=$user" >> "$GITHUB_ENV"
+          echo "RH_REGISTRY_PW=$pw" >> "$GITHUB_ENV"
+      - name: "Run"
+        env:
+          RUN_MULTICLUSTER_E2E: "true"
+          ENABLE_CENTRAL_EXTERNAL_CERTIFICATE: "true"
+        run: "scripts/ci/multicluster_tests/entrypoint.sh"
+
+  cleanup-clusters:
+    name: "Cleanup Test Infra Clusters"
+    runs-on: ubuntu-latest
+    needs: [create-cluster, e2e-test]
+    if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft && always() }} # do not run for PRs from forks
+    environment: development
+    env:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+    steps:
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+      - name: Delete test clusters
+        run: |
+          set -o pipefail
+          infractl delete "acscs1-${{ github.run_id }}${{ github.run_attempt }}"
+          infractl delete "acscs2-${{ github.run_id }}${{ github.run_attempt }}"
+          exit 0

.secrets.baseline

@@ -259,7 +259,7 @@
         "filename": "e2e/e2e_test.go",
         "hashed_secret": "7f38822bc2b03e97325ff310099f457f6f788daf",
         "is_verified": false,
-        "line_number": 298
+        "line_number": 299
       }
     ],
     "internal/dinosaur/pkg/api/public/api/openapi.yaml": [
@@ -416,5 +416,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-22T08:55:38Z"
+  "generated_at": "2025-01-22T14:22:15Z"
 }

Makefile

@@ -374,7 +374,7 @@ test/e2e/multicluster: $(GINKGO_BIN)
 	CLUSTER_ID=1234567890abcdef1234567890abcdef \
 	ENABLE_CENTRAL_EXTERNAL_CERTIFICATE=$(ENABLE_CENTRAL_EXTERNAL_CERTIFICATE) \
 	GITOPS_CONFIG_PATH=$(GITOPS_CONFIG_FILE) \
-	RUN_MULTICLUSTER_E2E=true
+	RUN_MULTICLUSTER_E2E=true \
 	$(GINKGO_BIN) -r $(GINKGO_FLAGS) \
 		--randomize-suites \
 		--fail-on-pending --keep-going \
@@ -383,7 +383,7 @@ test/e2e/multicluster: $(GINKGO_BIN)
 		--json-report=e2e-report.json \
 		--timeout=$(TEST_TIMEOUT) \
 		--poll-progress-after=5m \
-		 ./e2e/multicluster
+		 ./e2e/multicluster/...
 .PHONY: test/e2e/multicluster
 
 # Deploys the necessary applications to the selected cluster and runs e2e tests inside the container
@@ -525,9 +525,14 @@ db/generate/insert/cluster:
 
 # Login to the OpenShift internal registry
 docker/login/internal:
-	$(DOCKER) login -u kubeadmin --password-stdin <<< $(shell oc whoami -t) $(shell oc get route default-route -n openshift-image-registry -o jsonpath="{.spec.host}")
+	@$(DOCKER) login -u kubeadmin --password-stdin <<< $(shell oc whoami -t) $(shell oc get route default-route -n openshift-image-registry -o jsonpath="{.spec.host}")
 .PHONY: docker/login/internal
 
+# Login to registry.redhat.io
+docker/login/rh-registry:
+	@$(DOCKER) login -u "${RH_REGISTRY_USER}" --password-stdin <<< "${RH_REGISTRY_PW}" registry.redhat.io
+.PHONY: docker/login/rh-registry
+
 # Build the image
 image/build:
 	$(DOCKER) buildx build -t $(SHORT_IMAGE_REF) . --load
@@ -798,7 +803,7 @@ endif
 		-p QUOTA_TYPE="${QUOTA_TYPE}" \
 		-p DATAPLANE_CLUSTER_SCALING_TYPE="${DATAPLANE_CLUSTER_SCALING_TYPE}" \
 		-p CENTRAL_REQUEST_EXPIRATION_TIMEOUT="${CENTRAL_REQUEST_EXPIRATION_TIMEOUT}" \
-		-p CLUSTER_LIST='$(shell make cluster-list)' \
+		-p CLUSTER_LIST='$(shell make -s cluster-list)' \
 		-p ENABLE_HTTPS="$(ENABLE_HTTPS)" \
 		-p HEALTH_CHECK_SCHEME="$(HEALTH_CHECK_SCHEME)" \
 		-p CPU_REQUEST="$(CPU_REQUEST)" \

dev/env/defaults/cluster-type-infra-openshift/env

@@ -3,3 +3,4 @@ export EXPOSE_OPENSHIFT_ROUTER_DEFAULT="true"
 export ENABLE_EXTERNAL_CONFIG_DEFAULT="true"
 export AWS_AUTH_HELPER_DEFAULT="aws-saml"
 export INHERIT_IMAGEPULLSECRETS_DEFAULT="true" # pragma: allowlist secret
+export ENABLE_CENTRAL_EXTERNAL_CERTIFICATE_DEFAULT="true"

dev/env/scripts/docker.sh

@@ -70,9 +70,12 @@ ensure_fleetshard_operator_image_exists() {
     fi
 
     if [[ -z "${FLEETSHARD_OPERATOR_IMAGE:-}" ]]; then
-        FLEETSHARD_OPERATOR_IMAGE="fleetshard-operator:$(make tag)"
+        FLEETSHARD_OPERATOR_IMAGE="fleetshard-operator:$(make -s tag)"
         export FLEETSHARD_OPERATOR_IMAGE
         if [[ "$CLUSTER_TYPE" == "infra-openshift" ]]; then
+            if [[ -n "$RH_REGISTRY_USER" && -n "$RH_REGISTRY_PW" ]]; then
+                make -C "${GITROOT}" docker/login/rh-registry
+            fi
             log "Building fleetshard operator image ${FLEETSHARD_OPERATOR_IMAGE} and pushing it to internal registry"
             make -C "${GITROOT}" image/push/fleetshard-operator/internal
         else

e2e/e2e_test.go

@@ -6,12 +6,13 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/gitops"
 	"net"
 	"net/url"
 	"os"
 	"time"
 
+	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/gitops"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"

e2e/testutil/testutil.go

@@ -41,7 +41,7 @@ func DNSConfiguration(routesEnabled bool) (dnsEnabled bool, accessKey string, se
 	enableExternal := os.Getenv("ENABLE_CENTRAL_EXTERNAL_CERTIFICATE")
 	dnsEnabled = accessKey != "" &&
 		secretKey != "" &&
-		enableExternal != "" && routesEnabled
+		enableExternal == "true" && routesEnabled
 	return dnsEnabled, accessKey, secretKey
 }
 

fleetshard/pkg/central/reconciler/reconciler.go

@@ -8,11 +8,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sort"
 	"sync/atomic"
 	"time"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
 	"github.com/golang/glog"
 	"github.com/hashicorp/go-multierror"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"

scripts/ci/multicluster_tests/deploy.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -eo pipefail
 # This script assumes that there were two clusters created before execution.
 # It expects that those clusters are accessible through kubeconfig files at the path
 # value stored in following environment variables:
@@ -10,13 +10,9 @@ export CLUSTER_2_KUBECONFIG=${CLUSTER_2_KUBECONFIG:-"$HOME/.kube/cluster2"}
 
 # Bootstrap C1
 export KUBECONFIG="$CLUSTER_1_KUBECONFIG"
+export INHERIT_IMAGEPULLSECRETS="true" # pragma: allowlist secret
+export ENABLE_CENTRAL_EXTERNAL_CERTIFICATE="true"
 
-# TODO: Double check how setup is done in OSCI so that we
-# Get the propper certificates to allow enabling creation of routes and DNS entries
-# Get the propper secrets to allow communication of FM to Route 53
-# Get the quay configuration to pull images
-# Maybe we wanna rely on prebuild images instead of building them ourselves, which might
-# as well need additional / other commands
 make deploy/bootstrap
 make deploy/dev
 
@@ -30,7 +26,7 @@ export FM_URL
 kubectl get cm -n rhacs fleet-manager-dataplane-cluster-scaling-config -o yaml > fm-dataplane-config.yaml
 yq '.data."dataplane-cluster-configuration.yaml"' fm-dataplane-config.yaml | yq .clusters > cluster-list.json
 
-KUBECONFIG="$CLUSTER_2_KUBECONFIG" make cluster-list \
+KUBECONFIG="$CLUSTER_2_KUBECONFIG" make -s cluster-list \
   | jq '.[0] | .name="dev2" | .cluster_id="1234567890abcdef1234567890abcdeg"' \
   | jq --slurp . > cluster-list2.json
 

scripts/ci/multicluster_tests/entrypoint.sh

@@ -9,10 +9,61 @@ source "$ROOT_DIR/scripts/ci/lib.sh"
 source "$ROOT_DIR/scripts/lib/log.sh"
 source "$ROOT_DIR/dev/env/scripts/lib.sh"
 
+acscs_namespace="rhacs"
+
+log_common_cluster_resources() {
+  local cluster_name=$1
+
+  log "***** START logging resources for: $cluster_name *****"
+  kubectl describe deploy -n "$acscs_namespace"
+  kubectl describe pods -n "$acscs_namespace"
+  kubectl get routes -n "$acscs_namespace"
+  kubectl logs -n "$acscs_namespace" --prefix --all-containers -l app=fleetshard-sync
+  log "***** END logging resources for: $cluster_name *****"
+}
+
+log_failure() {
+  local step_name=$1
+  log "$step_name Failed with status: $EXIT_CODE"
+  log "Starting to log cluster resources and container logs"
+
+  export KUBECONFIG=$CLUSTER_1_KUBECONFIG
+  log_common_cluster_resources "CLUSTER_1"
+  kubectl logs -n "$acscs_namespace" --prefix --all-containers -l app=fleet-manager
+
+  export KUBECONFIG=$CLUSTER_2_KUBECONFIG
+  log_common_cluster_resources "CLUSTER_2"
+}
+
+
+# shellcheck source=scripts/lib/external_config.sh
+source "${ROOT_DIR}/scripts/lib/external_config.sh"
+
+# Executing this with all necessary credentials stored in AWS secretsmanager on ACSCS dev account
+# Name: "github-multicluster-test"
+# created and updated only manually by ACSCS engineerse
+init_chamber
+secrets=$(chamber env "github-multicluster-test" --backend secretsmanager)
+eval "$secrets"
+
+KUBECTL="$(which kubectl)"
+export KUBECTL
+
 bash "$SOURCE_DIR/deploy.sh"
 EXIT_CODE="$?"
 if [ "$EXIT_CODE" -ne "0" ]; then
-  echo "TODO(ROX-27073): add additional logging required here, once tests are actually executed"
+  log_failure Deploy
+  stat /tmp/pids-port-forward > /dev/null 2>&1 && xargs kill < /tmp/pids-port-forward
+  exit 1
+fi
+
+FM_URL="https://$(KUBECONFIG=$CLUSTER_1_KUBECONFIG kubectl get routes -n rhacs fleet-manager -o yaml | yq .spec.host)"
+export FM_URL
+
+make test/e2e/multicluster
+EXIT_CODE="$?"
+if [ "$EXIT_CODE" -ne "0" ]; then
+  log_failure Test
 fi
 
 stat /tmp/pids-port-forward > /dev/null 2>&1 && xargs kill < /tmp/pids-port-forward