Update IPsec CI lane configuration for IPsec tests

As per changes in openshift/origin#29437 for IPsec E2E tests, each IPsec mode
Full and External must be tested separately, so this commit updates
openshift-e2e-test step with new test type called ipsec-suite and ipsec test
suite is executed under this test type for each ipsec modes.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

ci-operator/config/openshift/cluster-network-operator/openshift-cluster-network-operator-master.yaml

@@ -115,7 +115,7 @@ tests:
     env:
       EXTRA_MG_ARGS: --host-network
     workflow: openshift-e2e-aws-ovn-serial
-- always_run: false
+- always_run: true
   as: e2e-aws-ovn-ipsec-serial
   optional: true
   steps:

ci-operator/config/openshift/cluster-network-operator/openshift-cluster-network-operator-release-4.19.yaml

@@ -115,7 +115,7 @@ tests:
     env:
       EXTRA_MG_ARGS: --host-network
     workflow: openshift-e2e-aws-ovn-serial
-- always_run: false
+- always_run: true
   as: e2e-aws-ovn-ipsec-serial
   optional: true
   steps:

ci-operator/config/openshift/cluster-network-operator/openshift-cluster-network-operator-release-4.20.yaml

@@ -116,7 +116,7 @@ tests:
     env:
       EXTRA_MG_ARGS: --host-network
     workflow: openshift-e2e-aws-ovn-serial
-- always_run: false
+- always_run: true
   as: e2e-aws-ovn-ipsec-serial
   optional: true
   steps:

ci-operator/jobs/openshift/cluster-network-operator/openshift-cluster-network-operator-master-presubmits.yaml

@@ -426,7 +426,7 @@ presubmits:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-aws-ovn-hypershift-conformance,?($|\s.*)
   - agent: kubernetes
-    always_run: false
+    always_run: true
     branches:
     - ^master$
     - ^master-

ci-operator/jobs/openshift/cluster-network-operator/openshift-cluster-network-operator-release-4.19-presubmits.yaml

@@ -426,7 +426,7 @@ presubmits:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-aws-ovn-hypershift-conformance,?($|\s.*)
   - agent: kubernetes
-    always_run: false
+    always_run: true
     branches:
     - ^release-4\.19$
     - ^release-4\.19-

ci-operator/jobs/openshift/cluster-network-operator/openshift-cluster-network-operator-release-4.20-presubmits.yaml

@@ -146,7 +146,7 @@ presubmits:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-aws-ovn-hypershift-conformance,?($|\s.*)
   - agent: kubernetes
-    always_run: false
+    always_run: true
     branches:
     - ^release-4\.20$
     - ^release-4\.20-

ci-operator/step-registry/openshift/e2e/aws/ovn/ipsec-serial/openshift-e2e-aws-ovn-ipsec-serial-workflow.yaml

@@ -17,8 +17,7 @@ workflow:
       - chain: ipi-deprovision
     env:
       NMSTATE_OPERATOR_SUB_SOURCE: qe-app-registry
-      TEST_SUITE: openshift/network/ipsec
-      TEST_ARGS: --run \[sig-network\]\[Feature:IPsec\]
+      TEST_TYPE: ipsec-suite
   documentation: |-
     The Openshift E2E AWS `ipsec serial` workflow executes the `openshift/network/ipsec` end-to-end
     IPsec test suite on AWS with the OVNKubernetes network plugin.

ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh

@@ -331,6 +331,38 @@ function suite() {
     set +x
 }
 
+function wait_for_ipsec_full_mode() {
+  until
+    timeout 30s oc rollout status daemonset/ovn-ipsec-host -n openshift-ovn-kubernetes && \
+    oc wait --for=delete daemonset/ovn-ipsec-containerized -n openshift-ovn-kubernetes --timeout=30s;
+  do
+    echo "ovn-ipsec-host daemonset is not available yet (or) ovn-ipsec-containerized daemonset is still deployed"
+    sleep 30s
+  done
+  wait_for_cluster_operators_ready
+}
+
+function wait_for_ipsec_external_mode() {
+  until
+    oc wait --for=delete daemonset/ovn-ipsec-host -n openshift-ovn-kubernetes --timeout=30s;
+  do
+    echo "ovn-ipsec-host daemonset is not removed yet"
+    sleep 30s
+  done
+  wait_for_cluster_operators_ready
+}
+
+function wait_for_cluster_operators_ready() {
+  until
+    oc wait clusteroperators --all --for='condition=Available=True' --timeout=30s && \
+    oc wait clusteroperators --all --for='condition=Progressing=False' --timeout=30s && \
+    oc wait clusteroperators --all --for='condition=Degraded=False' --timeout=30s;
+  do
+    echo "Cluster Operators Degraded=True,Progressing=True,or Available=False"
+    sleep 30s
+  done
+}
+
 echo "$(date +%s)" > "${SHARED_DIR}/TEST_TIME_TEST_START"
 
 oc -n openshift-config patch cm admin-acks --patch '{"data":{"ack-4.8-kube-1.22-api-removals-in-4.9":"true"}}' --type=merge || echo 'failed to ack the 4.9 Kube v1beta1 removals; possibly API-server issue, or a pre-4.8 release image'
@@ -516,6 +548,21 @@ suite-conformance)
 suite)
     suite
     ;;
+ipsec-suite)
+     # Rollout IPsec Full mode and run the suite.
+     echo "Rolling out IPsec Full mode"
+     oc patch networks.operator.openshift.io cluster --type=merge -p='{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipsecConfig":{"mode":"Full"}}}}}'
+     wait_for_ipsec_full_mode
+     echo "IPsec Full mode rollout complete. running IPsec test suite now"
+     TEST_SUITE=openshift/network/ipsec TEST_ARGS="--run \[sig-network\]\[Feature:IPsec\]" suite
+
+     # Rollout IPsec External mode and run the suite.
+     echo "Rolling out IPsec External mode"
+     oc patch networks.operator.openshift.io cluster --type=merge -p='{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipsecConfig":{"mode":"External"}}}}}'
+     wait_for_ipsec_external_mode
+     echo "IPsec External mode rollout complete. running IPsec test suite now"
+     TEST_SUITE=openshift/network/ipsec TEST_ARGS="--run \[sig-network\]\[Feature:IPsec\]" suite
+    ;;
 *)
     echo >&2 "Unsupported test type '${TEST_TYPE}'"
     exit 1

ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-ref.yaml

@@ -35,6 +35,9 @@ ref:
                   an upgrade completes or have explicit logic in their test to tolerate behavior after
                   upgrade.
       * 'upgrade-paused' - Perform an upgrade to the images defined by OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE with paused worker pool.
+      * 'ipsec-suite' - Runs IPsec test suite. The test suite is invoked separately for IPsec Full and External modes.
+                   1. Rollout IPsec Full mode, wait for it to complete and run the ipsec test suite.
+                   2. Change IPsec Full -> External mode, wait for rollout to complete, run the ipsec test suite.
   - name: TEST_SUITE
     default: openshift/conformance/parallel
     documentation: The test suite to run.  Use 'openshift-test run --help' to list available suites.