integration.yml

name: Deploy APL
on:
  workflow_call:
    inputs:
      linode_types:
        description: 'Linode instance types'
        type: string
        default: g6-dedicated-8
      kubernetes_versions:
        description: 'Kubernetes version'
        type: string
        default: "['1.31']"
      install_profile:
        description: 'APL installation profile'
        type: string
        default: minimal-with-team
      domain_zone:
        description: 'Select Domain Zone'
        type: string
        default: DNS-Integration
      kms:
        description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?'
        type: string
        default: age
      certificate:
        description: 'Select certificate issuer'
        type: string
        default: letsencrypt_production
      is_pre_installed:
        description: Fake if Otomi is pre-installed by Installer
        type: string
        default: 'false'
  workflow_dispatch:
    inputs:
      linode_types:
        description: 'Linode instance types'
        type: choice
        options:
          - g6-dedicated-4
          - g6-dedicated-8
          - g6-dedicated-16
          - g6-dedicated-32
          - g6-dedicated-48
          - g6-dedicated-50
          - g6-dedicated-56
        default: g6-dedicated-4
      kubernetes_versions:
        description: 'Kubernetes version'
        type: choice
        options:
          - "['1.30']"
          - "['1.31']"
          - "['1.32']"
        default: "['1.32']"
      install_profile:
        description: APL installation profile
        default: minimal-with-team
        type: choice
        options:
          - minimal
          - minimal-with-team
          - full
          - upgrade
          - no-apl
      domain_zone:
        type: choice
        description: 'Select Domain Zone'
        options:
          - Zone-1
          - Zone-2
          - Random
          - DNS-Integration
      kms:
        type: choice
        description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)?
        options:
          - age
          - no_kms
        default: age
      certificate:
        type: choice
        description: Select certificate issuer
        options:
          - gen_custom_ca
          - letsencrypt_staging
          - letsencrypt_production
        default: letsencrypt_production
      is_pre_installed:
        type: choice
        description: Fake if Otomi is pre-installed by Installer
        options:
          - 'true'
          - 'false'
        default: 'true'

env:
  CACHE_REGISTRY: ghcr.io
  CACHE_REPO: linode/apl-core
  REPO: linode/apl-core
  GIT_USER: svcAPLBot
  CHECK_CONTEXT: continuous-integration/integration-test
  COMMIT_ID: '${{ github.event.pull_request.head.sha || github.sha }}'
  BOT_EMAIL: ${{ vars.BOT_EMAIL }}
  BOT_USERNAME: ${{ vars.BOT_USERNAME }}
  DEV_DOMAINS: ${{ secrets.DEV_DOMAINS }}

jobs:
  preprocess-input:
    name: Preprocess input variables
    runs-on: ubuntu-latest
    steps:
      - name: Print user input
        run: |
          echo 'ref: ${{ github.event.pull_request.head.ref || github.ref }}'
          echo 'install_profile: ${{ inputs.install_profile }}'
          echo 'kubernetes_versions: ${{ inputs.kubernetes_versions }}'
          echo 'kms: ${{ inputs.kms }}'
          echo 'domain_zone: ${{ inputs.domain_zone }}'
          echo 'certificate: ${{ inputs.certificate }}'
          echo 'is_pre_installed: ${{ inputs.is_pre_installed }}'

  preprocess-linode-input:
    needs: preprocess-input
    name: Preprocess input variables for linode
    runs-on: ubuntu-latest
    outputs:
      kubernetes_versions: ${{ steps.k8s-versions.outputs.versions }}
    steps:
      - name: Install the Linode CLI
        uses: linode/action-linode-cli@v1
        with:
          token: ${{ secrets.LINODE_TOKEN }}
      - name: Check if cluster is running
        run: |
          case "${{ inputs.domain_zone }}" in
            "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;;
            "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;;
            "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;;
            "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;;
          esac
          [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl
          if [[ $(linode-cli lke clusters-list --json | jq --arg name "$LINODE_CLUSTER_NAME" '[.[] | select(.label == $name)] | length > 0') == "true" ]]; then
            echo "An LKE cluster with the same name ($LINODE_CLUSTER_NAME) already exists."
            echo "Visit https://cloud.linode.com/kubernetes/clusters to delete your cluster"
            echo "Exiting workflow..."
            exit 1
          fi
      - id: k8s-versions
        name: Process k8s version input
        run: |
          if [ -z '${{ inputs.kubernetes_versions }}' ]; then
            echo "Kubernetes versions not specified, determine Linode supported versions"
            versions=`linode-cli lke versions-list --json | jq -ce '.[] | .id'`
          else
            versions='${{ inputs.kubernetes_versions }}'
          fi
          echo $versions
          echo "versions=$versions" >> $GITHUB_OUTPUT

  run-integration-test-linode:
    name: Run integration test on linode cluster
    needs: preprocess-linode-input
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        kubernetes_versions: ${{ fromJSON(needs.preprocess-linode-input.outputs.kubernetes_versions) }}
      max-parallel: 5
    steps:
      - name: Install the Linode CLI
        uses: linode/action-linode-cli@v1
        with:
          token: ${{ secrets.LINODE_TOKEN }}
      - name: Set k8s cluster name
        run: |
          case "${{ inputs.domain_zone }}" in
            "Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;;
            "Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;;
            "Random") LINODE_CLUSTER_NAME=${{ github.actor }}-$RANDOM ;;
            "DNS-Integration") LINODE_CLUSTER_NAME=apl-test-${{ inputs.install_profile }} ;;
          esac
          [[ ${{ inputs.install_profile }} == 'no-apl' ]] && LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME-no-apl
          echo LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME >> $GITHUB_ENV
      - name: Determine exact k8s version
        run: |
          echo LINODE_K8S_VERSION=$(linode-cli lke versions-list --json | jq -ce --arg version "$(echo ${{ matrix.kubernetes_versions }} | sed -E 's/^([0-9]+\.[0-9])$/\10/')" '.[] | select(.id | tostring | startswith($version)) | .id') >> $GITHUB_ENV
      - name: Determine domain name to use for scheduled integration test
        env:
          EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
        if: ${{ inputs.domain_zone == 'DNS-Integration' && inputs.install_profile != 'no-apl'}}
        run: |
          RAND=$(openssl rand -hex 4)
          DOMAIN="integration-${RAND}.${EDGEDNS_ZONE}"
          echo "::add-mask::$DOMAIN"
          echo DOMAIN=$DOMAIN >> $GITHUB_ENV
      - name: Determine domain name
        if: ${{ inputs.domain_zone != 'DNS-Integration' && inputs.install_profile != 'no-apl' }}
        env:
          EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
        run: |
          # Mapping of domain_zone to domain names
          case "${{ inputs.domain_zone }}" in
            "Zone-1") DOMAIN=$(jq -r '."${{ github.actor }}"[0]' <<< ${{ env.DEV_DOMAINS }}) ;;
            "Zone-2") DOMAIN=$(jq -r '."${{ github.actor }}"[1]' <<< ${{ env.DEV_DOMAINS }}) ;;
            "Random") DOMAIN="$(openssl rand -hex 4)$(date +"%d%m%y").${EDGEDNS_ZONE}" ;;
          esac

          echo "::add-mask::$DOMAIN"
          echo DOMAIN=$DOMAIN >> $GITHUB_ENV
      - name: Create k8s cluster for testing
        run: |
          linode-cli lke cluster-create \
            --label ${{ env.LINODE_CLUSTER_NAME }} \
            --region nl-ams \
            --k8s_version ${{ env.LINODE_K8S_VERSION }} \
            --control_plane.high_availability true \
            --node_pools.type ${{ inputs.linode_types }} --node_pools.count 3 \
            --node_pools.autoscaler.enabled true \
            --node_pools.autoscaler.max 3 \
            --node_pools.autoscaler.min 3 \
            --tags testing \
            --no-defaults
      - name: Retrieve cluster id
        run: echo "LINODE_CLUSTER_ID=$(linode-cli lke clusters-list --json | jq -ce '.[] | select(.label | startswith("${{ env.LINODE_CLUSTER_NAME }}")) | .id')" >> $GITHUB_ENV
      - name: Wait for cluster to be ready
        run: |
          echo "Waiting for the cluster to be active..."

          while :; do
            rawOutput=$(linode-cli lke pools-list ${{ env.LINODE_CLUSTER_ID }} --json)

            allReady=$(echo "$rawOutput" | jq -r 'map(.nodes | .status == "ready") | all')
            echo "All nodes ready: $allReady"
            
            if [ "$allReady" == "true" ]; then
              echo "Cluster is ready"
              break
            fi
            
            sleep 30
          done
      - name: Save kubectl config with auth token
        if: ${{ inputs.install_profile != 'no-apl' }}
        run: |
          echo "Waiting for kubeconfig..."
          while :; do
            linode-cli get-kubeconfig --label "${{ env.LINODE_CLUSTER_NAME }}" 2> /dev/null && break
            echo "still waiting..."
            sleep 10
          done
          echo LINODE_CLUSTER_CONTEXT=`kubectl config current-context` >> $GITHUB_ENV
      - name: Create image pull secret on test cluster
        if: ${{ inputs.install_profile != 'no-apl' }}
        run: |
          kubectl create secret docker-registry reg-otomi-github \
            --docker-server=${{ env.CACHE_REGISTRY }} \
            --docker-username=${{ env.BOT_USERNAME }} \
            --docker-password='${{ secrets.BOT_PULL_TOKEN }}'
      - name: Checkout
        if: ${{ inputs.install_profile != 'no-apl' }}
        uses: actions/checkout@v4
      - name: Prepare APL chart
        if: ${{ inputs.install_profile != 'no-apl' }}
        run: |
          ref=${{ github.event.pull_request.head.ref || github.ref }}
          tag=${ref##*/}
          sed --in-place "s/APP_VERSION_PLACEHOLDER/$tag/g" chart/apl/Chart.yaml
          sed --in-place "s/CONTEXT_PLACEHOLDER/${{ env.LINODE_CLUSTER_CONTEXT }}/g" tests/integration/${{ inputs.install_profile }}.yaml
          sed --in-place "s/CLUSTER_NAME_PLACEHOLDER/aplinstall${{ env.LINODE_CLUSTER_ID }}/g" tests/integration/${{ inputs.install_profile }}.yaml
          sed --in-place "s/OTOMI_VERSION_PLACEHOLDER/${GITHUB_REF##*/}/g" tests/integration/${{ inputs.install_profile }}.yaml
          touch values-container-registry.yaml

          # If a pipeline installs APL from the semver tag then pull container image from DockerHub
          [[ ${GITHUB_REF##*/} =~ ^v[0-9].+$ ]] && exit 0

          # Pull image from cache registry
          cat << EOF > values-container-registry.yaml
          imageName: "${{ env.CACHE_REGISTRY }}/${{ env.CACHE_REPO }}"
          imagePullSecretNames:
            - reg-otomi-github
          EOF
      - name: APL install
        if: ${{ inputs.install_profile != 'no-apl' }}
        env:
          LETSENCRYPT_STAGING: ${{ secrets.LETSENCRYPT_STAGING }}
          LETSENCRYPT_PRODUCTION: ${{ secrets.LETSENCRYPT_PRODUCTION }}
          EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }}
          EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }}
          EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }}
          EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
          EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }}
        run: |
          touch values.yaml

          adminPassword="$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)"
          [[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml
          [[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml
          [[ '${{ inputs.kms }}' == 'age' ]] && kms="--set kms.sops.provider=age"

          if [[ '${{ inputs.is_pre_installed }}' == 'true' ]]; then
            cat <<EOF >> values.yaml
          otomi:
            isPreInstalled: true
          EOF
            fi

          if [[ '${{ inputs.kms }}' == 'age' ]]; then
            cat <<EOF >> values.yaml
          kms:
            sops:
              provider: age
          EOF
            fi

          install_args="otomi chart/apl --wait --wait-for-jobs --timeout 90m0s \
            --values tests/integration/${{ inputs.install_profile }}.yaml \
            --values values-container-registry.yaml \
            --values values.yaml \
            --set cluster.provider=linode \
            --set dns.domainFilters[0]=${{ env.DOMAIN }} \
            --set dns.provider.akamai.clientSecret=${EDGEDNS_CLIENT_SECRET} \
            --set dns.provider.akamai.host=${EDGEDNS_HOST} \
            --set dns.provider.akamai.accessToken=${EDGEDNS_ACCESS_TOKEN} \
            --set dns.provider.akamai.clientToken=${EDGEDNS_CLIENT_TOKEN} \
            --set otomi.hasExternalDNS=true \
            --set cluster.domainSuffix=${{ env.DOMAIN }} \
            --set otomi.adminPassword=$adminPassword \
            $kms"

          helm install $install_args &
          HELM_PID=$!
          sleep 120

          # While helm is installing we can crete the wildcard dns record
          while true; do
              PUB_IP=$(kubectl get svc ingress-nginx-platform-controller -n ingress -ojson | jq '.status.loadBalancer.ingress[0].ip' -r)
              if [[ -n "$PUB_IP" ]]; then
                echo "::add-mask::$PUB_IP"
                echo PUB_IP=$PUB_IP >> $GITHUB_ENV
                break
              else
                echo "Waiting for ingress-nginx-platform-controller IP..."
                sleep 5
              fi
          done

          pip3 install edgegrid-python requests
          python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP || \
            (echo "Will try to recreate it" && \
            python3 bin/edgedns_A_record.py delete "*.${DOMAIN}" && \
            python3 bin/edgedns_A_record.py create "*.${DOMAIN}" $PUB_IP)


          wait $HELM_PID
      - name: Gather k8s events on failure
        if: failure()
        run: |
          kubectl get events --sort-by='.lastTimestamp' -A
      - name: Gather k8s pods on failure
        if: failure()
        run: |
          kubectl get pods -A -o wide
      - name: Gather APL logs on failure
        if: failure()
        run: |
          kubectl logs jobs/otomi-apl --tail 150
      - name: Gather otomi-e2e logs on failure
        if: failure()
        run: |
          kubectl logs -n maintenance -l app.kubernetes.io/instance=job-e2e --tail 15000
      - name: Remove the test cluster
        if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }}
        run: |
          linode-cli lke cluster-delete ${{ env.LINODE_CLUSTER_ID }}
      - name: Delete Domain
        if: ${{ always() && inputs.domain_zone == 'DNS-Integration' }}
        env:
          EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }}
          EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }}
          EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }}
          EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
          EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }}
        run: |
          python3 bin/edgedns_A_record.py delete "*.${DOMAIN}"
      - name: Slack Notification
        if: always()
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_CHANNEL: github-ci
          SLACK_COLOR: ${{ job.status }}
          SLACK_ICON: https://github.com/redkubes.png?size=48
          SLACK_TITLE: Scheduled integration tests
          SLACK_USERNAME: RedKubesBot