test-go.yaml

name: Go Tests

on:
  push:
    branches:
      - main
      - patch-*
      - prepare-*
    paths:
      - '**.go'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/test-go.yaml'
      - 'server/authz/policy.rego'
      - 'docker-compose.yml'
  pull_request:
    paths:
      - '**.go'
      - 'go.mod'
      - 'go.sum'
      - '.github/workflows/test-go.yaml'
      - 'server/authz/policy.rego'
      - 'docker-compose.yml'
  workflow_dispatch: # Manual
  schedule:
    - cron: '0 4 * * *'

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
  cancel-in-progress: true

defaults:
  run:
    # fail-fast using bash -eo pipefail. See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
    shell: bash

permissions:
  contents: read

jobs:
  test-go:
    strategy:
      matrix:
        suite: ["integration-core", "integration-enterprise", "integration-mdm", "core", "mysql", "fleetctl", "vuln"]
        os: [ubuntu-latest]
        mysql: ["mysql:8.0.36", "mysql:8.4.3", "mysql:9.1.0"] # make sure to update supported versions docs when this changes
        isCron:
          - ${{ github.event_name == 'schedule' }}
        # Only run MySQL 9 tests on cron schedule
        exclude:
          - isCron: false
            mysql: "mysql:9.1.0"
    # Don't cancel all tests if an integration suite fails as integration suites are more likely to be flakey.
    # Don't cancel all tests if the vulns suite fails as vulns may start failing for unrelated reasons to PR'd code.
    continue-on-error: ${{ matrix.suite == 'vuln' || startsWith(matrix.suite, 'integration-') }}
    runs-on: ${{ matrix.os }}

    env:
      RACE_ENABLED: false
      GO_TEST_TIMEOUT: 20m

    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
      with:
        egress-policy: audit

    - name: Checkout Code
      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

    - name: Install Go
      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
      with:
        go-version-file: 'go.mod'

    # Pre-starting dependencies here means they are ready to go when we need them.
    - name: Start Infra Dependencies
      # Use & to background this
      run: FLEET_MYSQL_IMAGE=${{ matrix.mysql }} docker compose -f docker-compose.yml -f docker-compose-redis-cluster.yml up -d mysql_test mysql_replica_test redis redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6 redis-cluster-setup minio saml_idp mailhog mailpit smtp4dev_test &

    - name: Add TLS certificate for SMTP Tests
      run: |
        sudo cp tools/smtp4dev/fleet.crt /usr/local/share/ca-certificates/
        sudo update-ca-certificates

    # Go dependencies are cached by actions/setup-go
    - name: Install Go Dependencies
      run: make deps-go

    # This step takes >20 seconds and should be skipped for jobs that don't need it. https://github.com/fleetdm/fleet/issues/27435
    - name: Install ZSH
      run: sudo apt update && sudo apt install -y zsh

    - name: Generate static files
      run: |
        export PATH=$PATH:~/go/bin
        make generate-go

    - name: Set Go race setting on schedule
      if: github.event.schedule == '0 4 * * *'
      run: |
        echo "RACE_ENABLED=true" >> $GITHUB_ENV
        echo "GO_TEST_TIMEOUT=1h" >> $GITHUB_ENV

    - name: Wait for mysql
      run: |
        echo "waiting for mysql..."
        until docker compose exec -T mysql_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
            echo "."
            sleep 1
        done
        echo "mysql is ready"
        echo "waiting for mysql replica..."
        until docker compose exec -T mysql_replica_test sh -c "mysql -uroot -p\"\${MYSQL_ROOT_PASSWORD}\" -e \"SELECT 1=1\" fleet" &> /dev/null; do
            echo "."
            sleep 1
        done
        echo "mysql replica is ready"

    - name: Run Go Tests
      run: |
        if [[ "${{ matrix.suite }}" == "core" ]]; then
          CI_TEST_PKG=main
          RUN_TESTS_ARG='-skip=^TestIntegrations'
        elif [[ "${{ matrix.suite }}" == "integration-core" ]]; then
          CI_TEST_PKG=integration
          RUN_TESTS_ARG='-run=^TestIntegrations -skip "^(TestIntegrationsMDM|TestIntegrationsEnterprise)"'
        elif [[ "${{ matrix.suite }}" == "integration-mdm" ]]; then
          CI_TEST_PKG=integration
          RUN_TESTS_ARG='-run=^TestIntegrationsMDM'
        elif [[ "${{ matrix.suite }}" == "integration-enterprise" ]]; then
          CI_TEST_PKG=integration
          RUN_TESTS_ARG='-run=^TestIntegrationsEnterprise'
        else
          CI_TEST_PKG="${{ matrix.suite }}"
          RUN_TESTS_ARG=''
        fi
        GO_TEST_EXTRA_FLAGS="-v -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT $RUN_TESTS_ARG" \
          TEST_LOCK_FILE_PATH=$(pwd)/lock \
          TEST_CRON_NO_RECOVER=1 \
          NETWORK_TEST=1 \
          REDIS_TEST=1 \
          MYSQL_TEST=1 \
          MYSQL_REPLICA_TEST=1 \
          MINIO_STORAGE_TEST=1 \
          SAML_IDP_TEST=1 \
          MAIL_TEST=1 \
          NETWORK_TEST_GITHUB_TOKEN=${{ secrets.FLEET_RELEASE_GITHUB_PAT }} \
          CI_TEST_PKG="${CI_TEST_PKG}" \
          make test-go 2>&1 | tee /tmp/gotest.log

    - name: Create mysql identifier without colon
      if: always()
      run: |
        echo "MATRIX_MYSQL_ID=$(echo ${{ matrix.mysql }} | tr -d ':')" >> $GITHUB_ENV

    - name: Save coverage
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-coverage
        path: ./coverage.txt
        if-no-files-found: error

    - name: Generate summary of errors
      if: failure()
      run: |
        c1grep() { grep "$@" || test $? = 1; }
        c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
        c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
        c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
        c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
        c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
        GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
        echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
        if [[ -z "$GO_FAIL_SUMMARY" ]]; then
          GO_FAIL_SUMMARY="unknown, please check the build URL"
        fi
        GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json

    - name: Slack Notification
      if: github.event.schedule == '0 4 * * *' && failure()
      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
      with:
        payload-file-path: ./payload.json
      env:
        JOB_STATUS: ${{ job.status }}
        EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
        RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

    - name: Upload test log
      if: always()
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-test-log
        path: /tmp/gotest.log
        if-no-files-found: error

    - name: Upload summary test log
      if: always()
      uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
      with:
        name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-summary-test-log
        path: /tmp/summary.txt

  # Based on https://github.com/micromdm/nanomdm/blob/main/.github/workflows/on-push-pr.yml#L87
  test-go-nanomdm:
    runs-on: 'ubuntu-latest'
    services:
      mysql:
        image: mysql:8.0.36
        env:
          MYSQL_RANDOM_ROOT_PASSWORD: yes
          MYSQL_DATABASE: testdb
          MYSQL_USER: testuser
          MYSQL_PASSWORD: testpw
        ports:
          - 3800:3306
        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
    env:
      MYSQL_PWD: testpw
      PORT: 3800
      RACE_ENABLED: true
      GO_TEST_TIMEOUT: 20m
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

      - name: Install Go
        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
        with:
          go-version-file: 'go.mod'

      - name: verify mysql
        run: |
          while ! mysqladmin ping --host=localhost --port=$PORT --protocol=TCP --silent; do
            sleep 1
          done

      - name: mysql schema
        run: |
          mysql --version
          mysql --user=testuser --host=localhost --port=$PORT --protocol=TCP testdb < ./server/mdm/nanomdm/storage/mysql/schema.sql

      - name: set test dsn
        run: echo "NANOMDM_MYSQL_STORAGE_TEST_DSN=testuser:testpw@tcp(localhost:$PORT)/testdb" >> $GITHUB_ENV

      - name: Run Go tests
        run: |
          go test -v -parallel 8 -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT \
            -coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/server/mdm/nanomdm/... \
          ./server/mdm/nanomdm/storage/mysql 2>&1 | tee /tmp/gotest.log

      - name: Save coverage
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: nanomdm-coverage
          path: ./coverage.txt
          if-no-files-found: error

      - name: Generate summary of errors
        if: failure()
        run: |
          c1grep() { grep "$@" || test $? = 1; }
          c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
          c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
          c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
          c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
          c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
          GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
          echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
          if [[ -z "$GO_FAIL_SUMMARY" ]]; then
            GO_FAIL_SUMMARY="unknown, please check the build URL"
          fi
          GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json

      - name: Slack Notification
        if: github.event.schedule == '0 4 * * *' && failure()
        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
        with:
          payload-file-path: ./payload.json
        env:
          JOB_STATUS: ${{ job.status }}
          EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
          RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

      - name: Upload test log
        if: always()
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: nanomdm-test-log
          path: /tmp/gotest.log
          if-no-files-found: error

      - name: Upload summary test log
        if: always()
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
        with:
          name: nanomdm-summary-test-log
          path: /tmp/summary.txt

  # We upload all backend coverage in one step so that we're less like to end up in a situation with a partial coverage report.
  upload-coverage:
    needs: [test-go, test-go-nanomdm]
    runs-on: ubuntu-latest
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
        with:
          egress-policy: audit

      - name: Checkout Code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
      - name: Download artifacts
        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
        with:
          pattern: '*-coverage'
      - name: Upload to Codecov
        uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: backend