Release check #339
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release check | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| run_dataplex: | |
| description: 'Run Dataplex tests' | |
| required: false | |
| default: 'true' | |
| run_spark_dataproc: | |
| description: 'Run Spark Dataproc tests' | |
| required: false | |
| default: 'true' | |
| openlineage_release: | |
| description: 'Override OpenLineage release version' | |
| required: false | |
| spark_matrix: | |
| description: 'Overwrite matrix for spark tests' | |
| required: false | |
| permissions: | |
| id-token: write | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| initialize_workflow: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_dataplex: ${{ github.event.inputs.run_dataplex || 'true' }} | |
| run_spark_dataproc: ${{ github.event.inputs.run_spark_dataproc || 'true' }} | |
| openlineage_release: ${{ github.event.inputs.openlineage_release || steps.select-components.outputs.ol_release }} | |
| spark_matrix: ${{ github.event.inputs.spark_matrix || steps.set-matrix-values.outputs.spark_dataproc_matrix }} | |
| execution_time: ${{ steps.get-execution-time.outputs.execution_time }} | |
| steps: | |
| - name: Get execution time | |
| id: get-execution-time | |
| run: echo "execution_time=$(date +'%Y%d%m%H%M')" >> $GITHUB_OUTPUT | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| # there should be part about checking not only the OL version but also the producers and consumers | |
| # there is no way to check for new releases of spark_dataproc or dataplex so we just run their tests everytime | |
| # also normally new release of OL should trigger all producer tests but for now they are run anyway so no need to trigger | |
| - name: Select components to run | |
| id: select-components | |
| run: ./scripts/select_components.sh | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.select-components.outputs.releases_updated == 'true' | |
| with: | |
| name: updated-releases | |
| path: generated-files/updated-releases.json | |
| retention-days: 1 | |
| - name: set-matrix-values | |
| id: set-matrix-values | |
| run: | | |
| get_matrix() { | |
| local component="$1" | |
| local releases_file="./generated-files/updated-releases.json" | |
| local versions_file="producer/$component/versions.json" | |
| [[ -f "$releases_file" && -f "$versions_file" ]] || { | |
| echo "Missing releases.json or versions.json" >&2 | |
| return 1 | |
| } | |
| local latest_ol=$(jq -r '.[] | select(.name == "openlineage") | .latest_version' "$releases_file") | |
| local latest_comp=$(jq -r --arg name "$component" '.[] | select(.name == $name) | .latest_version' "$releases_file") | |
| jq -c --arg ol "$latest_ol" --arg comp "$latest_comp" ' | |
| .openlineage_versions |= (if $ol != "" and index($ol) == null then . + [$ol] else . end) | | |
| .component_version |= (if $comp != "" and index($comp) == null then . + [$comp] else . end) | |
| ' "$versions_file" | |
| } | |
| echo "spark_dataproc_matrix=$(get_matrix spark_dataproc)" >> $GITHUB_OUTPUT | |
| ######## COMPONENT VALIDATION ######## | |
| dataplex: | |
| needs: initialize_workflow | |
| if: ${{ needs.initialize_workflow.outputs.run_dataplex == 'true' }} | |
| uses: ./.github/workflows/consumer_dataplex.yml | |
| secrets: | |
| gcpKey: ${{ secrets.GCP_SA_KEY }} | |
| with: | |
| release: ${{ needs.initialize_workflow.outputs.openlineage_release }} | |
| spark-dataproc: | |
| needs: initialize_workflow | |
| if: ${{ needs.initialize_workflow.outputs.run_spark_dataproc == 'true' }} | |
| uses: ./.github/workflows/producer_spark_dataproc.yml | |
| strategy: | |
| matrix: ${{ fromJson(needs.initialize_workflow.outputs.spark_matrix) }} | |
| secrets: | |
| gcpKey: ${{ secrets.GCP_SA_KEY }} | |
| postgresqlUser: ${{ secrets.POSTGRESQL_USER }} | |
| postgresqlPassword: ${{ secrets.POSTGRESQL_PASSWORD }} | |
| with: | |
| ol_release: ${{ matrix.openlineage_versions }} | |
| spark_release: ${{ matrix.component_version }} | |
| get-latest-snapshots: 'false' | |
| ######## COLLECTION OF REPORTS AND EXECUTE APPROPRIATE ACTIONS ######## | |
| collect-and-compare-reports: | |
| needs: | |
| - initialize_workflow | |
| - dataplex | |
| - spark-dataproc | |
| if: ${{ !failure() }} | |
| uses: ./.github/workflows/collect_and_compare_reports.yml | |
| notify-maintainers: | |
| needs: | |
| - initialize_workflow | |
| - collect-and-compare-reports | |
| if: ${{ !failure() }} | |
| uses: ./.github/workflows/notify_maintainers.yml | |
| with: | |
| workflow-type: 'release' | |
| trigger-type: ${{ github.event.inputs.openlineage_release && 'manual' || 'scheduled' }} | |
| execution-time: ${{ needs.initialize_workflow.outputs.execution_time }} | |
| generate-compatibility-tables: | |
| needs: collect-and-compare-reports | |
| if: ${{ !failure() }} | |
| uses: ./.github/workflows/generate_compatibility_tables.yml | |
| update-repo-files: | |
| needs: | |
| - initialize_workflow | |
| - notify-maintainers | |
| - generate-compatibility-tables | |
| if: ${{ !failure() }} | |
| uses: ./.github/workflows/update_repo_files.yml | |
| secrets: | |
| CI_PUSH_TOKEN: ${{ secrets.CI_PUSH_TOKEN }} | |
| with: | |
| workflow-type: 'release' | |
| trigger-type: ${{ github.event.inputs.openlineage_release && 'manual' || 'scheduled' }} | |
| execution-time: ${{ needs.initialize_workflow.outputs.execution_time }} | |
| update-site: | |
| needs: | |
| - update-repo-files | |
| uses: ./.github/workflows/update_site.yml | |
| secrets: | |
| CI_PUSH_TOKEN: ${{ secrets.OL_SITE_PUSH_TOKEN }} |