Skip to content

Commit d2bafc3

Browse files
potiukpotiuk
committed
Improve caching strategy across the board of CI workflow
We are using various caches in our build and so far - due to the way how "standard" caching works, PRs from forks could not effectively use the cache from main Airflow repository - because caches are not shared with other repositories - so the PRs builds could only use cache effectively when they were rebased and continued running from the same fork. This PR improves caching strategy using "stash" action from the ASF. Unlike `cache` - the action uses artifacts to store cache, and that makes it possible for the stash action to use such cache uploaded from `main` canary builds in PRs coming from the fork. As part of this change all the places where setup-python was used and breeze installed afterwards were reviewed and updated to use only breeze installation action (it already installs python) and this action has been improved to use UV caching effectively. Overall this PR should decrease setup overhead for many jobs across the CI workflow. Follow-up after #45266
1 parent 52ed7d7 commit d2bafc3

File tree

8 files changed

+165
-70
lines changed

8 files changed

+165
-70
lines changed

‎.github/actions/breeze/action.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@ runs:
3333
uses: actions/setup-python@v5
3434
with:
3535
python-version: ${{ inputs.python-version }}
36-
cache: 'pip'
37-
cache-dependency-path: ./dev/breeze/pyproject.toml
36+
- name: "Restore uv cache for breeze"
37+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
38+
with:
39+
key: "breeze-uv-cache-${{ inputs.python-version }}-${{ hashFiles('dev/breeze/pyproject.toml') }}"
40+
path: ~/.cache/uv
3841
- name: "Install Breeze"
3942
shell: bash
4043
run: ./scripts/ci/install_breeze.sh

‎.github/actions/install-pre-commit/action.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,31 +19,31 @@
1919
name: 'Install pre-commit'
2020
description: 'Installs pre-commit and related packages'
2121
inputs:
22+
# TODO(potiuk): automate update of these versions
2223
python-version:
2324
description: 'Python version to use'
24-
default: 3.9
25+
default: "3.9"
2526
uv-version:
2627
description: 'uv version to use'
27-
default: 0.5.5
28+
default: "0.5.13"
2829
pre-commit-version:
2930
description: 'pre-commit version to use'
30-
default: 4.0.1
31+
default: "4.0.1"
3132
pre-commit-uv-version:
3233
description: 'pre-commit-uv version to use'
33-
default: 4.1.4
34+
default: "4.1.4"
3435
runs:
3536
using: "composite"
3637
steps:
38+
- name: "Restore pre-commit cache"
39+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
40+
with:
41+
key: "pre-commit-cache-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}"
42+
path: ~/.cache/pre-commit/
3743
- name: Install pre-commit, uv, and pre-commit-uv
3844
shell: bash
3945
run: |
4046
pip install uv==${{inputs.uv-version}} || true
4147
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
4248
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
43-
- name: Cache pre-commit envs
44-
uses: actions/cache@v4
45-
with:
46-
path: ~/.cache/pre-commit
47-
key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
48-
restore-keys: |
49-
pre-commit-${{inputs.python-version}}-
49+
pre-commit install-hooks

‎.github/workflows/basic-tests.yml

Lines changed: 72 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,11 @@ jobs:
7272
persist-credentials: false
7373
- name: "Cleanup docker"
7474
run: ./scripts/ci/cleanup_docker.sh
75-
- uses: actions/setup-python@v5
76-
with:
77-
python-version: "${{ inputs.default-python-version }}"
78-
cache: 'pip'
79-
cache-dependency-path: ./dev/breeze/pyproject.toml
80-
- run: pip install --editable ./dev/breeze/
81-
- run: python -m pytest -n auto --color=yes
75+
- name: "Install Breeze"
76+
uses: ./.github/actions/breeze
77+
- run: uv tool run --from apache-airflow-breeze pytest -n auto --color=ye
8278
working-directory: ./dev/breeze/
8379

84-
8580
tests-ui:
8681
timeout-minutes: 10
8782
name: React UI tests
@@ -108,15 +103,24 @@ jobs:
108103
node-version: 21
109104
cache: 'pnpm'
110105
cache-dependency-path: 'airflow/ui/pnpm-lock.yaml'
111-
- name: "Cache eslint"
112-
uses: actions/cache@v4
106+
- name: "Restore eslint cache (ui)"
107+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
113108
with:
114-
path: 'airflow/ui/node_modules'
109+
path: 'airflow/ui/node_modules/'
115110
key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
111+
id: restore-eslint-cache
116112
- run: cd airflow/ui && pnpm install --frozen-lockfile
117113
- run: cd airflow/ui && pnpm test
118114
env:
119115
FORCE_COLOR: 2
116+
- name: "Save eslint cache (ui)"
117+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
118+
with:
119+
path: 'airflow/ui/node_modules/'
120+
key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
121+
if-no-files-found: 'error'
122+
retention-days: '2'
123+
if: steps.restore-eslint-cache.outputs.cache-hit != 'true'
120124

121125
tests-www:
122126
timeout-minutes: 10
@@ -137,15 +141,60 @@ jobs:
137141
uses: actions/setup-node@v4
138142
with:
139143
node-version: 21
140-
- name: "Cache eslint"
141-
uses: actions/cache@v4
144+
- name: "Restore eslint cache (www)"
145+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
142146
with:
143-
path: 'airflow/www/node_modules'
147+
path: 'airflow/www/node_modules/'
144148
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
149+
id: restore-eslint-cache
145150
- run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
146151
- run: yarn --cwd airflow/www/ run test
147152
env:
148153
FORCE_COLOR: 2
154+
- name: "Save eslint cache (www)"
155+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
156+
with:
157+
path: 'airflow/www/node_modules/'
158+
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
159+
if-no-files-found: 'error'
160+
retention-days: '2'
161+
if: steps.restore-eslint-cache.outputs.cache-hit != 'true'
162+
163+
install-pre-commit:
164+
timeout-minutes: 5
165+
name: "Install pre-commit for cache"
166+
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
167+
env:
168+
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
169+
if: inputs.basic-checks-only == 'true'
170+
steps:
171+
- name: "Cleanup repo"
172+
shell: bash
173+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
174+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
175+
uses: actions/checkout@v4
176+
with:
177+
persist-credentials: false
178+
- name: "Install Breeze"
179+
uses: ./.github/actions/breeze
180+
id: breeze
181+
- name: "Install pre-commit"
182+
uses: ./.github/actions/install-pre-commit
183+
id: pre-commit
184+
with:
185+
python-version: ${{steps.breeze.outputs.host-python-version}}
186+
# Saving pre-commit cache should happen only in one job in the entire workflow - because otherwise
187+
# it will cause 409 conflict errors - see https://github.com/actions/upload-artifact/issues/478
188+
# the way it works with airflow - even if the same action is in "ci-image-tests" the if condition
189+
# above `if: inputs.basic-checks-only == 'true'` will prevent it from running in the other job
190+
- name: "Save pre-commit cache"
191+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
192+
with:
193+
# yamllint disable rule:line-length
194+
key: "pre-commit-cache-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}"
195+
path: ~/.cache/pre-commit/
196+
if-no-files-found: 'error'
197+
retention-days: '2'
149198

150199
# Those checks are run if no image needs to be built for checks. This is for simple changes that
151200
# Do not touch any of the python code or any of the important files that might require building
@@ -154,6 +203,7 @@ jobs:
154203
timeout-minutes: 30
155204
name: "Static checks: basic checks only"
156205
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
206+
needs: install-pre-commit
157207
if: inputs.basic-checks-only == 'true'
158208
steps:
159209
- name: "Cleanup repo"
@@ -165,18 +215,6 @@ jobs:
165215
persist-credentials: false
166216
- name: "Cleanup docker"
167217
run: ./scripts/ci/cleanup_docker.sh
168-
- name: "Setup python"
169-
uses: actions/setup-python@v5
170-
with:
171-
python-version: ${{ inputs.default-python-version }}
172-
cache: 'pip'
173-
cache-dependency-path: ./dev/breeze/pyproject.toml
174-
- name: "Setup python"
175-
uses: actions/setup-python@v5
176-
with:
177-
python-version: "${{ inputs.default-python-version }}"
178-
cache: 'pip'
179-
cache-dependency-path: ./dev/breeze/pyproject.toml
180218
- name: "Install Breeze"
181219
uses: ./.github/actions/breeze
182220
id: breeze
@@ -216,6 +254,7 @@ jobs:
216254
timeout-minutes: 45
217255
name: "Upgrade checks"
218256
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
257+
needs: install-pre-commit
219258
env:
220259
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
221260
if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -229,12 +268,14 @@ jobs:
229268
persist-credentials: false
230269
- name: "Cleanup docker"
231270
run: ./scripts/ci/cleanup_docker.sh
232-
# Install python from scratch. No cache used. We always want to have fresh version of everything
233-
- uses: actions/setup-python@v5
271+
- name: "Install Breeze"
272+
uses: ./.github/actions/breeze
273+
id: breeze
274+
- name: "Install pre-commit"
275+
uses: ./.github/actions/install-pre-commit
276+
id: pre-commit
234277
with:
235-
python-version: "${{ inputs.default-python-version }}"
236-
- name: "Install latest pre-commit"
237-
run: pip install pre-commit
278+
python-version: ${{steps.breeze.outputs.host-python-version}}
238279
- name: "Autoupdate all pre-commits"
239280
run: pre-commit autoupdate
240281
- name: "Run automated upgrade for black"

‎.github/workflows/ci-image-build.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,5 +162,5 @@ jobs:
162162
key: "ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
163163
path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
164164
if-no-files-found: 'error'
165-
retention-days: 2
165+
retention-days: '2'
166166
if: inputs.upload-image-artifact == 'true'

‎.github/workflows/ci-image-checks.yml

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,47 @@ on: # yamllint disable-line rule:truthy
106106
type: string
107107

108108
jobs:
109+
install-pre-commit:
110+
timeout-minutes: 5
111+
name: "Install pre-commit for cache"
112+
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
113+
env:
114+
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
115+
if: inputs.basic-checks-only == 'false'
116+
steps:
117+
- name: "Cleanup repo"
118+
shell: bash
119+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
120+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
121+
uses: actions/checkout@v4
122+
with:
123+
persist-credentials: false
124+
- name: "Install Breeze"
125+
uses: ./.github/actions/breeze
126+
id: breeze
127+
- name: "Install pre-commit"
128+
uses: ./.github/actions/install-pre-commit
129+
id: pre-commit
130+
with:
131+
python-version: ${{steps.breeze.outputs.host-python-version}}
132+
# Saving pre-commit cache should happen only in one job in the entire workflow - because otherwise
133+
# it will cause 409 conflict errors - see https://github.com/actions/upload-artifact/issues/478
134+
# the way it works with airflow - even if the same action is in "basic-tests" the if condition
135+
# above `if: inputs.basic-checks-only == 'false'` will prevent it from running in the other job
136+
- name: "Save pre-commit cache"
137+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
138+
with:
139+
# yamllint disable rule:line-length
140+
key: "pre-commit-cache-${{ steps.breeze.outputs.host-python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}"
141+
path: ~/.cache/pre-commit/
142+
if-no-files-found: 'error'
143+
retention-days: '2'
144+
109145
static-checks:
110146
timeout-minutes: 45
111147
name: "Static checks"
112148
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
149+
needs: install-pre-commit
113150
env:
114151
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
115152
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}"
@@ -123,12 +160,6 @@ jobs:
123160
uses: actions/checkout@v4
124161
with:
125162
persist-credentials: false
126-
- name: "Setup python"
127-
uses: actions/setup-python@v5
128-
with:
129-
python-version: ${{ inputs.default-python-version }}
130-
cache: 'pip'
131-
cache-dependency-path: ./dev/breeze/pyproject.toml
132163
- name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
133164
uses: ./.github/actions/prepare_breeze_and_image
134165
with:
@@ -154,6 +185,7 @@ jobs:
154185
timeout-minutes: 45
155186
name: "MyPy checks"
156187
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
188+
needs: install-pre-commit
157189
if: inputs.needs-mypy == 'true'
158190
strategy:
159191
fail-fast: false
@@ -221,24 +253,30 @@ jobs:
221253
with:
222254
platform: "linux/amd64"
223255
python: ${{ inputs.default-python-version }}
224-
- uses: actions/cache@v4
225-
id: cache-doc-inventories
256+
- name: "Restore docs inventory cache"
257+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
226258
with:
227259
path: ./docs/_inventory_cache/
228-
key: docs-inventory-${{ hashFiles('pyproject.toml;') }}
229-
restore-keys: |
230-
docs-inventory-${{ hashFiles('pyproject.toml;') }}
231-
docs-inventory-
260+
key: docs-inventory-${{ hashFiles('pyproject.toml') }}
261+
id: restore-eslint-cache
232262
- name: "Building docs with ${{ matrix.flag }} flag"
233263
run: >
234264
breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }}
265+
- name: "Save docs inventory cache"
266+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
267+
with:
268+
path: ./docs/_inventory_cache/
269+
key: docs-inventory-${{ hashFiles('pyproject.toml') }}
270+
if-no-files-found: 'error'
271+
retention-days: '2'
272+
if: steps.restore-eslint-cache.outputs.cache-hit != 'true'
235273
- name: "Upload build docs"
236274
uses: actions/upload-artifact@v4
237275
with:
238276
name: airflow-docs
239277
path: './docs/_build'
240-
retention-days: 7
241-
if-no-files-found: error
278+
retention-days: '7'
279+
if-no-files-found: 'error'
242280
if: matrix.flag == '--docs-only'
243281

244282
publish-docs:

‎.github/workflows/ci.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,13 @@ jobs:
160160
persist-credentials: false
161161
- name: "Install Breeze"
162162
uses: ./.github/actions/breeze
163+
- name: "Save uv cache for breeze"
164+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
165+
with:
166+
key: "breeze-uv-cache-${{ inputs.python-version }}-${{ hashFiles('dev/breeze/pyproject.toml') }}"
167+
path: ~/.cache/uv/
168+
if-no-files-found: 'error'
169+
retention-days: '2'
163170
- name: "Get information about the Workflow"
164171
id: source-run-info
165172
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
@@ -171,7 +178,6 @@ jobs:
171178
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
172179
COMMIT_REF: "${{ github.sha }}"
173180
VERBOSE: "false"
174-
175181
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
176182
- name: env
177183
run: printenv

0 commit comments

Comments
 (0)