Skip to content

Commit bd210d4

Browse files
potiukpotiuk
committed
Improve caching strategy across the board of CI workflow
We are using various caches in our build and so far - due to the way how "standard" caching works, PRs from forks could not effectively use the cache from main Airflow repository - because caches are not shared with other repositories - so the PRs builds could only use cache effectively when they were rebased and continued running from the same fork. This PR improves caching strategy using "stash" action from the ASF. Unlike `cache` - the action uses artifacts to store cache, and that makes it possible for the stash action to use such cache uploaded from `main` canary builds in PRs coming from the fork. As part of this change all the places where setup-python was used and breeze installed afterwards were reviewed and updated to use only breeze installation action (it already installs python) and this action has been improved to use UV caching effectively. Overall this PR should decrease setup overhead for many jobs across the CI workflow. Follow-up after #45266
1 parent 52ed7d7 commit bd210d4

31 files changed

+388
-151
lines changed

‎.github/actions/breeze/action.yml

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ inputs:
2222
python-version:
2323
description: 'Python version to use'
2424
default: "3.9"
25+
use-uv:
26+
description: 'Whether to use uv tool'
27+
required: "true"
28+
type: "string"
2529
outputs:
2630
host-python-version:
2731
description: Python version used in host
@@ -33,13 +37,11 @@ runs:
3337
uses: actions/setup-python@v5
3438
with:
3539
python-version: ${{ inputs.python-version }}
36-
cache: 'pip'
37-
cache-dependency-path: ./dev/breeze/pyproject.toml
40+
# NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has
41+
# so low overhead, that just running upload cache/restore cache is slower than installing it from scratch
3842
- name: "Install Breeze"
3943
shell: bash
4044
run: ./scripts/ci/install_breeze.sh
41-
env:
42-
PYTHON_VERSION: ${{ inputs.python-version }}
4345
- name: "Free space"
4446
shell: bash
4547
run: breeze ci free-space
@@ -56,3 +58,15 @@ runs:
5658
run: breeze setup config --no-cheatsheet --no-asciiart
5759
env:
5860
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
61+
- name: "Use uv "
62+
shell: bash
63+
run: breeze setup config --use-uv
64+
env:
65+
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
66+
if: inputs.use-uv == 'true'
67+
- name: "Don't use uv "
68+
shell: bash
69+
run: breeze setup config --no-use-uv
70+
env:
71+
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
72+
if: inputs.use-uv != 'true'

‎.github/actions/install-pre-commit/action.yml

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,19 @@
1919
name: 'Install pre-commit'
2020
description: 'Installs pre-commit and related packages'
2121
inputs:
22+
# TODO(potiuk): automate update of these versions
2223
python-version:
2324
description: 'Python version to use'
24-
default: 3.9
25+
default: "3.9"
2526
uv-version:
2627
description: 'uv version to use'
27-
default: 0.5.5
28+
default: "0.5.13"
2829
pre-commit-version:
2930
description: 'pre-commit version to use'
30-
default: 4.0.1
31+
default: "4.0.1"
3132
pre-commit-uv-version:
3233
description: 'pre-commit-uv version to use'
33-
default: 4.1.4
34+
default: "4.1.4"
3435
runs:
3536
using: "composite"
3637
steps:
@@ -40,10 +41,35 @@ runs:
4041
pip install uv==${{inputs.uv-version}} || true
4142
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
4243
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
43-
- name: Cache pre-commit envs
44-
uses: actions/cache@v4
44+
working-directory: ${{ github.workspace }}
45+
# We need to use tar file with archive to restore all the permissions and symlinks
46+
- name: "Delete ~.cache"
47+
run: |
48+
du ~/ --max-depth=2
49+
echo
50+
echo Deleting ~/.cache
51+
echo
52+
rm -rf ~/.cache
53+
echo
54+
shell: bash
55+
- name: "Restore pre-commit cache"
56+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4557
with:
46-
path: ~/.cache/pre-commit
47-
key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
48-
restore-keys: |
49-
pre-commit-${{inputs.python-version}}-
58+
key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
59+
path: /tmp/
60+
id: restore-pre-commit-cache
61+
- name: "Restore .cache from the tar file"
62+
run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
63+
shell: bash
64+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
65+
- name: "Show restored files"
66+
run: |
67+
echo "Restored files"
68+
du ~/ --max-depth=2
69+
echo
70+
shell: bash
71+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
72+
- name: Install pre-commit hooks
73+
shell: bash
74+
run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1)
75+
working-directory: ${{ github.workspace }}

‎.github/actions/prepare_breeze_and_image/action.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ inputs:
2828
platform:
2929
description: 'Platform for the build - linux/amd64 or linux/arm64'
3030
required: true
31+
use-uv:
32+
description: 'Whether to use uv'
33+
required: true
3134
outputs:
3235
host-python-version:
3336
description: Python version used in host
@@ -40,11 +43,13 @@ runs:
4043
shell: bash
4144
- name: "Install Breeze"
4245
uses: ./.github/actions/breeze
46+
with:
47+
use-uv: ${{ inputs.use-uv }}
4348
id: breeze
4449
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
4550
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4651
with:
47-
key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}"
52+
key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
4853
path: "/tmp/"
4954
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
5055
run: >

‎.github/actions/prepare_single_ci_image/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ runs:
3838
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
3939
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4040
with:
41-
key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}"
41+
key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
4242
path: "/tmp/"
4343
if: contains(inputs.python-versions-list-as-string, inputs.python)
4444
- name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"

‎.github/workflows/additional-ci-image-checks.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ jobs:
111111
python-versions: ${{ inputs.python-versions }}
112112
branch: ${{ inputs.branch }}
113113
constraints-branch: ${{ inputs.constraints-branch }}
114-
use-uv: ${{ inputs.use-uv}}
114+
use-uv: ${{ inputs.use-uv }}
115115
include-success-outputs: ${{ inputs.include-success-outputs }}
116116
docker-cache: ${{ inputs.docker-cache }}
117117
disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -143,6 +143,8 @@ jobs:
143143
run: ./scripts/ci/cleanup_docker.sh
144144
- name: "Install Breeze"
145145
uses: ./.github/actions/breeze
146+
with:
147+
use-uv: ${{ inputs.use-uv }}
146148
- name: "Login to ghcr.io"
147149
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
148150
- name: "Check that image builds quickly"
@@ -167,7 +169,7 @@ jobs:
167169
# platform: "linux/arm64"
168170
# branch: ${{ inputs.branch }}
169171
# constraints-branch: ${{ inputs.constraints-branch }}
170-
# use-uv: ${{ inputs.use-uv}}
172+
# use-uv: ${{ inputs.use-uv }}
171173
# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
172174
# docker-cache: ${{ inputs.docker-cache }}
173175
# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}

‎.github/workflows/additional-prod-image-tests.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy
5656
description: "Which version of python should be used by default"
5757
required: true
5858
type: string
59+
use-uv:
60+
description: "Whether to use uv"
61+
required: true
62+
type: string
5963
jobs:
6064
prod-image-extra-checks-main:
6165
name: PROD image extra checks (main)
@@ -117,6 +121,7 @@ jobs:
117121
platform: "linux/amd64"
118122
image-type: "prod"
119123
python: ${{ inputs.default-python-version }}
124+
use-uv: ${{ inputs.use-uv }}
120125
- name: "Test examples of PROD image building"
121126
run: "
122127
cd ./docker_tests && \
@@ -150,6 +155,7 @@ jobs:
150155
platform: "linux/amd64"
151156
image-type: "prod"
152157
python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
158+
use-uv: ${{ inputs.use-uv }}
153159
id: breeze
154160
- name: "Test docker-compose quick start"
155161
run: breeze testing docker-compose-tests

‎.github/workflows/basic-tests.yml

Lines changed: 79 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy
5656
description: "Whether to run only latest version checks (true/false)"
5757
required: true
5858
type: string
59+
use-uv:
60+
description: "Whether to use uv in the image"
61+
required: true
62+
type: string
5963
jobs:
6064
run-breeze-tests:
6165
timeout-minutes: 10
@@ -72,16 +76,12 @@ jobs:
7276
persist-credentials: false
7377
- name: "Cleanup docker"
7478
run: ./scripts/ci/cleanup_docker.sh
75-
- uses: actions/setup-python@v5
79+
- name: "Install Breeze"
80+
uses: ./.github/actions/breeze
7681
with:
77-
python-version: "${{ inputs.default-python-version }}"
78-
cache: 'pip'
79-
cache-dependency-path: ./dev/breeze/pyproject.toml
80-
- run: pip install --editable ./dev/breeze/
81-
- run: python -m pytest -n auto --color=yes
82+
use-uv: ${{ inputs.use-uv }}
83+
- run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
8284
working-directory: ./dev/breeze/
83-
84-
8585
tests-ui:
8686
timeout-minutes: 10
8787
name: React UI tests
@@ -108,15 +108,24 @@ jobs:
108108
node-version: 21
109109
cache: 'pnpm'
110110
cache-dependency-path: 'airflow/ui/pnpm-lock.yaml'
111-
- name: "Cache eslint"
112-
uses: actions/cache@v4
111+
- name: "Restore eslint cache (ui)"
112+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
113113
with:
114-
path: 'airflow/ui/node_modules'
115-
key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
114+
path: airflow/ui/node_modules/
115+
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
116+
id: restore-eslint-cache
116117
- run: cd airflow/ui && pnpm install --frozen-lockfile
117118
- run: cd airflow/ui && pnpm test
118119
env:
119120
FORCE_COLOR: 2
121+
- name: "Save eslint cache (ui)"
122+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
123+
with:
124+
path: airflow/ui/node_modules/
125+
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
126+
if-no-files-found: 'error'
127+
retention-days: '2'
128+
if: steps.restore-eslint-cache.outputs.stash-hit != 'true'
120129

121130
tests-www:
122131
timeout-minutes: 10
@@ -137,15 +146,54 @@ jobs:
137146
uses: actions/setup-node@v4
138147
with:
139148
node-version: 21
140-
- name: "Cache eslint"
141-
uses: actions/cache@v4
149+
- name: "Restore eslint cache (www)"
150+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
142151
with:
143-
path: 'airflow/www/node_modules'
144-
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
152+
path: airflow/www/node_modules/
153+
key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
154+
id: restore-eslint-cache
145155
- run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
146156
- run: yarn --cwd airflow/www/ run test
147157
env:
148158
FORCE_COLOR: 2
159+
- name: "Save eslint cache (www)"
160+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
161+
with:
162+
path: airflow/www/node_modules/
163+
key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
164+
if-no-files-found: 'error'
165+
retention-days: '2'
166+
if: steps.restore-eslint-cache.outputs.stash-hit != 'true'
167+
168+
install-pre-commit:
169+
timeout-minutes: 5
170+
name: "Install pre-commit for cache"
171+
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
172+
env:
173+
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
174+
if: inputs.basic-checks-only == 'true'
175+
steps:
176+
- name: "Cleanup repo"
177+
shell: bash
178+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
179+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
180+
uses: actions/checkout@v4
181+
with:
182+
persist-credentials: false
183+
- name: "Install Breeze"
184+
uses: ./.github/actions/breeze
185+
with:
186+
use-uv: ${{ inputs.use-uv }}
187+
id: breeze
188+
- name: "Install pre-commit"
189+
uses: ./.github/actions/install-pre-commit
190+
id: pre-commit
191+
with:
192+
python-version: ${{steps.breeze.outputs.host-python-version}}
193+
- name: "Prepare .tar file from pre-commit cache"
194+
run: |
195+
tar -C ~ -czf /tmp/cache-pre-commit.tar.gz .cache/pre-commit .cache/uv
196+
shell: bash
149197

150198
# Those checks are run if no image needs to be built for checks. This is for simple changes that
151199
# Do not touch any of the python code or any of the important files that might require building
@@ -154,6 +202,7 @@ jobs:
154202
timeout-minutes: 30
155203
name: "Static checks: basic checks only"
156204
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
205+
needs: install-pre-commit
157206
if: inputs.basic-checks-only == 'true'
158207
steps:
159208
- name: "Cleanup repo"
@@ -165,20 +214,10 @@ jobs:
165214
persist-credentials: false
166215
- name: "Cleanup docker"
167216
run: ./scripts/ci/cleanup_docker.sh
168-
- name: "Setup python"
169-
uses: actions/setup-python@v5
170-
with:
171-
python-version: ${{ inputs.default-python-version }}
172-
cache: 'pip'
173-
cache-dependency-path: ./dev/breeze/pyproject.toml
174-
- name: "Setup python"
175-
uses: actions/setup-python@v5
176-
with:
177-
python-version: "${{ inputs.default-python-version }}"
178-
cache: 'pip'
179-
cache-dependency-path: ./dev/breeze/pyproject.toml
180217
- name: "Install Breeze"
181218
uses: ./.github/actions/breeze
219+
with:
220+
use-uv: ${{ inputs.use-uv }}
182221
id: breeze
183222
- name: "Install pre-commit"
184223
uses: ./.github/actions/install-pre-commit
@@ -216,6 +255,7 @@ jobs:
216255
timeout-minutes: 45
217256
name: "Upgrade checks"
218257
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
258+
needs: install-pre-commit
219259
env:
220260
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
221261
if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -229,12 +269,16 @@ jobs:
229269
persist-credentials: false
230270
- name: "Cleanup docker"
231271
run: ./scripts/ci/cleanup_docker.sh
232-
# Install python from scratch. No cache used. We always want to have fresh version of everything
233-
- uses: actions/setup-python@v5
272+
- name: "Install Breeze"
273+
uses: ./.github/actions/breeze
234274
with:
235-
python-version: "${{ inputs.default-python-version }}"
236-
- name: "Install latest pre-commit"
237-
run: pip install pre-commit
275+
use-uv: ${{ inputs.use-uv }}
276+
id: breeze
277+
- name: "Install pre-commit"
278+
uses: ./.github/actions/install-pre-commit
279+
id: pre-commit
280+
with:
281+
python-version: ${{steps.breeze.outputs.host-python-version}}
238282
- name: "Autoupdate all pre-commits"
239283
run: pre-commit autoupdate
240284
- name: "Run automated upgrade for black"
@@ -305,6 +349,8 @@ jobs:
305349
run: ./scripts/ci/cleanup_docker.sh
306350
- name: "Install Breeze"
307351
uses: ./.github/actions/breeze
352+
with:
353+
use-uv: ${{ inputs.use-uv }}
308354
- name: "Cleanup dist files"
309355
run: rm -fv ./dist/*
310356
- name: Setup git for tagging

0 commit comments

Comments
 (0)