Skip to content

Commit 740f63d

Browse files
potiukpotiuk
committed
Improve caching strategy across the board of CI workflow
We are using various caches in our build and so far - due to the way how "standard" caching works, PRs from forks could not effectively use the cache from main Airflow repository - because caches are not shared with other repositories - so the PRs builds could only use cache effectively when they were rebased and continued running from the same fork. This PR improves caching strategy using "stash" action from the ASF. Unlike `cache` - the action uses artifacts to store cache, and that makes it possible for the stash action to use such cache uploaded from `main` canary builds in PRs coming from the fork. As part of this change all the places where setup-python was used and breeze installed afterwards were reviewed and updated to use only breeze installation action (it already installs python) and this action has been improved to use UV caching effectively. Overall this PR should decrease setup overhead for many jobs across the CI workflow. Follow-up after #45266
1 parent 52ed7d7 commit 740f63d

31 files changed

+385
-151
lines changed

‎.github/actions/breeze/action.yml

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ inputs:
2222
python-version:
2323
description: 'Python version to use'
2424
default: "3.9"
25+
use-uv:
26+
description: 'Whether to use uv tool'
27+
required: "true"
28+
type: "string"
2529
outputs:
2630
host-python-version:
2731
description: Python version used in host
@@ -33,13 +37,11 @@ runs:
3337
uses: actions/setup-python@v5
3438
with:
3539
python-version: ${{ inputs.python-version }}
36-
cache: 'pip'
37-
cache-dependency-path: ./dev/breeze/pyproject.toml
40+
# NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has
41+
# so low overhead, that just running upload cache/restore cache is slower than installing it from scratch
3842
- name: "Install Breeze"
3943
shell: bash
4044
run: ./scripts/ci/install_breeze.sh
41-
env:
42-
PYTHON_VERSION: ${{ inputs.python-version }}
4345
- name: "Free space"
4446
shell: bash
4547
run: breeze ci free-space
@@ -56,3 +58,15 @@ runs:
5658
run: breeze setup config --no-cheatsheet --no-asciiart
5759
env:
5860
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
61+
- name: "Use uv "
62+
shell: bash
63+
run: breeze setup config --use-uv
64+
env:
65+
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
66+
if: inputs.use-uv == 'true'
67+
- name: "Don't use uv "
68+
shell: bash
69+
run: breeze setup config --no-use-uv
70+
env:
71+
AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
72+
if: inputs.use-uv != 'true'

‎.github/actions/install-pre-commit/action.yml

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,19 @@
1919
name: 'Install pre-commit'
2020
description: 'Installs pre-commit and related packages'
2121
inputs:
22+
# TODO(potiuk): automate update of these versions
2223
python-version:
2324
description: 'Python version to use'
24-
default: 3.9
25+
default: "3.9"
2526
uv-version:
2627
description: 'uv version to use'
27-
default: 0.5.5
28+
default: "0.5.13"
2829
pre-commit-version:
2930
description: 'pre-commit version to use'
30-
default: 4.0.1
31+
default: "4.0.1"
3132
pre-commit-uv-version:
3233
description: 'pre-commit-uv version to use'
33-
default: 4.1.4
34+
default: "4.1.4"
3435
runs:
3536
using: "composite"
3637
steps:
@@ -40,10 +41,35 @@ runs:
4041
pip install uv==${{inputs.uv-version}} || true
4142
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
4243
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
43-
- name: Cache pre-commit envs
44-
uses: actions/cache@v4
44+
working-directory: ${{ github.workspace }}
45+
# We need to use tar file with archive to restore all the permissions and symlinks
46+
- name: "Delete ~.cache"
47+
run: |
48+
du ~/ --max-depth=2
49+
echo
50+
echo Deleting ~/.cache
51+
echo
52+
rm -rf ~/.cache
53+
echo
54+
shell: bash
55+
- name: "Restore pre-commit cache"
56+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4557
with:
46-
path: ~/.cache/pre-commit
47-
key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
48-
restore-keys: |
49-
pre-commit-${{inputs.python-version}}-
58+
key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
59+
path: /tmp/
60+
id: restore-pre-commit-cache
61+
- name: "Restore .cache from the tar file"
62+
run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
63+
shell: bash
64+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
65+
- name: "Show restored files"
66+
run: |
67+
echo "Restored files"
68+
du ~/ --max-depth=2
69+
echo
70+
shell: bash
71+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
72+
- name: Install pre-commit hooks
73+
shell: bash
74+
run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1)
75+
working-directory: ${{ github.workspace }}

‎.github/actions/prepare_breeze_and_image/action.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ inputs:
2828
platform:
2929
description: 'Platform for the build - linux/amd64 or linux/arm64'
3030
required: true
31+
use-uv:
32+
description: 'Whether to use uv'
33+
required: true
3134
outputs:
3235
host-python-version:
3336
description: Python version used in host
@@ -40,11 +43,13 @@ runs:
4043
shell: bash
4144
- name: "Install Breeze"
4245
uses: ./.github/actions/breeze
46+
with:
47+
use-uv: ${{ inputs.use-uv }}
4348
id: breeze
4449
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
4550
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4651
with:
47-
key: "${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}"
52+
key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
4853
path: "/tmp/"
4954
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
5055
run: >

‎.github/actions/prepare_single_ci_image/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ runs:
3838
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
3939
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4040
with:
41-
key: "ci-image-save-${{ inputs.platform }}-${{ inputs.python }}"
41+
key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }}
4242
path: "/tmp/"
4343
if: contains(inputs.python-versions-list-as-string, inputs.python)
4444
- name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}"

‎.github/workflows/additional-ci-image-checks.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ jobs:
111111
python-versions: ${{ inputs.python-versions }}
112112
branch: ${{ inputs.branch }}
113113
constraints-branch: ${{ inputs.constraints-branch }}
114-
use-uv: ${{ inputs.use-uv}}
114+
use-uv: ${{ inputs.use-uv }}
115115
include-success-outputs: ${{ inputs.include-success-outputs }}
116116
docker-cache: ${{ inputs.docker-cache }}
117117
disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -143,6 +143,8 @@ jobs:
143143
run: ./scripts/ci/cleanup_docker.sh
144144
- name: "Install Breeze"
145145
uses: ./.github/actions/breeze
146+
with:
147+
use-uv: ${{ inputs.use-uv }}
146148
- name: "Login to ghcr.io"
147149
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
148150
- name: "Check that image builds quickly"
@@ -167,7 +169,7 @@ jobs:
167169
# platform: "linux/arm64"
168170
# branch: ${{ inputs.branch }}
169171
# constraints-branch: ${{ inputs.constraints-branch }}
170-
# use-uv: ${{ inputs.use-uv}}
172+
# use-uv: ${{ inputs.use-uv }}
171173
# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
172174
# docker-cache: ${{ inputs.docker-cache }}
173175
# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}

‎.github/workflows/additional-prod-image-tests.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy
5656
description: "Which version of python should be used by default"
5757
required: true
5858
type: string
59+
use-uv:
60+
description: "Whether to use uv"
61+
required: true
62+
type: string
5963
jobs:
6064
prod-image-extra-checks-main:
6165
name: PROD image extra checks (main)
@@ -117,6 +121,7 @@ jobs:
117121
platform: "linux/amd64"
118122
image-type: "prod"
119123
python: ${{ inputs.default-python-version }}
124+
use-uv: ${{ inputs.use-uv }}
120125
- name: "Test examples of PROD image building"
121126
run: "
122127
cd ./docker_tests && \
@@ -150,6 +155,7 @@ jobs:
150155
platform: "linux/amd64"
151156
image-type: "prod"
152157
python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
158+
use-uv: ${{ inputs.use-uv }}
153159
id: breeze
154160
- name: "Test docker-compose quick start"
155161
run: breeze testing docker-compose-tests

‎.github/workflows/basic-tests.yml

Lines changed: 75 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy
5656
description: "Whether to run only latest version checks (true/false)"
5757
required: true
5858
type: string
59+
use-uv:
60+
description: "Whether to use uv in the image"
61+
required: true
62+
type: string
5963
jobs:
6064
run-breeze-tests:
6165
timeout-minutes: 10
@@ -72,16 +76,12 @@ jobs:
7276
persist-credentials: false
7377
- name: "Cleanup docker"
7478
run: ./scripts/ci/cleanup_docker.sh
75-
- uses: actions/setup-python@v5
79+
- name: "Install Breeze"
80+
uses: ./.github/actions/breeze
7681
with:
77-
python-version: "${{ inputs.default-python-version }}"
78-
cache: 'pip'
79-
cache-dependency-path: ./dev/breeze/pyproject.toml
80-
- run: pip install --editable ./dev/breeze/
81-
- run: python -m pytest -n auto --color=yes
82+
use-uv: ${{ inputs.use-uv }}
83+
- run: uv tool run --from apache-airflow-breeze pytest -n auto --color=yes
8284
working-directory: ./dev/breeze/
83-
84-
8585
tests-ui:
8686
timeout-minutes: 10
8787
name: React UI tests
@@ -108,15 +108,24 @@ jobs:
108108
node-version: 21
109109
cache: 'pnpm'
110110
cache-dependency-path: 'airflow/ui/pnpm-lock.yaml'
111-
- name: "Cache eslint"
112-
uses: actions/cache@v4
111+
- name: "Restore eslint cache (ui)"
112+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
113113
with:
114-
path: 'airflow/ui/node_modules'
115-
key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
114+
path: airflow/ui/node_modules/
115+
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
116+
id: restore-eslint-cache
116117
- run: cd airflow/ui && pnpm install --frozen-lockfile
117118
- run: cd airflow/ui && pnpm test
118119
env:
119120
FORCE_COLOR: 2
121+
- name: "Save eslint cache (ui)"
122+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
123+
with:
124+
path: airflow/ui/node_modules/
125+
key: cache-ui-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/ui/**/pnpm-lock.yaml') }}
126+
if-no-files-found: 'error'
127+
retention-days: '2'
128+
if: steps.restore-eslint-cache.outputs.stash-hit != 'true'
120129

121130
tests-www:
122131
timeout-minutes: 10
@@ -137,15 +146,50 @@ jobs:
137146
uses: actions/setup-node@v4
138147
with:
139148
node-version: 21
140-
- name: "Cache eslint"
141-
uses: actions/cache@v4
149+
- name: "Restore eslint cache (www)"
150+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
142151
with:
143-
path: 'airflow/www/node_modules'
144-
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
152+
path: airflow/www/node_modules/
153+
key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
154+
id: restore-eslint-cache
145155
- run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
146156
- run: yarn --cwd airflow/www/ run test
147157
env:
148158
FORCE_COLOR: 2
159+
- name: "Save eslint cache (www)"
160+
uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
161+
with:
162+
path: airflow/www/node_modules/
163+
key: cache-www-node-modules-v1-${{ runner.os }}-${{ hashFiles('airflow/www/**/yarn.lock') }}
164+
if-no-files-found: 'error'
165+
retention-days: '2'
166+
if: steps.restore-eslint-cache.outputs.stash-hit != 'true'
167+
168+
install-pre-commit:
169+
timeout-minutes: 5
170+
name: "Install pre-commit for cache"
171+
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
172+
env:
173+
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
174+
if: inputs.basic-checks-only == 'true'
175+
steps:
176+
- name: "Cleanup repo"
177+
shell: bash
178+
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
179+
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
180+
uses: actions/checkout@v4
181+
with:
182+
persist-credentials: false
183+
- name: "Install Breeze"
184+
uses: ./.github/actions/breeze
185+
with:
186+
use-uv: ${{ inputs.use-uv }}
187+
id: breeze
188+
- name: "Install pre-commit"
189+
uses: ./.github/actions/install-pre-commit
190+
id: pre-commit
191+
with:
192+
python-version: ${{steps.breeze.outputs.host-python-version}}
149193

150194
# Those checks are run if no image needs to be built for checks. This is for simple changes that
151195
# Do not touch any of the python code or any of the important files that might require building
@@ -154,6 +198,7 @@ jobs:
154198
timeout-minutes: 30
155199
name: "Static checks: basic checks only"
156200
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
201+
needs: install-pre-commit
157202
if: inputs.basic-checks-only == 'true'
158203
steps:
159204
- name: "Cleanup repo"
@@ -165,20 +210,10 @@ jobs:
165210
persist-credentials: false
166211
- name: "Cleanup docker"
167212
run: ./scripts/ci/cleanup_docker.sh
168-
- name: "Setup python"
169-
uses: actions/setup-python@v5
170-
with:
171-
python-version: ${{ inputs.default-python-version }}
172-
cache: 'pip'
173-
cache-dependency-path: ./dev/breeze/pyproject.toml
174-
- name: "Setup python"
175-
uses: actions/setup-python@v5
176-
with:
177-
python-version: "${{ inputs.default-python-version }}"
178-
cache: 'pip'
179-
cache-dependency-path: ./dev/breeze/pyproject.toml
180213
- name: "Install Breeze"
181214
uses: ./.github/actions/breeze
215+
with:
216+
use-uv: ${{ inputs.use-uv }}
182217
id: breeze
183218
- name: "Install pre-commit"
184219
uses: ./.github/actions/install-pre-commit
@@ -216,6 +251,7 @@ jobs:
216251
timeout-minutes: 45
217252
name: "Upgrade checks"
218253
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
254+
needs: install-pre-commit
219255
env:
220256
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
221257
if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -229,12 +265,16 @@ jobs:
229265
persist-credentials: false
230266
- name: "Cleanup docker"
231267
run: ./scripts/ci/cleanup_docker.sh
232-
# Install python from scratch. No cache used. We always want to have fresh version of everything
233-
- uses: actions/setup-python@v5
268+
- name: "Install Breeze"
269+
uses: ./.github/actions/breeze
270+
with:
271+
use-uv: ${{ inputs.use-uv }}
272+
id: breeze
273+
- name: "Install pre-commit"
274+
uses: ./.github/actions/install-pre-commit
275+
id: pre-commit
234276
with:
235-
python-version: "${{ inputs.default-python-version }}"
236-
- name: "Install latest pre-commit"
237-
run: pip install pre-commit
277+
python-version: ${{steps.breeze.outputs.host-python-version}}
238278
- name: "Autoupdate all pre-commits"
239279
run: pre-commit autoupdate
240280
- name: "Run automated upgrade for black"
@@ -305,6 +345,8 @@ jobs:
305345
run: ./scripts/ci/cleanup_docker.sh
306346
- name: "Install Breeze"
307347
uses: ./.github/actions/breeze
348+
with:
349+
use-uv: ${{ inputs.use-uv }}
308350
- name: "Cleanup dist files"
309351
run: rm -fv ./dist/*
310352
- name: Setup git for tagging

0 commit comments

Comments
 (0)