Skip to content

Commit 1d31fb0

Browse files
pdurbinpdurbin
authored
Merge pull request #10827 from gdcc/10478-version-base-img
Releasing versioned (base) images
2 parents d4e9a4f + dc6b597 commit 1d31fb0

26 files changed

+861
-103
lines changed

‎.github/actions/setup-maven/action.yml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
name: "Setup Maven and Caches"
3+
description: "Determine Java version and setup Maven, including necessary caches."
4+
inputs:
5+
git-reference:
6+
description: 'The git reference (branch/tag) to check out'
7+
required: false
8+
default: '${{ github.ref }}'
9+
pom-paths:
10+
description: "List of paths to Maven POM(s) for cache dependency setup"
11+
required: false
12+
default: 'pom.xml'
13+
runs:
14+
using: composite
15+
steps:
16+
- name: Checkout repository
17+
uses: actions/checkout@v4
18+
with:
19+
ref: ${{ inputs.git-reference }}
20+
- name: Determine Java version by reading the Maven property
21+
shell: bash
22+
run: |
23+
echo "JAVA_VERSION=$(grep '<target.java.version>' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV}
24+
- name: Set up JDK ${{ env.JAVA_VERSION }}
25+
id: setup-java
26+
uses: actions/setup-java@v4
27+
with:
28+
java-version: ${{ env.JAVA_VERSION }}
29+
distribution: 'temurin'
30+
cache: 'maven'
31+
cache-dependency-path: ${{ inputs.pom-paths }}
32+
- name: Download common cache on branch cache miss
33+
if: ${{ steps.setup-java.outputs.cache-hit != 'true' }}
34+
uses: actions/cache/restore@v4
35+
with:
36+
key: dataverse-maven-cache
37+
path: ~/.m2/repository

‎.github/workflows/container_app_push.yml

+27-14
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ on:
55
# We are deliberately *not* running on push events here to avoid double runs.
66
# Instead, push events will trigger from the base image and maven unit tests via workflow_call.
77
workflow_call:
8+
inputs:
9+
base-image-ref:
10+
type: string
11+
description: "Reference of the base image to build on in full qualified form [<registry>/]<namespace>/<repo>:<tag>"
12+
required: false
13+
default: "gdcc/base:unstable"
814
pull_request:
915
branches:
1016
- develop
@@ -16,7 +22,6 @@ on:
1622

1723
env:
1824
IMAGE_TAG: unstable
19-
BASE_IMAGE_TAG: unstable
2025
REGISTRY: "" # Empty means default to Docker Hub
2126
PLATFORMS: "linux/amd64,linux/arm64"
2227
MASTER_BRANCH_TAG: alpha
@@ -33,20 +38,24 @@ jobs:
3338
if: ${{ github.repository_owner == 'IQSS' }}
3439

3540
steps:
36-
- name: Checkout repository
37-
uses: actions/checkout@v3
38-
39-
- name: Set up JDK
40-
uses: actions/setup-java@v3
41+
- name: Checkout and Setup Maven
42+
uses: IQSS/dataverse/.github/actions/setup-maven@develop
4143
with:
42-
java-version: "17"
43-
distribution: temurin
44-
cache: maven
44+
pom-paths: |
45+
pom.xml
46+
modules/container-configbaker/pom.xml
47+
modules/dataverse-parent/pom.xml
48+
49+
# TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above.
50+
# Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow
51+
# will be triggered by the other workflows already (base image or java changes)
52+
# To become a part of #10618.
4553

4654
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
4755
run: >
4856
mvn -B -f modules/dataverse-parent
4957
-P ct -pl edu.harvard.iq:dataverse -am
58+
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
5059
install
5160
5261
# TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
@@ -106,11 +115,13 @@ jobs:
106115
if: needs.check-secrets.outputs.available == 'true' &&
107116
( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
108117
steps:
109-
- uses: actions/checkout@v3
110-
- uses: actions/setup-java@v3
118+
- name: Checkout and Setup Maven
119+
uses: IQSS/dataverse/.github/actions/setup-maven@develop
111120
with:
112-
java-version: "17"
113-
distribution: temurin
121+
pom-paths: |
122+
pom.xml
123+
modules/container-configbaker/pom.xml
124+
modules/dataverse-parent/pom.xml
114125
115126
# Depending on context, we push to different targets. Login accordingly.
116127
- if: github.event_name != 'pull_request'
@@ -146,11 +157,13 @@ jobs:
146157
run: >
147158
mvn -B -f modules/dataverse-parent
148159
-P ct -pl edu.harvard.iq:dataverse -am
160+
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
149161
install
150162
- name: Deploy multi-arch application and configbaker container image
151163
run: >
152164
mvn
153-
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
165+
-Dapp.image.tag=${{ env.IMAGE_TAG }}
166+
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
154167
${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
155168
-P ct deploy
156169

‎.github/workflows/container_base_push.yml

+92-61
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,130 @@
11
---
2-
name: Base Container Image
2+
name: Container Images Releasing
33

44
on:
55
push:
6+
tags:
7+
- 'v[6-9].**'
68
branches:
79
- 'develop'
8-
- 'master'
10+
# "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore
911
paths:
1012
- 'modules/container-base/**'
13+
- '!modules/container-base/src/backports/**'
14+
- '!modules/container-base/README.md'
1115
- 'modules/dataverse-parent/pom.xml'
1216
- '.github/workflows/container_base_push.yml'
13-
pull_request:
14-
branches:
15-
- 'develop'
16-
- 'master'
17-
paths:
18-
- 'modules/container-base/**'
19-
- 'modules/dataverse-parent/pom.xml'
20-
- '.github/workflows/container_base_push.yml'
21-
schedule:
22-
- cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC
17+
18+
# These TODOs are left for #10618
19+
# TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!)
20+
# TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!)
2321

2422
env:
25-
IMAGE_TAG: unstable
2623
PLATFORMS: linux/amd64,linux/arm64
24+
DEVELOPMENT_BRANCH: develop
2725

2826
jobs:
2927
build:
30-
name: Build image
28+
name: Base Image
3129
runs-on: ubuntu-latest
3230
permissions:
3331
contents: read
3432
packages: read
35-
strategy:
36-
matrix:
37-
jdk: [ '17' ]
3833
# Only run in upstream repo - avoid unnecessary runs in forks
3934
if: ${{ github.repository_owner == 'IQSS' }}
35+
outputs:
36+
base-image-ref: ${{ steps.finalize.outputs.base-image-ref }}
4037

4138
steps:
42-
- name: Checkout repository
43-
uses: actions/checkout@v3
44-
45-
- name: Set up JDK ${{ matrix.jdk }}
46-
uses: actions/setup-java@v3
39+
- name: Checkout and Setup Maven
40+
uses: IQSS/dataverse/.github/actions/setup-maven@develop
4741
with:
48-
java-version: ${{ matrix.jdk }}
49-
distribution: 'adopt'
50-
- name: Cache Maven packages
51-
uses: actions/cache@v3
52-
with:
53-
path: ~/.m2
54-
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
55-
restore-keys: ${{ runner.os }}-m2
56-
57-
- name: Build base container image with local architecture
58-
run: mvn -f modules/container-base -Pct package
42+
pom-paths: modules/container-base/pom.xml
5943

60-
# Run anything below only if this is not a pull request.
61-
# Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets.
62-
63-
- if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
64-
name: Push description to DockerHub
65-
uses: peter-evans/dockerhub-description@v3
44+
# Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and
45+
# on events in context of upstream because secrets. PRs run in context of forks by default!
46+
- name: Log in to the Container registry
47+
uses: docker/login-action@v3
6648
with:
6749
username: ${{ secrets.DOCKERHUB_USERNAME }}
6850
password: ${{ secrets.DOCKERHUB_TOKEN }}
69-
repository: gdcc/base
70-
short-description: "Dataverse Base Container image providing Payara application server and optimized configuration"
71-
readme-filepath: ./modules/container-base/README.md
7251

73-
- if: ${{ github.event_name != 'pull_request' }}
74-
name: Log in to the Container registry
75-
uses: docker/login-action@v2
52+
# In case this is a push to develop, we care about buildtime.
53+
# Configure a remote ARM64 build host in addition to the local AMD64 in two steps.
54+
- name: Setup SSH agent
55+
if: ${{ github.event_name != 'schedule' }}
56+
uses: webfactory/[email protected]
7657
with:
77-
registry: ${{ env.REGISTRY }}
78-
username: ${{ secrets.DOCKERHUB_USERNAME }}
79-
password: ${{ secrets.DOCKERHUB_TOKEN }}
80-
- if: ${{ github.event_name != 'pull_request' }}
81-
name: Set up QEMU for multi-arch builds
82-
uses: docker/setup-qemu-action@v2
83-
- name: Re-set image tag based on branch
84-
if: ${{ github.ref_name == 'master' }}
85-
run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
86-
- if: ${{ github.event_name != 'pull_request' }}
87-
name: Deploy multi-arch base container image to Docker Hub
88-
run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }}
58+
ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }}
59+
- name: Provide the known hosts key and the builder config
60+
if: ${{ github.event_name != 'schedule' }}
61+
run: |
62+
echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts
63+
mkdir -p modules/container-base/target/buildx-state/buildx/instances
64+
cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF
65+
{ "Name": "maven",
66+
"Driver": "docker-container",
67+
"Dynamic": false,
68+
"Nodes": [{"Name": "maven0",
69+
"Endpoint": "unix:///var/run/docker.sock",
70+
"Platforms": [{"os": "linux", "architecture": "amd64"}],
71+
"DriverOpts": null,
72+
"Flags": ["--allow-insecure-entitlement=network.host"],
73+
"Files": null},
74+
{"Name": "maven1",
75+
"Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}",
76+
"Platforms": [{"os": "linux", "architecture": "arm64"}],
77+
"DriverOpts": null,
78+
"Flags": ["--allow-insecure-entitlement=network.host"],
79+
"Files": null}]}
80+
EOF
81+
82+
# Determine the base image name we are going to use from here on
83+
- name: Determine base image name
84+
run: |
85+
if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
86+
echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
87+
echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
88+
else
89+
echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
90+
fi
91+
- name: Calculate revision number for immutable tag (on release branches only)
92+
if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }}
93+
id: revision-tag
94+
uses: ./.github/actions/get-image-revision
95+
with:
96+
image-ref: ${{ env.BASE_IMAGE }}
97+
tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision="
98+
- name: Configure update of "latest" tag for development branch
99+
id: develop-tag
100+
if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }}
101+
run: |
102+
echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}"
103+
104+
- name: Deploy multi-arch base container image to Docker Hub
105+
id: build
106+
run: |
107+
mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \
108+
-Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }}
109+
110+
- name: Determine appropriate base image ref for app image
111+
id: finalize
112+
run: |
113+
if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
114+
echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
115+
else
116+
echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT"
117+
fi
118+
89119
push-app-img:
90120
name: "Rebase & Publish App Image"
91121
permissions:
92122
contents: read
93123
packages: write
94124
pull-requests: write
95-
needs: build
96-
# We do not release a new base image for pull requests, so do not trigger.
97-
if: ${{ github.event_name != 'pull_request' }}
98-
uses: ./.github/workflows/container_app_push.yml
99125
secrets: inherit
126+
needs:
127+
- build
128+
uses: ./.github/workflows/container_app_push.yml
129+
with:
130+
base-image-ref: ${{ needs.build.outputs.base-image-ref }}

0 commit comments

Comments
 (0)