Skip to content

Commit f0bcc8b

Browse files
kcondonkcondon
authored
Merge pull request #9754 from IQSS/develop
merge v5.14 into master
2 parents 79d6e57 + b215f56 commit f0bcc8b

File tree

469 files changed

+31899
-7199
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

469 files changed

+31899
-7199
lines changed

‎.env

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
APP_IMAGE=gdcc/dataverse:unstable
2+
POSTGRES_VERSION=13
3+
DATAVERSE_DB_USER=dataverse
4+
SOLR_VERSION=8.11.1

‎.github/workflows/container_app_pr.yml

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
---
2+
name: Preview Application Container Image
3+
4+
on:
5+
# We only run the push commands if we are asked to by an issue comment with the correct command.
6+
# This workflow is always taken from the default branch and runs in repo context with access to secrets.
7+
repository_dispatch:
8+
types: [ push-image-command ]
9+
10+
env:
11+
IMAGE_TAG: unstable
12+
BASE_IMAGE_TAG: unstable
13+
PLATFORMS: "linux/amd64,linux/arm64"
14+
15+
jobs:
16+
deploy:
17+
name: "Package & Push"
18+
runs-on: ubuntu-latest
19+
# Only run in upstream repo - avoid unnecessary runs in forks
20+
if: ${{ github.repository_owner == 'IQSS' }}
21+
steps:
22+
# Checkout the pull request code as when merged
23+
- uses: actions/checkout@v3
24+
with:
25+
ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
26+
- uses: actions/setup-java@v3
27+
with:
28+
java-version: "11"
29+
distribution: 'adopt'
30+
- uses: actions/cache@v3
31+
with:
32+
path: ~/.m2
33+
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
34+
restore-keys: ${{ runner.os }}-m2
35+
36+
# Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
37+
- name: Login to Github Container Registry
38+
uses: docker/login-action@v2
39+
with:
40+
registry: ghcr.io
41+
username: ${{ secrets.GHCR_USERNAME }}
42+
password: ${{ secrets.GHCR_TOKEN }}
43+
44+
- name: Set up QEMU for multi-arch builds
45+
uses: docker/setup-qemu-action@v2
46+
47+
# Get the image tag from either the command or default to branch name (Not used for now)
48+
#- name: Get the target tag name
49+
# id: vars
50+
# run: |
51+
# tag=${{ github.event.client_payload.slash_command.args.named.tag }}
52+
# if [[ -z "$tag" ]]; then tag=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-'); fi
53+
# echo "IMAGE_TAG=$tag" >> $GITHUB_ENV
54+
55+
# Set image tag to branch name of the PR
56+
- name: Set image tag to branch name
57+
run: |
58+
echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
59+
60+
# Necessary to split as otherwise the submodules are not available (deploy skips install)
61+
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
62+
run: >
63+
mvn -B -f modules/dataverse-parent
64+
-P ct -pl edu.harvard.iq:dataverse -am
65+
install
66+
- name: Deploy multi-arch application and configbaker container image
67+
run: >
68+
mvn
69+
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
70+
-Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
71+
-Pct deploy
72+
73+
- uses: marocchino/sticky-pull-request-comment@v2
74+
with:
75+
header: registry-push
76+
hide_and_recreate: true
77+
hide_classify: "OUTDATED"
78+
number: ${{ github.event.client_payload.pull_request.number }}
79+
message: |
80+
:package: Pushed preview images as
81+
```
82+
ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
83+
```
84+
```
85+
ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
86+
```
87+
:ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
88+
89+
# Leave a note when things have gone sideways
90+
- uses: peter-evans/create-or-update-comment@v3
91+
if: ${{ failure() }}
92+
with:
93+
issue-number: ${{ github.event.client_payload.pull_request.number }}
94+
body: >
95+
:package: Could not push preview images :disappointed:.
96+
See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.

‎.github/workflows/container_app_push.yml

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
---
2+
name: Application Container Image
3+
4+
on:
5+
# We are deliberately *not* running on push events here to avoid double runs.
6+
# Instead, push events will trigger from the base image and maven unit tests via workflow_call.
7+
workflow_call:
8+
pull_request:
9+
branches:
10+
- develop
11+
- master
12+
paths:
13+
- 'src/main/docker/**'
14+
- 'modules/container-configbaker/**'
15+
- '.github/workflows/container_app_push.yml'
16+
17+
env:
18+
IMAGE_TAG: unstable
19+
BASE_IMAGE_TAG: unstable
20+
REGISTRY: "" # Empty means default to Docker Hub
21+
PLATFORMS: "linux/amd64,linux/arm64"
22+
MASTER_BRANCH_TAG: alpha
23+
24+
jobs:
25+
build:
26+
name: "Build & Test"
27+
runs-on: ubuntu-latest
28+
permissions:
29+
contents: read
30+
packages: write
31+
pull-requests: write
32+
# Only run in upstream repo - avoid unnecessary runs in forks
33+
if: ${{ github.repository_owner == 'IQSS' }}
34+
35+
steps:
36+
- name: Checkout repository
37+
uses: actions/checkout@v3
38+
39+
- name: Set up JDK 11
40+
uses: actions/setup-java@v3
41+
with:
42+
java-version: "11"
43+
distribution: temurin
44+
cache: maven
45+
46+
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
47+
run: >
48+
mvn -B -f modules/dataverse-parent
49+
-P ct -pl edu.harvard.iq:dataverse -am
50+
install
51+
52+
# TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
53+
54+
hub-description:
55+
needs: build
56+
name: Push image descriptions to Docker Hub
57+
# Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
58+
# Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
59+
if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/checkout@v3
63+
- uses: peter-evans/dockerhub-description@v3
64+
with:
65+
username: ${{ secrets.DOCKERHUB_USERNAME }}
66+
password: ${{ secrets.DOCKERHUB_TOKEN }}
67+
repository: gdcc/dataverse
68+
short-description: "Dataverse Application Container Image providing the executable"
69+
readme-filepath: ./src/main/docker/README.md
70+
- uses: peter-evans/dockerhub-description@v3
71+
with:
72+
username: ${{ secrets.DOCKERHUB_USERNAME }}
73+
password: ${{ secrets.DOCKERHUB_TOKEN }}
74+
repository: gdcc/configbaker
75+
short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
76+
readme-filepath: ./modules/container-configbaker/README.md
77+
78+
# Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
79+
# We check for them here and subsequent jobs can rely on this to decide if they shall run.
80+
check-secrets:
81+
needs: build
82+
name: Check for Secrets Availability
83+
runs-on: ubuntu-latest
84+
outputs:
85+
available: ${{ steps.secret-check.outputs.available }}
86+
steps:
87+
- id: secret-check
88+
# perform secret check & put boolean result as an output
89+
shell: bash
90+
run: |
91+
if [ "${{ secrets.DOCKERHUB_TOKEN }}" != '' ]; then
92+
echo "available=true" >> $GITHUB_OUTPUT;
93+
else
94+
echo "available=false" >> $GITHUB_OUTPUT;
95+
fi
96+
97+
deploy:
98+
needs: check-secrets
99+
name: "Package & Publish"
100+
runs-on: ubuntu-latest
101+
# Only run this job if we have access to secrets. This is true for events like push/schedule which run in
102+
# context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
103+
if: needs.check-secrets.outputs.available == 'true'
104+
steps:
105+
- uses: actions/checkout@v3
106+
- uses: actions/setup-java@v3
107+
with:
108+
java-version: "11"
109+
distribution: temurin
110+
111+
# Depending on context, we push to different targets. Login accordingly.
112+
- if: ${{ github.event_name != 'pull_request' }}
113+
name: Log in to Docker Hub registry
114+
uses: docker/login-action@v2
115+
with:
116+
username: ${{ secrets.DOCKERHUB_USERNAME }}
117+
password: ${{ secrets.DOCKERHUB_TOKEN }}
118+
- if: ${{ github.event_name == 'pull_request' }}
119+
name: Login to Github Container Registry
120+
uses: docker/login-action@v2
121+
with:
122+
registry: ghcr.io
123+
username: ${{ secrets.GHCR_USERNAME }}
124+
password: ${{ secrets.GHCR_TOKEN }}
125+
126+
- name: Set up QEMU for multi-arch builds
127+
uses: docker/setup-qemu-action@v2
128+
129+
- name: Re-set image tag based on branch (if master)
130+
if: ${{ github.ref_name == 'master' }}
131+
run: |
132+
echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
133+
echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
134+
- name: Re-set image tag and container registry when on PR
135+
if: ${{ github.event_name == 'pull_request' }}
136+
run: |
137+
echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
138+
echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
139+
140+
# Necessary to split as otherwise the submodules are not available (deploy skips install)
141+
- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
142+
run: >
143+
mvn -B -f modules/dataverse-parent
144+
-P ct -pl edu.harvard.iq:dataverse -am
145+
install
146+
- name: Deploy multi-arch application and configbaker container image
147+
run: >
148+
mvn
149+
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
150+
${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
151+
-P ct deploy
152+
153+
- uses: marocchino/sticky-pull-request-comment@v2
154+
if: ${{ github.event_name == 'pull_request' }}
155+
with:
156+
header: registry-push
157+
hide_and_recreate: true
158+
hide_classify: "OUTDATED"
159+
message: |
160+
:package: Pushed preview images as
161+
```
162+
ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
163+
```
164+
```
165+
ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
166+
```
167+
:ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.

‎.github/workflows/container_base_push.yml

+17-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
name: Container Base Module
2+
name: Base Container Image
33

44
on:
55
push:
@@ -18,9 +18,12 @@ on:
1818
- 'modules/container-base/**'
1919
- 'modules/dataverse-parent/pom.xml'
2020
- '.github/workflows/container_base_push.yml'
21+
schedule:
22+
- cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC
2123

2224
env:
2325
IMAGE_TAG: unstable
26+
PLATFORMS: linux/amd64,linux/arm64
2427

2528
jobs:
2629
build:
@@ -79,7 +82,18 @@ jobs:
7982
uses: docker/setup-qemu-action@v2
8083
- name: Re-set image tag based on branch
8184
if: ${{ github.ref_name == 'master' }}
82-
run: echo "IMAGE_TAG=stable"
85+
run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
8386
- if: ${{ github.event_name != 'pull_request' }}
8487
name: Deploy multi-arch base container image to Docker Hub
85-
run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }}
88+
run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }}
89+
push-app-img:
90+
name: "Rebase & Publish App Image"
91+
permissions:
92+
contents: read
93+
packages: write
94+
pull-requests: write
95+
needs: build
96+
# We do not release a new base image for pull requests, so do not trigger.
97+
if: ${{ github.event_name != 'pull_request' }}
98+
uses: ./.github/workflows/container_app_push.yml
99+
secrets: inherit

‎.github/workflows/deploy_beta_testing.yml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: 'Deploy to Beta Testing'
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
environment: beta-testing
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
16+
- uses: actions/setup-java@v3
17+
with:
18+
distribution: 'zulu'
19+
java-version: '11'
20+
21+
- name: Build application war
22+
run: mvn package
23+
24+
- name: Get war file name
25+
working-directory: target
26+
run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
27+
28+
- name: Upload war artifact
29+
uses: actions/upload-artifact@v3
30+
with:
31+
name: built-app
32+
path: ./target/${{ env.war_file }}
33+
34+
deploy-to-payara:
35+
needs: build
36+
runs-on: ubuntu-latest
37+
environment: beta-testing
38+
39+
steps:
40+
- uses: actions/checkout@v3
41+
42+
- name: Download war artifact
43+
uses: actions/download-artifact@v3
44+
with:
45+
name: built-app
46+
path: ./
47+
48+
- name: Get war file name
49+
run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
50+
51+
- name: Copy war file to remote instance
52+
uses: appleboy/scp-action@master
53+
with:
54+
host: ${{ secrets.PAYARA_INSTANCE_HOST }}
55+
username: ${{ secrets.PAYARA_INSTANCE_USERNAME }}
56+
key: ${{ secrets.PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
57+
source: './${{ env.war_file }}'
58+
target: '/home/${{ secrets.PAYARA_INSTANCE_USERNAME }}'
59+
overwrite: true
60+
61+
- name: Execute payara war deployment remotely
62+
uses: appleboy/[email protected]
63+
env:
64+
INPUT_WAR_FILE: ${{ env.war_file }}
65+
with:
66+
host: ${{ secrets.PAYARA_INSTANCE_HOST }}
67+
username: ${{ secrets.PAYARA_INSTANCE_USERNAME }}
68+
key: ${{ secrets.PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
69+
envs: INPUT_WAR_FILE
70+
script: |
71+
APPLICATION_NAME=dataverse-backend
72+
ASADMIN='/usr/local/payara5/bin/asadmin --user admin'
73+
$ASADMIN undeploy $APPLICATION_NAME
74+
$ASADMIN stop-domain
75+
rm -rf /usr/local/payara5/glassfish/domains/domain1/generated
76+
rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache
77+
$ASADMIN start-domain
78+
$ASADMIN deploy --name $APPLICATION_NAME $INPUT_WAR_FILE
79+
$ASADMIN stop-domain
80+
$ASADMIN start-domain

0 commit comments

Comments
 (0)