Skip to content

Commit 2fe7d33

Browse files
chewy-zlaichewy-zlai
committed
workflow to push quickstart to GCP and AWS
1 parent cc767f3 commit 2fe7d33

File tree

2 files changed

+164
-0
lines changed

2 files changed

+164
-0
lines changed

‎.github/workflows/push_to_canary.yaml

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
name: Push To Canary
2+
3+
on:
4+
push:
5+
# branches:
6+
# - 'main'
7+
8+
concurrency:
9+
group: ${{ github.workflow }}-${{ github.ref }}
10+
cancel-in-progress: true
11+
12+
env:
13+
AWS_REGION: ${{secrets.AWS_REGION}}
14+
AWS_APP_REPOSITORY: ${{secrets.APP_REPOSITORY}}
15+
AWS_FRONTEND_REPOSITORY: ${{secrets.FRONTEND_REPOSITORY}}
16+
AWS_ACCOUNT_ID: ${{secrets.AWS_ACCOUNT_ID}}
17+
GCP_REGION: us-west1
18+
GCP_PROJECT_ID: canary-443022
19+
GAR_QUICKSTART_REPOSITORY: us-west1-docker.pkg.dev/canary-443022/canary-images/quickstart
20+
21+
jobs:
22+
Push-To-ECR:
23+
runs-on: ubuntu-latest
24+
25+
permissions:
26+
id-token: write
27+
contents: read
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Set up QEMU
33+
uses: docker/setup-qemu-action@v3
34+
35+
- name: Setup JDK
36+
uses: actions/setup-java@v4
37+
with:
38+
distribution: corretto
39+
java-version: 17
40+
41+
- name: Install Thrift
42+
env:
43+
THRIFT_VERSION: 0.21.0
44+
run: |
45+
sudo apt-get install automake bison flex g++ git libboost-all-dev libevent-dev libssl-dev libtool make pkg-config && \
46+
curl -LSs https://archive.apache.org/dist/thrift/${{env.THRIFT_VERSION}}/thrift-${{env.THRIFT_VERSION}}.tar.gz -o thrift-${{env.THRIFT_VERSION}}.tar.gz && \
47+
tar -xzf thrift-${{env.THRIFT_VERSION}}.tar.gz && \
48+
cd thrift-${{env.THRIFT_VERSION}} && \
49+
sudo ./configure --without-python --without-cpp --without-nodejs --without-java --disable-debug --disable-tests --disable-libs && \
50+
sudo make && \
51+
sudo make install && \
52+
cd .. && \
53+
sudo rm -rf thrift-${{env.THRIFT_VERSION}} thrift-${{env.THRIFT_VERSION}}.tar.gz
54+
55+
56+
- name: Build SBT Project
57+
id: sbt-dist
58+
run: |
59+
sbt clean && sbt assembly
60+
sbt dist
61+
sbt "project frontend" buildFrontend
62+
63+
- name: Build AWS Quickstart Image
64+
id: build-aws-app
65+
shell: bash
66+
env:
67+
USER: root
68+
SPARK_SUBMIT_PATH: spark-submit
69+
PYTHONPATH: /srv/chronon
70+
SPARK_VERSION: 3.1.1
71+
JOB_MODE: local[*]
72+
PARALLELISM: 2
73+
EXECUTOR_MEMORY: 2G
74+
EXECUTOR_CORES: 4
75+
DRIVER_MEMORY: 1G
76+
CHRONON_LOG_TABLE: default.chronon_log_table
77+
CHRONON_ONLINE_CLASS: ai.chronon.integrations.aws.AwsApiImpl
78+
AWS_DEFAULT_REGION: ${{env.AWS_REGION}}
79+
DYNAMO_ENDPOINT: https://dynamodb.${{env.AWS_REGION}}.amazonaws.com
80+
JAVA_OPTS: "-Xms1g -Xmx1g"
81+
CLOUD_AWS_JAR: /app/cli/cloud_aws.jar
82+
run:
83+
docker build "." -f "./Dockerfile" -t "aws-quickstart-image:latest"
84+
85+
- name: Build GCP Quickstart Image
86+
id: build-gcp-app
87+
shell: bash
88+
env:
89+
USER: root
90+
SPARK_SUBMIT_PATH: spark-submit
91+
PYTHONPATH: /srv/chronon
92+
SPARK_VERSION: 3.1.1
93+
JOB_MODE: local[*]
94+
PARALLELISM: 2
95+
EXECUTOR_MEMORY: 2G
96+
EXECUTOR_CORES: 4
97+
DRIVER_MEMORY: 1G
98+
CHRONON_LOG_TABLE: default.chronon_log_table
99+
CHRONON_ONLINE_CLASS: ai.chronon.integrations.cloud_gcp.GcpApiImpl
100+
GCP_DEFAULT_REGION: ${{env.GCP_REGION}}
101+
BIGTABLE_ENDPOINT: https://${{env.GCP_REGION}}-bigtable.googleapis.com
102+
JAVA_OPTS: "-Xms1g -Xmx1g"
103+
CLOUD_GCP_JAR: /app/cli/cloud_gcp.jar
104+
run:
105+
docker build "." -f "./Dockerfile" -t "gcp-quickstart-image:latest"
106+
107+
- name: Configure AWS Credentials
108+
uses: aws-actions/configure-aws-credentials@v4
109+
with:
110+
role-to-assume: arn:aws:iam::${{env.AWS_ACCOUNT_ID}}:role/github-canary-updater
111+
aws-region: ${{env.AWS_REGION}}
112+
113+
114+
- name: Login to Amazon ECR
115+
id: login-ecr
116+
uses: aws-actions/amazon-ecr-login@v2
117+
with:
118+
registries: ${{env.AWS_ACCOUNT_ID}}
119+
120+
- name: Tag, and push quickstart image to Amazon ECR
121+
env:
122+
ECR_REPOSITORY: ${{steps.login-ecr.outputs.registry}}/${{env.AWS_QUICKSTART_REPOSITORY}}
123+
IMAGE_TAG: canary
124+
shell: bash
125+
run: |
126+
docker tag "aws-quickstart-image:latest" "${{env.ECR_REPOSITORY}}:$IMAGE_TAG"
127+
docker push "${{env.ECR_REPOSITORY}}:$IMAGE_TAG"
128+
docker tag "${{env.ECR_REPOSITORY}}:$IMAGE_TAG" "${{env.ECR_REPOSITORY}}:${{github.sha}}"
129+
docker push "${{env.ECR_REPOSITORY}}:${{github.sha}}"
130+
echo "IMAGE $IMAGE_TAG is pushed to ${{env.ECR_REPOSITORY}}"
131+
echo "image_tag=$IMAGE_TAG"
132+
echo "full_image=${{env.ECR_REPOSITORY}}:$IMAGE_TAG"
133+
134+
- name: Configure GCP Credentials
135+
uses: google-github-actions/auth@v2
136+
with:
137+
project_id: ${{env.GCP_PROJECT_ID}}
138+
workload_identity_provider: projects/703996152583/locations/global/workloadIdentityPools/github-actions/providers/github
139+
service_account: [email protected]
140+
141+
- name: Set up Google Cloud SDK
142+
uses: google-github-actions/setup-gcloud@v2
143+
144+
- name: Google Cloud Docker Auth
145+
shell: bash
146+
run: |-
147+
gcloud auth configure-docker ${{env.GCP_REGION}}-docker.pkg.dev --quiet
148+
149+
- name: Push Quickstart to Artifact Registry
150+
shell: bash
151+
env:
152+
IMAGE_TAG: canary
153+
run: |
154+
docker tag "gcp-quickstart-image:latest" "${{env.GAR_QUICKSTART_REPOSITORY}}:$IMAGE_TAG"
155+
docker push "${{env.GAR_QUICKSTART_REPOSITORY}}:$IMAGE_TAG"
156+
docker tag "${{env.GAR_QUICKSTART_REPOSITORY}}:$IMAGE_TAG" "${{env.GAR_QUICKSTART_REPOSITORY}}:${{github.sha}}"
157+
docker push "${{env.GAR_QUICKSTART_REPOSITORY}}:${{github.sha}}"
158+
echo "IMAGE $IMAGE_TAG is pushed to ${{env.GAR_QUICKSTART_REPOSITORY}}"
159+
echo "image_tag=$IMAGE_TAG"
160+
echo "full_image=${{env.GAR_QUICKSTART_REPOSITORY}}:$IMAGE_TAG"

‎Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ FROM openjdk:11-jdk-slim
44
# Set this manually before building the image, requires a local build of the jar
55

66
ENV CHRONON_JAR_PATH=spark/target/scala-2.12/spark-assembly-0.1.0-SNAPSHOT.jar
7+
ENV CLOUD_AWS_JAR_PATH=cloud_aws/target/scala-2.12/cloud_aws-assembly-0.1.0-SNAPSHOT.jar
8+
ENV CLOUD_GCP_JAR_PATH=cloud_gcp/target/scala-2.12/cloud_gcp-assembly-0.1.0-SNAPSHOT.jar
79

810
# Update package lists and install necessary tools
911
RUN apt-get update && apt-get install -y \
@@ -79,5 +81,7 @@ ENV DRIVER_JAR_PATH="/srv/spark/spark_embedded.jar"
7981
COPY api/py/test/sample ./
8082
COPY quickstart/mongo-online-impl /srv/onlineImpl
8183
COPY $CHRONON_JAR_PATH "$DRIVER_JAR_PATH"
84+
COPY $CLOUD_AWS_JAR_PATH "$CLOUD_AWS_JAR"
85+
COPY $CLOUD_GCP_JAR_PATH "$CLOUD_GCP_JAR"
8286

8387
ENV CHRONON_DRIVER_JAR="$DRIVER_JAR_PATH"

0 commit comments

Comments
 (0)