Skip to content

Commit 33bfc75

Browse files
chewy-zlaichewy-zlai
authored
Add AWS to Artifact Distribution Script (#421)
## Summary Adds the ability to push artifacts to aws in addition to gcp. Also adds ability to specify specific customer ids to push to. ## Checklist - [ ] Added Unit Tests - [ ] Covered by existing CI - [ ] Integration tested - [ ] Documentation update <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced a new automation script that streamlines the process of building artifacts and deploying them to both AWS and GCP with improved error handling and user confirmation. - **Chores** - Removed a legacy artifact upload script that previously handled only GCP deployments. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent bd4064d commit 33bfc75

File tree

2 files changed

+251
-129
lines changed

2 files changed

+251
-129
lines changed

‎distribution/build_and_upload_artifacts.sh

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
#!/bin/bash
2+
3+
function print_usage() {
4+
echo "Usage: $0 [OPTIONS]"
5+
echo "Options:"
6+
echo " --all Build and upload all artifacts (GCP and AWS)"
7+
echo " --gcp Build and upload only GCP artifacts"
8+
echo " --aws Build and upload only AWS artifacts"
9+
echo " --aws_customer_ids <customer_id> Specify AWS customer IDs to upload artifacts to."
10+
echo " --gcp_customer_ids <customer_id> Specify GCP customer IDs to upload artifacts to."
11+
echo " -h, --help Show this help message"
12+
}
13+
14+
# No arguments provided
15+
if [ $# -eq 0 ]; then
16+
print_usage
17+
exit 1
18+
fi
19+
20+
BUILD_AWS=false
21+
BUILD_GCP=false
22+
23+
while [[ $# -gt 0 ]]; do
24+
case $1 in
25+
--all)
26+
BUILD_GCP=true
27+
BUILD_AWS=true
28+
shift
29+
;;
30+
--gcp)
31+
BUILD_GCP=true
32+
shift
33+
;;
34+
--aws)
35+
BUILD_AWS=true
36+
shift
37+
;;
38+
-h|--help)
39+
print_usage
40+
exit 0
41+
;;
42+
--aws_customer_ids)
43+
if [[ -z $2 ]]; then
44+
echo "Error: --customer_ids requires a value"
45+
print_usage
46+
exit 1
47+
fi
48+
INPUT_AWS_CUSTOMER_IDS=("$2")
49+
shift 2
50+
;;
51+
--gcp_customer_ids)
52+
if [[ -z $2 ]]; then
53+
echo "Error: --customer_ids requires a value"
54+
print_usage
55+
exit 1
56+
fi
57+
INPUT_GCP_CUSTOMER_IDS=("$2")
58+
shift 2
59+
;;
60+
*)
61+
echo "Unknown option: $1"
62+
print_usage
63+
exit 1
64+
;;
65+
esac
66+
done
67+
68+
69+
if [[ -n $(git diff HEAD) ]]; then
70+
echo "Error: You have uncommitted changes. Please commit and push them to git so we can track them."
71+
exit 1
72+
fi
73+
74+
# Get current branch name
75+
local_branch=$(git rev-parse --abbrev-ref HEAD)
76+
77+
# Fetch latest from remote
78+
git fetch origin $local_branch
79+
80+
# Check if local is behind remote
81+
if [[ -n $(git diff HEAD..origin/$local_branch) ]]; then
82+
echo "Error: Your branch is not in sync with remote"
83+
echo "Please push your local changes and sync your local branch $local_branch with remote"
84+
exit 1
85+
fi
86+
87+
set -e
88+
89+
SCRIPT_DIRECTORY=$(dirname -- "$(realpath -- "$0")")
90+
CHRONON_ROOT_DIR=$(dirname "$SCRIPT_DIRECTORY")
91+
92+
echo "Working in $CHRONON_ROOT_DIR"
93+
cd $CHRONON_ROOT_DIR
94+
95+
echo "Building wheel"
96+
#Check python version >= 3.9
97+
MAJOR_PYTHON_VERSION=$(python --version | cut -d " " -f2 | cut -d "." -f 1)
98+
MINOR_PYTHON_VERSION=$(python --version | cut -d " " -f2 | cut -d "." -f 2)
99+
100+
EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION=3
101+
EXPECTED_MINIMUM_MINOR_PYTHON_VERSION=9
102+
103+
if [[ $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION -gt $MAJOR_PYTHON_VERSION ]] ; then
104+
echo "Failed major version of $MAJOR_PYTHON_VERSION. Expecting python version of at least $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION.$EXPECTED_MINIMUM_MINOR_PYTHON_VERSION to build wheel. Your version is $(python --version)"
105+
exit 1
106+
fi
107+
108+
if [[ $EXPECTED_MINIMUM_MINOR_PYTHON_VERSION -gt $MINOR_PYTHON_VERSION ]] ; then
109+
echo "Failed minor version of $MINOR_PYTHON_VERSION. Expecting python version of at least $EXPECTED_MINIMUM_MAJOR_PYTHON_VERSION.$EXPECTED_MINIMUM_MINOR_PYTHON_VERSION to build wheel. Your version is $(python --version)"
110+
exit 1
111+
fi
112+
113+
114+
thrift --gen py -out api/py/ api/thrift/common.thrift
115+
thrift --gen py -out api/py/ api/thrift/api.thrift
116+
thrift --gen py -out api/py/ api/thrift/observability.thrift
117+
VERSION=$(cat version.sbt | cut -d " " -f3 | tr -d '"') pip wheel api/py
118+
EXPECTED_ZIPLINE_WHEEL="zipline_ai-0.1.0.dev0-py3-none-any.whl"
119+
if [ ! -f "$EXPECTED_ZIPLINE_WHEEL" ]; then
120+
echo "$EXPECTED_ZIPLINE_WHEEL not found"
121+
exit 1
122+
fi
123+
124+
echo "Building jars"
125+
126+
bazel build //flink:flink_assembly_deploy.jar
127+
bazel build //service:service_assembly_deploy.jar
128+
129+
FLINK_JAR="$CHRONON_ROOT_DIR/bazel-bin/flink/flink_assembly_deploy.jar"
130+
SERVICE_JAR="$CHRONON_ROOT_DIR/bazel-bin/service/service_assembly_deploy.jar"
131+
132+
if [ ! -f "$SERVICE_JAR" ]; then
133+
echo "$SERVICE_JAR not found"
134+
exit 1
135+
fi
136+
137+
if [ ! -f "$FLINK_JAR" ]; then
138+
echo "$FLINK_JAR not found"
139+
exit 1
140+
fi
141+
142+
143+
144+
if [ "$BUILD_AWS" = true ]; then
145+
bazel build //cloud_aws:cloud_aws_lib_deploy.jar
146+
147+
CLOUD_AWS_JAR="$CHRONON_ROOT_DIR/bazel-bin/cloud_aws/cloud_aws_lib_deploy.jar"
148+
149+
if [ ! -f "$CLOUD_AWS_JAR" ]; then
150+
echo "$CLOUD_AWS_JAR not found"
151+
exit 1
152+
fi
153+
fi
154+
if [ "$BUILD_GCP" = true ]; then
155+
bazel build //cloud_gcp:cloud_gcp_lib_deploy.jar
156+
bazel build //cloud_gcp:cloud_gcp_submitter_deploy.jar
157+
158+
CLOUD_GCP_JAR="$CHRONON_ROOT_DIR/bazel-bin/cloud_gcp/cloud_gcp_lib_deploy.jar"
159+
CLOUD_GCP_SUBMITTER_JAR="$CHRONON_ROOT_DIR/bazel-bin/cloud_gcp/cloud_gcp_submitter_deploy.jar"
160+
161+
if [ ! -f "$CLOUD_GCP_JAR" ]; then
162+
echo "$CLOUD_GCP_JAR not found"
163+
exit 1
164+
fi
165+
166+
if [ ! -f "$CLOUD_GCP_SUBMITTER_JAR" ]; then
167+
echo "$CLOUD_GCP_SUBMITTER_JAR not found"
168+
exit 1
169+
fi
170+
171+
fi
172+
173+
174+
175+
176+
# all customer ids
177+
GCP_CUSTOMER_IDS=("canary" "etsy")
178+
179+
# Takes in array of customer ids
180+
function upload_to_gcp() {
181+
# Disabling this so that we can set the custom metadata on these jars
182+
gcloud config set storage/parallel_composite_upload_enabled False
183+
customer_ids_to_upload=("$@")
184+
echo "Are you sure you want to upload to these customer ids: ${customer_ids_to_upload[*]}"
185+
select yn in "Yes" "No"; do
186+
case $yn in
187+
Yes )
188+
set -euxo pipefail
189+
for element in "${customer_ids_to_upload[@]}"
190+
do
191+
ELEMENT_JAR_PATH=gs://zipline-artifacts-$element/jars
192+
gcloud storage cp "$CLOUD_GCP_JAR" "$ELEMENT_JAR_PATH" --custom-metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
193+
gcloud storage cp "$CLOUD_GCP_SUBMITTER_JAR" "$ELEMENT_JAR_PATH" --custom-metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
194+
gcloud storage cp "$SERVICE_JAR" "$ELEMENT_JAR_PATH" --custom-metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
195+
gcloud storage cp "$EXPECTED_ZIPLINE_WHEEL" "$ELEMENT_JAR_PATH" --custom-metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
196+
gcloud storage cp "$FLINK_JAR" "$ELEMENT_JAR_PATH" --custom-metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
197+
done
198+
echo "Succeeded"
199+
break;;
200+
No ) break;;
201+
esac
202+
done
203+
gcloud config set storage/parallel_composite_upload_enabled True
204+
}
205+
206+
AWS_CUSTOMER_IDS=("canary")
207+
208+
# Takes in array of customer ids
209+
function upload_to_aws() {
210+
customer_ids_to_upload=("$@")
211+
echo "Are you sure you want to upload to these customer ids: ${customer_ids_to_upload[*]}"
212+
select yn in "Yes" "No"; do
213+
case $yn in
214+
Yes )
215+
set -euxo pipefail
216+
for element in "${customer_ids_to_upload[@]}"
217+
do
218+
ELEMENT_JAR_PATH=s3://zipline-artifacts-$element/jars/
219+
aws s3 cp "$CLOUD_AWS_JAR" "$ELEMENT_JAR_PATH" --metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
220+
aws s3 cp "$SERVICE_JAR" "$ELEMENT_JAR_PATH" --metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
221+
aws s3 cp "$EXPECTED_ZIPLINE_WHEEL" "$ELEMENT_JAR_PATH" --metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
222+
aws s3 cp "$FLINK_JAR" "$ELEMENT_JAR_PATH" --metadata="zipline_user=$USER,updated_date=$(date),commit=$(git rev-parse HEAD),branch=$(git rev-parse --abbrev-ref HEAD)"
223+
done
224+
echo "Succeeded"
225+
break;;
226+
No ) break;;
227+
esac
228+
done
229+
}
230+
231+
232+
if [ "$BUILD_AWS" = true ]; then
233+
if [ ${#INPUT_AWS_CUSTOMER_IDS[@]} -eq 0 ]; then
234+
echo "No customer ids provided for AWS. Using default: ${AWS_CUSTOMER_IDS[*]}"
235+
else
236+
AWS_CUSTOMER_IDS=("${INPUT_AWS_CUSTOMER_IDS[@]}")
237+
fi
238+
upload_to_aws "${AWS_CUSTOMER_IDS[@]}"
239+
fi
240+
if [ "$BUILD_GCP" = true ]; then
241+
if [ ${#INPUT_GCP_CUSTOMER_IDS[@]} -eq 0 ]; then
242+
echo "No customer ids provided for GCP. Using default: ${GCP_CUSTOMER_IDS[*]}"
243+
else
244+
GCP_CUSTOMER_IDS=("${INPUT_GCP_CUSTOMER_IDS[@]}")
245+
fi
246+
upload_to_gcp "${GCP_CUSTOMER_IDS[@]}"
247+
fi
248+
249+
250+
# Cleanup wheel stuff
251+
rm ./*.whl

‎distribution/build_and_upload_gcp_artifacts.sh

Lines changed: 0 additions & 129 deletions
This file was deleted.

0 commit comments

Comments
 (0)