Skip to content

Commit 804bf47

Browse files
supertophersupertopher
authored
Major overhaul of check images (#17979)
* Major overhaul of check images actually fail on missed images detect rate limit for dockerhub sleep when hit rate limit and registry more clear user messaging Reduce output from 5000 lines to ~ 1000 More clear help messages for failures * Addresses GH feedback minor bugfixes
1 parent e5fccaa commit 804bf47

File tree

1 file changed

+87
-28
lines changed

1 file changed

+87
-28
lines changed

‎tools/bin/check_images_exist.sh

Lines changed: 87 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,137 @@
11
#!/usr/bin/env bash
22

3-
set -e
3+
# ------------- Import some defaults for the shell
4+
5+
# Source shell defaults
6+
# $0 is the currently running program (this file)
7+
this_file_directory=$(dirname $0)
8+
relative_path_to_defaults=$this_file_directory/../shell_defaults
9+
10+
# if a file exists there, source it. otherwise complain
11+
if test -f $relative_path_to_defaults; then
12+
# source and '.' are the same program
13+
source $relative_path_to_defaults
14+
else
15+
echo -e "\033[31m\nFAILED TO SOURCE TEST RUNNING OPTIONS.\033[39m"
16+
echo -e "\033[31mTried $relative_path_to_defaults\033[39m"
17+
exit 1
18+
fi
19+
20+
set +o xtrace # +x easier human reading here
421

522
. tools/lib/lib.sh
623

24+
725
function docker_tag_exists() {
8-
# Added check for images pushed to github container registry
9-
if [[ $1 == ghcr* ]]
26+
# Is true for images stored in the Github Container Registry
27+
repo=$1
28+
tag=$2
29+
# we user [[ here because test doesn't support globbing well
30+
if [[ $repo == ghcr* ]]
1031
then
1132
TOKEN_URL=https://ghcr.io/token\?scope\="repository:$1:pull"
12-
token=$(curl $TOKEN_URL | jq -r '.token')
33+
token=$(curl $TOKEN_URL | jq -r '.token' > /dev/null)
1334
URL=https://ghcr.io/v2/$1/manifests/$2
14-
printf "\tURL: %s\n" "$URL"
15-
curl --silent -H "Authorization: Bearer $token" -f -lSL "$URL" > /dev/null
35+
echo -e "$blue_text""\n\n\tURL: $URL""$default_text"
36+
curl -H "Authorization: Bearer $token" --location --silent --show-error --dump-header header.txt "$URL" > /dev/null
37+
curl_success=$?
1638
else
1739
URL=https://hub.docker.com/v2/repositories/"$1"/tags/"$2"
18-
printf "\tURL: %s\n" "$URL"
19-
curl --silent -f -lSL "$URL" > /dev/null
40+
echo -e "$blue_text""\n\n\tURL: $URL""$default_text"
41+
curl --silent --show-error --location --dump-header header.txt "$URL" > /dev/null
42+
curl_success=$?
43+
# some bullshit to get the number out of a header that looks like this
44+
# < content-length: 1039
45+
# < x-ratelimit-limit: 180
46+
# < x-ratelimit-reset: 1665683196
47+
# < x-ratelimit-remaining: 180
48+
docker_rate_limit_remaining=$(grep 'x-ratelimit-remaining: ' header.txt | grep --only-matching --extended-regexp "\d+")
49+
# too noisy when set to < 1. Dockerhub starts complaining somewhere around 10
50+
if test "$docker_rate_limit_remaining" -lt 20; then
51+
echo -e "$red_text""We are close to a sensitive dockerhub rate limit!""$default_text"
52+
echo -e "$red_text""SLEEPING 60s sad times""$default_text"
53+
sleep 60
54+
docker_tag_exists $1 $2
55+
elif test $docker_rate_limit_remaining -lt 50; then
56+
echo -e "$red_text""Rate limit reported as $docker_rate_limit_remaining""$default_text"
57+
fi
58+
fi
59+
if test $curl_success -ne 0; then
60+
echo -e "$red_text""Curl Said this didn't work. Please investigate""$default_text"
61+
exit 1
2062
fi
2163
}
2264

2365
checkPlatformImages() {
24-
echo "Checking platform images exist..."
25-
docker-compose pull || exit 1
26-
echo "Success! All platform images exist!"
66+
echo -e "$blue_text""Checking platform images exist...""$default_text"
67+
#Pull without printing progress information and send error stream because that where image names are
68+
docker-compose pull --quiet 2> compose_output
69+
docker_compose_success=$?
70+
# quiet output is just SHAs ie: f8a3d002a8a6
71+
images_pulled_count=$(docker images --quiet | wc -l)
72+
if test $images_pulled_count -eq 0; then
73+
echo -e "$red_text""Nothing was pulled! This script may be broken! We expect to pull something""$default_text"
74+
exit 1
75+
elif test $docker_compose_success -eq 0; then
76+
echo -e "$blue_text""Docker successfully pulled all images""$default_text"
77+
else
78+
echo -e "$red_text""docker-compose failed to pull all images""$default_text"
79+
cat compose_output
80+
exit 1
81+
fi
2782
}
2883

2984
checkNormalizationImages() {
85+
echo -e "$blue_text""Checking Normalization images exist...""$default_text"
3086
# the only way to know what version of normalization the platform is using is looking in NormalizationRunnerFactory.
3187
local image_version;
3288
factory_path=airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java
33-
# This check should fail if the file doesn't exist, so just try to ls the file
34-
ls $factory_path > /dev/null
89+
# -f True if file exists and is a regular file
90+
if ! test -f $factory_path; then
91+
echo -e "$red_text""No NormalizationRunnerFactory found at path! H4LP!!!""$default_text"
92+
fi
3593
image_version=$(cat $factory_path | grep 'NORMALIZATION_VERSION =' | cut -d"=" -f2 | sed 's:;::' | sed -e 's:"::g' | sed -e 's:[[:space:]]::g')
36-
echo "Checking normalization images with version $image_version exist..."
37-
VERSION=$image_version docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.yaml pull || exit 1
38-
echo "Success! All normalization images exist!"
94+
echo -e "$blue_text""Checking normalization images with version $image_version exist...""$default_text"
95+
VERSION=$image_version docker-compose --file airbyte-integrations/bases/base-normalization/docker-compose.yaml pull --quiet
96+
docker_compose_success=$?
97+
if test $docker_compose_success -eq 0; then
98+
echo -e "$blue_text""Docker successfully pulled all images for normalization""$default_text"
99+
else
100+
echo -e "$red_text""docker-compose failed to pull all images for normalization""$default_text"
101+
exit 1
102+
fi
39103
}
40104

41105
checkConnectorImages() {
42-
echo "Checking connector images exist..."
43-
106+
echo -e "$blue_text""Checking connector images exist...""$default_text"
44107
CONNECTOR_DEFINITIONS=$(grep "dockerRepository" -h -A1 airbyte-config/init/src/main/resources/seed/*.yaml | grep -v -- "^--$" | tr -d ' ')
45108
[ -z "CONNECTOR_DEFINITIONS" ] && echo "ERROR: Could not find any connector definition." && exit 1
46109

47110
while IFS=":" read -r _ REPO; do
48111
IFS=":" read -r _ TAG
49-
printf "${REPO}: ${TAG}\n"
112+
printf "\t${REPO}: ${TAG}\n"
50113
if docker_tag_exists "$REPO" "$TAG"; then
51114
printf "\tSTATUS: found\n"
52115
else
53116
printf "\tERROR: not found!\n" && exit 1
54117
fi
55-
# Docker hub has a rate limit of 180 requests per minute, so slow down our rate of calling the API
56-
# https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting
57-
sleep 1
58118
done <<< "${CONNECTOR_DEFINITIONS}"
59-
60-
echo "Success! All connector images exist!"
119+
echo -e "$blue_text""Success! All connector images exist!""$default_text"
61120
}
62121

63122
main() {
64123
assert_root
65124

66125
SUBSET=${1:-all} # default to all.
67126
[[ ! "$SUBSET" =~ ^(all|platform|connectors)$ ]] && echo "Usage ./tools/bin/check_image_exists.sh [all|platform|connectors]" && exit 1
68-
69-
echo "checking images for: $SUBSET"
127+
echo -e "$blue_text""checking images for: $SUBSET""$default_text"
70128

71129
[[ "$SUBSET" =~ ^(all|platform)$ ]] && checkPlatformImages
72130
[[ "$SUBSET" =~ ^(all|platform|connectors)$ ]] && checkNormalizationImages
73131
[[ "$SUBSET" =~ ^(all|connectors)$ ]] && checkConnectorImages
74-
75-
echo "Image check complete."
132+
echo -e "$blue_text""Image check complete.""$default_text"
133+
test -f header.txt && rm header.txt
134+
test -f compose_output && rm compose_output
76135
}
77136

78137
main "$@"

0 commit comments

Comments
 (0)