Skip to content

Commit b64c334

Browse files
Ani1357Ani1357
authored
feat: Integration test domains (#1763)
1 parent 600f584 commit b64c334

File tree

2 files changed

+148
-53
lines changed

2 files changed

+148
-53
lines changed

‎.github/workflows/integration.yml

+66-53
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Deploy Otomi
1+
name: Deploy APL
22
on:
33
workflow_call:
44
inputs:
@@ -7,25 +7,17 @@ on:
77
type: string
88
default: "['1.31']"
99
install_profile:
10-
description: 'Otomi installation profile'
10+
description: 'APL installation profile'
1111
type: string
1212
default: minimal-with-team
13-
cluster_persistence:
14-
description: 'Should a cluster be destroyed on pipeline finish?'
15-
type: string
16-
default: destroy
1713
domain_zone:
1814
description: 'Select Domain Zone'
1915
type: string
2016
default: DNS-Integration
2117
kms:
22-
description: 'Should Otomi encrypt secrets in values repo (DNS or KMS is turned on)?'
18+
description: 'Should APL encrypt secrets in values repo (DNS or KMS is turned on)?'
2319
type: string
2420
default: age
25-
generate_password:
26-
description: 'Should a unique password be generated?'
27-
type: string
28-
default: 'yes'
2921
certificate:
3022
description: 'Select certificate issuer'
3123
type: string
@@ -41,7 +33,7 @@ on:
4133
- "['1.31']"
4234
default: "['1.31']"
4335
install_profile:
44-
description: Otomi installation profile
36+
description: APL installation profile
4537
default: minimal-with-team
4638
type: choice
4739
options:
@@ -50,14 +42,6 @@ on:
5042
- monitoring-with-team
5143
- full
5244
- upgrade
53-
- no-otomi
54-
cluster_persistence:
55-
type: choice
56-
description: Should a cluster be destroyed on pipeline finish?
57-
options:
58-
- preserve
59-
- destroy
60-
default: preserve
6145
domain_zone:
6246
type: choice
6347
description: 'Select Domain Zone'
@@ -66,18 +50,11 @@ on:
6650
- Zone-2
6751
kms:
6852
type: choice
69-
description: Should Otomi encrypt secrets in values repo (DNS or KMS is turned on)?
53+
description: Should APL encrypt secrets in values repo (DNS or KMS is turned on)?
7054
options:
7155
- age
7256
- no_kms
7357
default: age
74-
generate_password:
75-
type: choice
76-
description: Should a unique password be generated?
77-
options:
78-
- 'yes'
79-
- 'no'
80-
default: 'yes'
8158
certificate:
8259
type: choice
8360
description: Select certificate issuer
@@ -108,10 +85,8 @@ jobs:
10885
echo 'ref: ${{ github.event.pull_request.head.ref || github.ref }}'
10986
echo 'install_profile: ${{ inputs.install_profile }}'
11087
echo 'kubernetes_versions: ${{ inputs.kubernetes_versions }}'
111-
echo 'cluster_persistence: ${{ inputs.cluster_persistence }}'
11288
echo 'kms: ${{ inputs.kms }}'
11389
echo 'domain_zone: ${{ inputs.domain_zone }}'
114-
echo 'generate_password: ${{ inputs.generate_password }}'
11590
echo 'certificate: ${{ inputs.certificate }}'
11691
11792
preprocess-linode-input:
@@ -130,7 +105,6 @@ jobs:
130105
case "${{ inputs.domain_zone }}" in
131106
"Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;;
132107
"Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;;
133-
"DNS-Integration") LINODE_CLUSTER_NAME=nightly-apl-test ;;
134108
esac
135109
136110
if [[ $(linode-cli lke clusters-list --json | jq --arg name "$LINODE_CLUSTER_NAME" '[.[] | select(.label == $name)] | length > 0') == "true" ]]; then
@@ -169,19 +143,30 @@ jobs:
169143
case "${{ inputs.domain_zone }}" in
170144
"Zone-1") LINODE_CLUSTER_NAME=${{ github.actor }}-1 ;;
171145
"Zone-2") LINODE_CLUSTER_NAME=${{ github.actor }}-2 ;;
172-
"DNS-Integration") LINODE_CLUSTER_NAME=nightly-apl-test ;;
146+
"DNS-Integration") LINODE_CLUSTER_NAME=nightly-apl-test-$RANDOM ;;
173147
esac
174148
echo LINODE_CLUSTER_NAME=$LINODE_CLUSTER_NAME >> $GITHUB_ENV
175149
- name: Determine exact k8s version
176150
run: |
177151
echo LINODE_K8S_VERSION=$(linode-cli lke versions-list --json | jq -ce --arg version "$(echo ${{ matrix.kubernetes_versions }} | sed -E 's/^([0-9]+\.[0-9])$/\10/')" '.[] | select(.id | tostring | startswith($version)) | .id') >> $GITHUB_ENV
152+
- name: Creating domain for scheduled integration test
153+
env:
154+
EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
155+
if: ${{ inputs.domain_zone == 'DNS-Integration' }}
156+
run: |
157+
# Generating a random 5 char string
158+
RAND=$(openssl rand -hex 4)
159+
DOMAIN="integration-${RAND}.${EDGEDNS_ZONE}"
160+
echo "::add-mask::$DOMAIN"
161+
echo DOMAIN=$DOMAIN >> $GITHUB_ENV
162+
178163
- name: Determine domain name to use
164+
if: ${{ inputs.domain_zone != 'DNS-Integration' }}
179165
run: |
180166
# Mapping of domain_zone to domain names
181167
case "${{ inputs.domain_zone }}" in
182168
"Zone-1") DOMAIN=$(jq '."${{ github.actor }}"[0]' <<< ${{ env.DEV_DOMAINS }}) ;;
183169
"Zone-2") DOMAIN=$(jq '."${{ github.actor }}"[1]' <<< ${{ env.DEV_DOMAINS }}) ;;
184-
"DNS-Integration") DOMAIN=$(jq '."DNS-Integration"[0]' <<< ${{ env.DEV_DOMAINS }}) ;;
185170
esac
186171
187172
echo "::add-mask::$DOMAIN"
@@ -198,6 +183,7 @@ jobs:
198183
--node_pools.autoscaler.max 3 \
199184
--node_pools.autoscaler.min 3 \
200185
--tags testing \
186+
--tags delete_me_tonight \
201187
--no-defaults
202188
- name: Retrieve cluster id
203189
run: echo "LINODE_CLUSTER_ID=$(linode-cli lke clusters-list --json | jq -ce '.[] | select(.label | startswith("${{ env.LINODE_CLUSTER_NAME }}")) | .id')" >> $GITHUB_ENV
@@ -244,8 +230,7 @@ jobs:
244230
--docker-password='${{ secrets.BOT_PULL_TOKEN }}'
245231
- name: Checkout
246232
uses: actions/checkout@v4
247-
- name: Prepare Otomi chart
248-
if: ${{ inputs.install_profile != 'no-otomi' }}
233+
- name: Prepare APL chart
249234
run: |
250235
ref=${{ github.event.pull_request.head.ref || github.ref }}
251236
tag=${ref##*/}
@@ -254,7 +239,7 @@ jobs:
254239
sed --in-place "s/OTOMI_VERSION_PLACEHOLDER/${GITHUB_REF##*/}/g" tests/integration/${{ inputs.install_profile }}.yaml
255240
touch values-container-registry.yaml
256241
257-
# If a pipeline installs Otomi from the semver tag then pull container image from DockerHub
242+
# If a pipeline installs APL from the semver tag then pull container image from DockerHub
258243
[[ ${GITHUB_REF##*/} =~ ^v[0-9].+$ ]] && exit 0
259244
260245
# Pull image from cache registry
@@ -263,40 +248,59 @@ jobs:
263248
imagePullSecretNames:
264249
- reg-otomi-github
265250
EOF
266-
- name: Otomi install
267-
if: ${{ inputs.install_profile != 'no-otomi' }}
251+
- name: APL install
268252
env:
269253
LETSENCRYPT_STAGING: ${{ secrets.LETSENCRYPT_STAGING }}
270254
LETSENCRYPT_PRODUCTION: ${{ secrets.LETSENCRYPT_PRODUCTION }}
271-
HOST: ${{ secrets.EDGEDNS_HOST }}
272-
ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }}
273-
CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }}
274-
CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }}
255+
EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }}
256+
EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }}
257+
EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }}
258+
EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
259+
EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }}
275260
run: |
276261
touch values.yaml
277-
adminPassword=welcomeotomi
262+
263+
adminPassword="$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)"
278264
[[ '${{ inputs.certificate }}' == 'letsencrypt_staging' ]] && echo "$LETSENCRYPT_STAGING" >> values.yaml
279265
[[ '${{ inputs.certificate }}' == 'letsencrypt_production' ]] && echo "$LETSENCRYPT_PRODUCTION" >> values.yaml
280266
[[ '${{ inputs.kms }}' == 'age' ]] && kms="--set kms.sops.provider=age"
281-
[[ '${{ inputs.generate_password }}' == 'yes' ]] && adminPassword="$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)"
282-
283267
284268
install_args="otomi chart/apl --wait --wait-for-jobs --timeout 90m0s \
285269
--values tests/integration/${{ inputs.install_profile }}.yaml \
286270
--values values-container-registry.yaml \
287271
--values values.yaml \
288272
--set cluster.provider=linode \
289273
--set dns.domainFilters[0]=${{ env.DOMAIN }} \
290-
--set dns.provider.akamai.clientSecret=${CLIENT_SECRET} \
291-
--set dns.provider.akamai.host=${HOST} \
292-
--set dns.provider.akamai.accessToken=${ACCESS_TOKEN} \
293-
--set dns.provider.akamai.clientToken=${CLIENT_TOKEN} \
274+
--set dns.provider.akamai.clientSecret=${EDGEDNS_CLIENT_SECRET} \
275+
--set dns.provider.akamai.host=${EDGEDNS_HOST} \
276+
--set dns.provider.akamai.accessToken=${EDGEDNS_ACCESS_TOKEN} \
277+
--set dns.provider.akamai.clientToken=${EDGEDNS_CLIENT_TOKEN} \
294278
--set otomi.hasExternalDNS=true \
295279
--set cluster.domainSuffix=${{ env.DOMAIN }} \
296280
--set otomi.adminPassword=$adminPassword \
297281
$kms"
298282
299-
helm install $install_args
283+
helm install $install_args &
284+
HELM_PID=$!
285+
sleep 120
286+
287+
# While helm is installing we can crete the wildcard dns record
288+
while true; do
289+
PUB_IP=$(kubectl get svc ingress-nginx-platform-controller -n ingress -ojson | jq '.status.loadBalancer.ingress[0].ip' -r)
290+
if [[ -n "$PUB_IP" ]]; then
291+
echo "::add-mask::$PUB_IP"
292+
echo PUB_IP=$PUB_IP >> $GITHUB_ENV
293+
break
294+
else
295+
echo "Waiting for ingress-nginx-platform-controller IP..."
296+
sleep 5
297+
fi
298+
done
299+
300+
pip3 install edgegrid-python requests
301+
python3 bin/edgedns_A_record.py create $DOMAIN $PUB_IP
302+
303+
wait $HELM_PID
300304
301305
- name: Gather k8s events on failure
302306
if: failure()
@@ -306,7 +310,7 @@ jobs:
306310
if: failure()
307311
run: |
308312
kubectl get pods -A -o wide
309-
- name: Gather otomi logs on failure
313+
- name: Gather APL logs on failure
310314
if: failure()
311315
run: |
312316
kubectl logs jobs/otomi --tail 150
@@ -315,10 +319,19 @@ jobs:
315319
run: |
316320
kubectl logs -n maintenance -l app.kubernetes.io/instance=job-e2e --tail 15000
317321
- name: Remove the test cluster
318-
if: always()
322+
if: ${{ inputs.domain_zone == 'DNS-Integration' }}
319323
run: |
320-
[[ "${{ inputs.cluster_persistence }}" == "preserve" ]] && echo "The cluster ${{ env.LINODE_CLUSTER_NAME }} will NOT be destroyed!!" && exit 0
321324
linode-cli lke cluster-delete ${{ env.LINODE_CLUSTER_ID }}
325+
- name: Delete Domain
326+
if: ${{ inputs.domain_zone == 'DNS-Integration' }}
327+
env:
328+
EDGEDNS_ACCESS_TOKEN: ${{ secrets.EDGEDNS_ACCESS_TOKEN }}
329+
EDGEDNS_CLIENT_TOKEN: ${{ secrets.EDGEDNS_CLIENT_TOKEN }}
330+
EDGEDNS_CLIENT_SECRET: ${{ secrets.EDGEDNS_CLIENT_SECRET }}
331+
EDGEDNS_ZONE: ${{ secrets.EDGEDNS_ZONE }}
332+
EDGEDNS_HOST: ${{ secrets.EDGEDNS_HOST }}
333+
run: |
334+
python3 bin/edgedns_A_record.py delete $DOMAIN
322335
- name: Slack Notification
323336
if: always()
324337
uses: rtCamp/action-slack-notify@v2

‎bin/edgedns_A_record.py

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import os
2+
import sys
3+
from akamai.edgegrid import EdgeGridAuth
4+
from requests import Session, HTTPError
5+
6+
EDGEDNS_ZONE = os.environ.get('EDGEDNS_ZONE')
7+
EDGEDNS_HOST = os.environ.get('EDGEDNS_HOST')
8+
9+
# Validate required environment variables
10+
if not EDGEDNS_ZONE or not EDGEDNS_HOST:
11+
raise ValueError("EDGEDNS_ZONE and EDGEDNS_HOST environment variables must be set.")
12+
13+
# Create a session with EdgeGrid
14+
def create_session():
15+
session = Session()
16+
session.auth = EdgeGridAuth(
17+
client_token=os.environ.get('EDGEDNS_CLIENT_TOKEN'),
18+
client_secret=os.environ.get('EDGEDNS_CLIENT_SECRET'),
19+
access_token=os.environ.get('EDGEDNS_ACCESS_TOKEN')
20+
)
21+
session.headers.update({
22+
'Accept': 'application/json',
23+
'Content-Type': 'application/json'
24+
})
25+
return session
26+
27+
# Function to construct the DNS API URL
28+
def construct_dns_url(domain, record_type="A"):
29+
return f"https://{EDGEDNS_HOST}/config-dns/v2/zones/{EDGEDNS_ZONE}/names/{domain}/types/{record_type}"
30+
31+
# Function to create DNS record
32+
def create_dns_record(session, domain, ip):
33+
url = construct_dns_url(domain)
34+
data = {
35+
"name": domain,
36+
"rdata": [ip],
37+
"ttl": 300,
38+
"type": "A"
39+
}
40+
41+
try:
42+
response = session.post(url, json=data)
43+
response.raise_for_status()
44+
print(f"DNS record created successfully !")
45+
except HTTPError as e:
46+
print(f"Failed to create DNS record!")
47+
48+
# Function to delete DNS record
49+
def delete_dns_record(session, domain):
50+
url = construct_dns_url(domain)
51+
52+
try:
53+
response = session.delete(url)
54+
response.raise_for_status()
55+
print(f"DNS record deleted successfully!")
56+
except HTTPError as e:
57+
print(f"Failed to delete DNS record!")
58+
59+
def main():
60+
if len(sys.argv) < 3 or (sys.argv[1].lower() == "create" and len(sys.argv) != 4):
61+
print("Usage: python edgedns_A_record.py <action> <domain> [<ip>]")
62+
sys.exit(1)
63+
64+
action = sys.argv[1].lower()
65+
domain = sys.argv[2].lower()
66+
ip = sys.argv[3] if action == "create" else None
67+
68+
session = create_session()
69+
70+
if action == "create":
71+
if not ip:
72+
print("IP address must be specified for create action.")
73+
sys.exit(1)
74+
create_dns_record(session, domain, ip)
75+
elif action == "delete":
76+
delete_dns_record(session, domain)
77+
else:
78+
print("Invalid action. Use 'create' or 'delete'.")
79+
sys.exit(1)
80+
81+
if __name__ == "__main__":
82+
main()

0 commit comments

Comments
 (0)