Skip to content

Commit b32b8d3

Browse files
Merge pull request #6708 from bbguimaraes/steps
step-registry: add configure and install IPI steps
2 parents dec4e93 + 7f3ca24 commit b32b8d3

File tree

10 files changed

+368
-24
lines changed

10 files changed

+368
-24
lines changed

ci-operator/config/openshift/installer/openshift-installer-master.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,3 +197,7 @@ tests:
197197
commands: TEST_SUITE=openshift/conformance/parallel run-tests
198198
openshift_installer_upi:
199199
cluster_profile: vsphere
200+
- as: e2e-steps
201+
steps:
202+
cluster_profile: aws
203+
workflow: ipi

ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1518,6 +1518,65 @@ presubmits:
15181518
name: prow-job-cluster-launch-installer-ovirt-e2e
15191519
name: job-definition
15201520
trigger: (?m)^/test( | .* )e2e-ovirt,?($|\s.*)
1521+
- agent: kubernetes
1522+
always_run: true
1523+
branches:
1524+
- master
1525+
context: ci/prow/e2e-steps
1526+
decorate: true
1527+
decoration_config:
1528+
skip_cloning: true
1529+
labels:
1530+
ci-operator.openshift.io/prowgen-controlled: "true"
1531+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
1532+
name: pull-ci-openshift-installer-master-e2e-steps
1533+
optional: true
1534+
rerun_command: /test e2e-steps
1535+
skip_report: true
1536+
spec:
1537+
containers:
1538+
- args:
1539+
- --artifact-dir=$(ARTIFACTS)
1540+
- --branch=master
1541+
- --give-pr-author-access-to-namespace=true
1542+
- --lease-server=http://boskos
1543+
- --org=openshift
1544+
- --repo=installer
1545+
- --resolver-address=http://ci-operator-configresolver-ci.svc.ci.openshift.org
1546+
- --secret-dir=/usr/local/e2e-steps-cluster-profile
1547+
- --sentry-dsn-path=/etc/sentry-dsn/ci-operator
1548+
- --target=e2e-steps
1549+
command:
1550+
- ci-operator
1551+
env:
1552+
- name: CONFIG_SPEC
1553+
valueFrom:
1554+
configMapKeyRef:
1555+
key: openshift-installer-master.yaml
1556+
name: ci-operator-master-configs
1557+
image: ci-operator:latest
1558+
imagePullPolicy: Always
1559+
name: ""
1560+
resources:
1561+
requests:
1562+
cpu: 10m
1563+
volumeMounts:
1564+
- mountPath: /usr/local/e2e-steps-cluster-profile
1565+
name: cluster-profile
1566+
- mountPath: /etc/sentry-dsn
1567+
name: sentry-dsn
1568+
readOnly: true
1569+
serviceAccountName: ci-operator
1570+
volumes:
1571+
- name: cluster-profile
1572+
projected:
1573+
sources:
1574+
- secret:
1575+
name: cluster-secrets-aws
1576+
- name: sentry-dsn
1577+
secret:
1578+
secretName: sentry-dsn
1579+
trigger: (?m)^/test( | .* )e2e-steps,?($|\s.*)
15211580
- agent: kubernetes
15221581
always_run: false
15231582
branches:
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
approvers:
2+
- smarterclayton
3+
- wking
4+
- stevekuznetsov
5+
- vrutkovs
6+
- abhinavdahiya
7+
- deads2k
8+
- crawford
9+
- ewolinetz
10+
- csrwng
11+
- staebler
Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
#!/bin/bash
2+
3+
set -o nounset
4+
set -o errexit
5+
set -o pipefail
6+
7+
trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
8+
9+
cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile
10+
cluster_name=${NAMESPACE}-${JOB_NAME_HASH}
11+
12+
out=/tmp/secret/install-config.yaml
13+
mkdir "$(dirname "${out}")"
14+
15+
cluster_variant=
16+
if [[ -e "${SHARED_DIR}/install-config-variant.txt" ]]; then
17+
cluster_variant=$(<"${SHARED_DIR}/install-config-variant.txt")
18+
fi
19+
20+
function has_variant() {
21+
regex="(^|,)$1($|,)"
22+
if [[ $cluster_variant =~ $regex ]]; then
23+
return 0
24+
fi
25+
return 1
26+
}
27+
28+
base_domain=
29+
if [[ -e "${SHARED_DIR}/install-config-base-domain.txt" ]]; then
30+
base_domain=$(<"${SHARED_DIR}/install-config-base-domain.txt")
31+
else
32+
case "${CLUSTER_TYPE}" in
33+
aws) base_domain=origin-ci-int-aws.dev.rhcloud.com;;
34+
azure) base_domain=ci.azure.devcluster.openshift.com;;
35+
gcp) base_domain=origin-ci-int-gce.dev.openshift.com;;
36+
*) echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'"
37+
esac
38+
fi
39+
40+
echo "Installing from release ${RELEASE_IMAGE_LATEST}"
41+
42+
expiration_date=$(date -d '4 hours' --iso=minutes --utc)
43+
ssh_pub_key=$(<"${cluster_profile}/ssh-publickey")
44+
pull_secret=$(<"${cluster_profile}/pull-secret")
45+
46+
workers=3
47+
if has_variant compact; then
48+
workers=0
49+
fi
50+
51+
case "${CLUSTER_TYPE}" in
52+
aws)
53+
case "$((RANDOM % 4))" in
54+
0) aws_region=us-east-1
55+
zone_1=us-east-1b
56+
zone_2=us-east-1c;;
57+
1) aws_region=us-east-2;;
58+
2) aws_region=us-west-1;;
59+
3) aws_region=us-west-2;;
60+
*) echo >&2 "invalid AWS region index"; exit 1;;
61+
esac
62+
echo "AWS region: ${aws_region} (zones: ${zone_1:-${aws_region}a} ${zone_2:-${aws_region}b})"
63+
master_type=null
64+
if has_variant xlarge; then
65+
master_type=m5.8xlarge
66+
elif has_variant large; then
67+
master_type=m5.4xlarge
68+
fi
69+
subnets="[]"
70+
if has_variant "shared-vpc"; then
71+
case "${aws_region}_$((RANDOM % 4))" in
72+
us-east-1_0) subnets="['subnet-030a88e6e97101ab2','subnet-0e07763243186cac5','subnet-02c5fea7482f804fb','subnet-0291499fd1718ee01','subnet-01c4667ad446c8337','subnet-025e9043c44114baa']";;
73+
us-east-1_1) subnets="['subnet-0170ee5ccdd7e7823','subnet-0d50cac95bebb5a6e','subnet-0094864467fc2e737','subnet-0daa3919d85296eb6','subnet-0ab1e11d3ed63cc97','subnet-07681ad7ce2b6c281']";;
74+
us-east-1_2) subnets="['subnet-00de9462cf29cd3d3','subnet-06595d2851257b4df','subnet-04bbfdd9ca1b67e74','subnet-096992ef7d807f6b4','subnet-0b3d7ba41fc6278b2','subnet-0b99293450e2edb13']";;
75+
us-east-1_3) subnets="['subnet-047f6294332aa3c1c','subnet-0c3bce80bbc2c8f1c','subnet-038c38c7d96364d7f','subnet-027a025e9d9db95ce','subnet-04d9008469025b101','subnet-02f75024b00b20a75']";;
76+
us-east-2_0) subnets="['subnet-0faf6d16c378ee7a7','subnet-0e104572db1b7d092','subnet-014ca96c04f36adec','subnet-0ea06057dceadfe8e','subnet-0689efe5e1f9f4212','subnet-0d36bb8edbcb3d916']";;
77+
us-east-2_1) subnets="['subnet-085787cc4b80b84b2','subnet-09dfbf66e8f6e5b50','subnet-0db5d90ff3087444e','subnet-047f15f2a0210fbe0','subnet-0bf13f041c4233849','subnet-0e2a5320549e289d8']";;
78+
us-east-2_2) subnets="['subnet-07d59b122f7a76f67','subnet-0d1a413c66cd59a3b','subnet-020df1de666b06b20','subnet-0ce9183380508d88d','subnet-04c83a79a1913824c','subnet-0d97ed1a54b1e9235']";;
79+
us-east-2_3) subnets="['subnet-0d689957169836114','subnet-081c5c0c7bc351205','subnet-023b79f57b84894e5','subnet-070c0b96148b58787','subnet-0c693d11c33437345','subnet-0249c4ec2d6509b4e']";;
80+
us-west-1_0) subnets="['subnet-0b0a3190ff0b05fb0','subnet-038719a99ae7f208c','subnet-0afc43ade6ca7f8e0','subnet-0df272b93eb3d79a5']";;
81+
us-west-1_1) subnets="['subnet-070d5f1a70aa7b2ad','subnet-0e371618c77a58409','subnet-046cbad6141e391ba','subnet-0528b85478ef9d2b5']";;
82+
us-west-1_2) subnets="['subnet-0a51561b99949d3c4','subnet-0de96f5675188f16f','subnet-05d1cbeccfb032e31','subnet-01e489eab26e95ec9']";;
83+
us-west-1_3) subnets="['subnet-0029d43cd2d22bfe4','subnet-0b5476fddae459d10','subnet-0955a46cb4b379c91','subnet-04e3dae5b3fdcbe61']";;
84+
us-west-2_0) subnets="['subnet-0a1956a6a6babc86b','subnet-07252d4a4737ec97e','subnet-00bcec6286b15a024','subnet-0f979e13d715cc03a','subnet-02e3b436e780363c5','subnet-02f0597dc582d3bde']";;
85+
us-west-2_1) subnets="['subnet-0e2979f62a537ab59','subnet-060b22e9f90846c58','subnet-0c61f833b2a4caa2a','subnet-022d5d9affc6a2241','subnet-02c903aa40cf463ef','subnet-0db7df4231255086d']";;
86+
us-west-2_2) subnets="['subnet-0d9b5481442b7d212','subnet-07795ec1097c5e34c','subnet-000d265d2bf4729f3','subnet-0d419e59ee340211c','subnet-0c8027d8d9794d822','subnet-05a19cfee3f602c7e']";;
87+
us-west-2_3) subnets="['subnet-08c871a474ab034cc','subnet-0fe9e5f0d33e16eb0','subnet-0731dfd7678a5bac8','subnet-0d476b24170ac5942','subnet-0f0da17f8581745e6','subnet-0842d7a0250595e13']";;
88+
*) echo >&2 "invalid subnets index"; exit 1;;
89+
esac
90+
echo "Subnets : ${subnets}"
91+
fi
92+
cat > "${out}" << EOF
93+
apiVersion: v1
94+
baseDomain: ${base_domain}
95+
metadata:
96+
name: ${cluster_name}
97+
controlPlane:
98+
name: master
99+
replicas: 3
100+
platform:
101+
aws:
102+
type: ${master_type}
103+
zones:
104+
- ${zone_1:-${aws_region}a}
105+
- ${zone_2:-${aws_region}b}
106+
compute:
107+
- name: worker
108+
replicas: ${workers}
109+
platform:
110+
aws:
111+
type: m4.xlarge
112+
zones:
113+
- ${zone_1:-${aws_region}a}
114+
- ${zone_2:-${aws_region}b}
115+
platform:
116+
aws:
117+
region: ${aws_region}
118+
userTags:
119+
expirationDate: ${expiration_date}
120+
subnets: ${subnets}
121+
pullSecret: >
122+
${pull_secret}
123+
sshKey: |
124+
${ssh_pub_key}
125+
EOF
126+
;;
127+
azure4)
128+
case $((RANDOM % 8)) in
129+
0) azure_region=centralus;;
130+
1) azure_region=centralus;;
131+
2) azure_region=centralus;;
132+
3) azure_region=centralus;;
133+
4) azure_region=centralus;;
134+
5) azure_region=eastus;;
135+
6) azure_region=eastus2;;
136+
7) azure_region=westus;;
137+
esac
138+
echo "Azure region: ${azure_region}"
139+
vnetrg=""
140+
vnetname=""
141+
ctrlsubnet=""
142+
computesubnet=""
143+
if has_variant shared-vpc; then
144+
vnetrg="os4-common"
145+
vnetname="do-not-delete-shared-vnet-${azure_region}"
146+
ctrlsubnet="subnet-1"
147+
computesubnet="subnet-2"
148+
fi
149+
cat > "${out}" << EOF
150+
apiVersion: v1
151+
baseDomain: ${base_domain}
152+
metadata:
153+
name: ${cluster_name}
154+
controlPlane:
155+
name: master
156+
replicas: 3
157+
compute:
158+
- name: worker
159+
replicas: ${workers}
160+
platform:
161+
azure:
162+
baseDomainResourceGroupName: os4-common
163+
region: ${azure_region}
164+
networkResourceGroupName: ${vnetrg}
165+
virtualNetwork: ${vnetname}
166+
controlPlaneSubnet: ${ctrlsubnet}
167+
computeSubnet: ${computesubnet}
168+
pullSecret: >
169+
${pull_secret}
170+
sshKey: |
171+
${ssh_pub_key}
172+
EOF
173+
;;
174+
gcp)
175+
gcp_region=us-east1
176+
gcp_project=openshift-gce-devel-ci
177+
# HACK: try to "poke" the token endpoint before the test starts
178+
for i in $(seq 1 30); do
179+
code="$( curl -s -o /dev/null -w "%{http_code}" https://oauth2.googleapis.com/token -X POST -d '' || echo "Failed to POST https://oauth2.googleapis.com/token with $?" 1>&2)"
180+
if [[ "${code}" == "400" ]]; then
181+
break
182+
fi
183+
echo "error: Unable to resolve https://oauth2.googleapis.com/token: $code" 1>&2
184+
if [[ "${i}" == "30" ]]; then
185+
echo "error: Unable to resolve https://oauth2.googleapis.com/token within timeout, exiting" 1>&2
186+
exit 1
187+
fi
188+
sleep 1
189+
done
190+
network=""
191+
ctrlsubnet=""
192+
computesubnet=""
193+
if has_variant shared-vpc; then
194+
network="do-not-delete-shared-network"
195+
ctrlsubnet="do-not-delete-shared-master-subnet"
196+
computesubnet="do-not-delete-shared-worker-subnet"
197+
fi
198+
cat > "${out}" << EOF
199+
apiVersion: v1
200+
baseDomain: ${base_domain}
201+
metadata:
202+
name: ${cluster_name}
203+
controlPlane:
204+
name: master
205+
replicas: 3
206+
compute:
207+
- name: worker
208+
replicas: ${workers}
209+
platform:
210+
gcp:
211+
projectID: ${gcp_project}
212+
region: ${gcp_region}
213+
network: ${network}
214+
controlPlaneSubnet: ${ctrlsubnet}
215+
computeSubnet: ${computesubnet}
216+
pullSecret: >
217+
${pull_secret}
218+
sshKey: |
219+
${ssh_pub_key}
220+
EOF
221+
;;
222+
*)
223+
echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'"
224+
exit 1;;
225+
esac
226+
227+
# TODO proxy variant
228+
# TODO CLUSTER_NETWORK_TYPE / ovn variant
229+
# TODO mirror variant
230+
# TODO fips variant
231+
# TODO CLUSTER_NETWORK_MANIFEST
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
ref:
2+
as: ipi-conf
3+
from: centos:7
4+
commands: ipi-conf-commands.sh
5+
resources:
6+
requests:
7+
cpu: 1000m
8+
memory: 100Mi
9+
documentation: |-
10+
The IPI configure step generates the install-config.yaml file based on the cluster profile and optional input files.

ci-operator/step-registry/ipi/deprovision/artifacts/artifacts/ipi-deprovision-artifacts-artifacts-commands.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ echo "Gathering artifacts ..."
2222
mkdir -p ${ARTIFACT_DIR}/pods ${ARTIFACT_DIR}/nodes ${ARTIFACT_DIR}/metrics ${ARTIFACT_DIR}/bootstrap ${ARTIFACT_DIR}/network
2323

2424
oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}' > /tmp/nodes
25+
oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' > /tmp/node-provider-IDs
26+
oc --insecure-skip-tls-verify --request-timeout=5s -n openshift-machine-api get machines -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' >> /tmp/node-provider-IDs
2527
oc --insecure-skip-tls-verify --request-timeout=5s get pods --all-namespaces --template '{{ range .items }}{{ $name := .metadata.name }}{{ $ns := .metadata.namespace }}{{ range .spec.containers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ range .spec.initContainers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ end }}' > /tmp/containers
2628
oc --insecure-skip-tls-verify --request-timeout=5s get pods -l openshift.io/component=api --all-namespaces --template '{{ range .items }}-n {{ .metadata.namespace }} {{ .metadata.name }}{{ "\n" }}{{ end }}' > /tmp/pods-api
2729

@@ -85,3 +87,4 @@ done < /tmp/containers
8587

8688
echo "Snapshotting prometheus (may take 15s) ..."
8789
queue ${ARTIFACT_DIR}/metrics/prometheus.tar.gz oc --insecure-skip-tls-verify exec -n openshift-monitoring prometheus-k8s-0 -- tar cvzf - -C /prometheus .
90+
FILTER=gzip queue ${ARTIFACT_DIR}/metrics/prometheus-target-metadata.json.gz oc --insecure-skip-tls-verify exec -n openshift-monitoring prometheus-k8s-0 -- /bin/bash -c "curl -G http://localhost:9090/api/v1/targets/metadata --data-urlencode 'match_target={instance!=\"\"}'"

ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ set -o nounset
44
set -o errexit
55
set -o pipefail
66

7+
cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile
8+
export AWS_SHARED_CREDENTIALS_FILE=$cluster_profile/.awscred
9+
export AZURE_AUTH_LOCATION=$cluster_profile/osServicePrincipal.json
10+
711
echo "Deprovisioning cluster ..."
812
cp -ar "${SHARED_DIR}" /tmp/installer
913
openshift-install --dir /tmp/installer destroy cluster

0 commit comments

Comments
 (0)