Skip to content

Commit f3917ea

Browse files
authored
Merge pull request #5225 from nojnhuh/prow-mp-ci-version-patch
translate machine-pool-ci-version KubeadmConfig merge patch to JSON patch
2 parents ea98959 + 04bc24a commit f3917ea

File tree

3 files changed

+128
-116
lines changed

3 files changed

+128
-116
lines changed

templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ patches:
2424
name: .*-mp-win
2525
namespace: default
2626
version: v1beta1
27+
- path: patches/kubeadm-bootstrap-k8s-ci-binaries.yaml
28+
target:
29+
group: bootstrap.cluster.x-k8s.io
30+
kind: KubeadmConfig
31+
name: .*-mp-0
32+
namespace: default
33+
version: v1beta1
2734
- path: ../patches/control-plane-image-ci-version.yaml
2835
- path: ../patches/controller-manager.yaml
2936
- path: patches/machine-pool-ci-version.yaml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
- op: add
2+
path: /spec/files/0
3+
value:
4+
path: /tmp/kubeadm-bootstrap.sh
5+
owner: "root:root"
6+
permissions: "0744"
7+
content: |
8+
#!/bin/bash
9+
10+
set -o nounset
11+
set -o pipefail
12+
set -o errexit
13+
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
14+
15+
# This test installs release packages or binaries that are a result of the CI and release builds.
16+
# It runs '... --version' commands to verify that the binaries are correctly installed
17+
# and finally uninstalls the packages.
18+
# For the release packages it tests all versions in the support skew.
19+
LINE_SEPARATOR="*************************************************"
20+
echo "$$LINE_SEPARATOR"
21+
CI_VERSION=${CI_VERSION}
22+
if [[ "$${CI_VERSION}" != "" ]]; then
23+
CI_DIR=/tmp/k8s-ci
24+
mkdir -p $$CI_DIR
25+
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
26+
declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
27+
CONTAINER_EXT="tar"
28+
echo "* testing CI version $$CI_VERSION"
29+
# Check for semver
30+
if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
31+
VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
32+
DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
33+
curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
34+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
35+
apt-get update
36+
# replace . with \.
37+
VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
38+
PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
39+
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
40+
echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
41+
DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
42+
done
43+
else
44+
CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
45+
fi
46+
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
47+
echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
48+
wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
49+
chmod +x "$$CI_DIR/$$CI_PACKAGE"
50+
mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
51+
done
52+
IMAGE_REGISTRY_PREFIX=registry.k8s.io
53+
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
54+
echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
55+
wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
56+
$${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
57+
$${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
58+
$${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
59+
done
60+
fi
61+
systemctl restart kubelet
62+
fi
63+
echo "* checking binary versions"
64+
echo "ctr version: " $(ctr version)
65+
echo "kubeadm version: " $(kubeadm version -o=short)
66+
echo "kubectl version: " $(kubectl version --client=true)
67+
echo "kubelet version: " $(kubelet --version)
68+
echo "$$LINE_SEPARATOR"
69+
- op: add
70+
path: /spec/files/0
71+
value:
72+
path: /tmp/oot-cred-provider.sh
73+
owner: "root:root"
74+
permissions: "0744"
75+
content: |
76+
#!/bin/bash
77+
78+
set -o nounset
79+
set -o pipefail
80+
set -o errexit
81+
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
82+
83+
# Run the az login command with managed identity
84+
if az login --identity > /dev/null 2>&1; then
85+
echo "Logged in Azure with managed identity"
86+
echo "Use OOT credential provider"
87+
mkdir -p /var/lib/kubelet/credential-provider
88+
az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
89+
chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
90+
az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
91+
chmod 644 /var/lib/kubelet/credential-provider-config.yaml
92+
else
93+
echo "Using curl to download the OOT credential provider"
94+
mkdir -p /var/lib/kubelet/credential-provider
95+
curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
96+
chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
97+
curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
98+
chmod 644 /var/lib/kubelet/credential-provider-config.yaml
99+
fi
100+
- op: test
101+
path: /spec/preKubeadmCommands
102+
value: null
103+
- op: add
104+
path: /spec/preKubeadmCommands
105+
value: []
106+
- op: add
107+
path: /spec/preKubeadmCommands/-
108+
value:
109+
bash -c /tmp/oot-cred-provider.sh
110+
- op: add
111+
path: /spec/preKubeadmCommands/-
112+
value:
113+
bash -c /tmp/kubeadm-bootstrap.sh
114+
- op: add
115+
path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir
116+
value:
117+
/var/lib/kubelet/credential-provider
118+
- op: add
119+
path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config
120+
value:
121+
/var/lib/kubelet/credential-provider-config.yaml

templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml

-116
Original file line numberDiff line numberDiff line change
@@ -1,119 +1,3 @@
1-
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
2-
kind: KubeadmConfig
3-
metadata:
4-
name: ${CLUSTER_NAME}-mp-0
5-
spec:
6-
joinConfiguration:
7-
nodeRegistration:
8-
kubeletExtraArgs:
9-
cloud-provider: external
10-
image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider
11-
image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml
12-
preKubeadmCommands:
13-
- bash -c /tmp/oot-cred-provider.sh
14-
- bash -c /tmp/kubeadm-bootstrap.sh
15-
files:
16-
- path: /tmp/oot-cred-provider.sh
17-
owner: "root:root"
18-
permissions: "0744"
19-
content: |
20-
#!/bin/bash
21-
22-
set -o nounset
23-
set -o pipefail
24-
set -o errexit
25-
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
26-
27-
# Run the az login command with managed identity
28-
if az login --identity > /dev/null 2>&1; then
29-
echo "Logged in Azure with managed identity"
30-
echo "Use OOT credential provider"
31-
mkdir -p /var/lib/kubelet/credential-provider
32-
az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" -f /var/lib/kubelet/credential-provider/acr-credential-provider --auth-mode login
33-
chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
34-
az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml" -f /var/lib/kubelet/credential-provider-config.yaml --auth-mode login
35-
chmod 644 /var/lib/kubelet/credential-provider-config.yaml
36-
else
37-
echo "Using curl to download the OOT credential provider"
38-
mkdir -p /var/lib/kubelet/credential-provider
39-
curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
40-
chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider
41-
curl --retry 10 --retry-delay 5 -w "response status code is %{http_code}" -Lo /var/lib/kubelet/credential-provider-config.yaml "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
42-
chmod 644 /var/lib/kubelet/credential-provider-config.yaml
43-
fi
44-
- path: /tmp/kubeadm-bootstrap.sh
45-
owner: "root:root"
46-
permissions: "0744"
47-
content: |
48-
#!/bin/bash
49-
50-
set -o nounset
51-
set -o pipefail
52-
set -o errexit
53-
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
54-
55-
# This test installs release packages or binaries that are a result of the CI and release builds.
56-
# It runs '... --version' commands to verify that the binaries are correctly installed
57-
# and finally uninstalls the packages.
58-
# For the release packages it tests all versions in the support skew.
59-
LINE_SEPARATOR="*************************************************"
60-
echo "$$LINE_SEPARATOR"
61-
CI_VERSION=${CI_VERSION}
62-
if [[ "$${CI_VERSION}" != "" ]]; then
63-
CI_DIR=/tmp/k8s-ci
64-
mkdir -p $$CI_DIR
65-
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
66-
declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
67-
CONTAINER_EXT="tar"
68-
echo "* testing CI version $$CI_VERSION"
69-
# Check for semver
70-
if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
71-
VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
72-
DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
73-
curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
74-
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
75-
apt-get update
76-
# replace . with \.
77-
VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
78-
PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
79-
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
80-
echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
81-
DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
82-
done
83-
else
84-
CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
85-
fi
86-
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
87-
echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
88-
wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE"
89-
chmod +x "$$CI_DIR/$$CI_PACKAGE"
90-
mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
91-
done
92-
IMAGE_REGISTRY_PREFIX=registry.k8s.io
93-
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
94-
echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
95-
wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
96-
$${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
97-
$${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
98-
$${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
99-
done
100-
fi
101-
systemctl restart kubelet
102-
fi
103-
echo "* checking binary versions"
104-
echo "ctr version: " $(ctr version)
105-
echo "kubeadm version: " $(kubeadm version -o=short)
106-
echo "kubectl version: " $(kubectl version --client=true)
107-
echo "kubelet version: " $(kubelet --version)
108-
echo "$$LINE_SEPARATOR"
109-
- path: /etc/kubernetes/azure.json
110-
owner: "root:root"
111-
permissions: "0644"
112-
contentFrom:
113-
secret:
114-
key: worker-node-azure.json
115-
name: ${CLUSTER_NAME}-mp-0-azure-json
116-
---
1171
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
1182
kind: AzureMachinePool
1193
metadata:

0 commit comments

Comments
 (0)