Skip to content
This repository was archived by the owner on Apr 25, 2023. It is now read-only.

Commit 5934e63

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1346 from mesosphere/hectorj2f/fix_broken_upgrade
fix: broken upgrade path from previous versions
2 parents c999f0d + 7fb0814 commit 5934e63

File tree

5 files changed

+173
-24
lines changed

5 files changed

+173
-24
lines changed

‎charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml

+3-2
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,13 @@ spec:
2424
{{- if .Values.featureGates }}
2525
- name: PushReconciler
2626
configuration: {{ .Values.featureGates.PushReconciler | default "Enabled" | quote }}
27-
- name: RawResourceStatusCollection
28-
configuration: {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }}
2927
- name: SchedulerPreferences
3028
configuration: {{ .Values.featureGates.SchedulerPreferences | default "Enabled" | quote }}
3129
- name: CrossClusterServiceDiscovery
3230
configuration: {{ .Values.featureGates.CrossClusterServiceDiscovery | default "Disabled" | quote }}
3331
- name: FederatedIngress
3432
configuration: {{ .Values.featureGates.FederatedIngress | default "Disabled" | quote }}
33+
# NOTE: Commented feature gate to fix https://github.com/kubernetes-sigs/kubefed/issues/1333
34+
#- name: RawResourceStatusCollection
35+
# configuration: {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }}
3536
{{- end }}

‎charts/kubefed/charts/controllermanager/templates/post-install-job.yaml

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
---
2+
kind: ConfigMap
3+
apiVersion: v1
4+
metadata:
5+
name: "{{ .Release.Name }}-kubefed-config-hook"
6+
namespace: "{{ .Release.Namespace }}"
7+
annotations:
8+
"helm.sh/hook": post-install, post-upgrade
9+
"helm.sh/hook-weight": "-5"
10+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
11+
data:
12+
setup.sh: |-
13+
#!/bin/bash
14+
set -euo pipefail
15+
16+
kubectl patch kubefedconfig -n {{ .Release.Namespace }} kubefed --type='json' -p='[{"op": "add", "path": "/spec/featureGates", "value":[{"configuration": {{ .Values.featureGates.PushReconciler | default "Enabled" | quote }},"name":"PushReconciler"},{"configuration": {{ .Values.featureGates.CrossClusterServiceDiscovery | default "Disabled" | quote }},"name":"CrossClusterServiceDiscovery"},{"configuration": {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }},"name":"RawResourceStatusCollection"},{"configuration": {{ .Values.featureGates.FederatedIngress | default "Disabled" | quote }},"name":"FederatedIngress"},{"configuration": {{ .Values.featureGates.SchedulerPreferences | default "Enabled" | quote }},"name":"SchedulerPreferences"}]}]'
17+
18+
echo "Kubefedconfig patched successfully!"
19+
20+
kubectl rollout restart deployment/kubefed-controller-manager -n {{ .Release.Namespace }}
21+
---
22+
apiVersion: batch/v1
23+
kind: Job
24+
metadata:
25+
name: "{{ .Release.Name }}-{{ randAlphaNum 10 | lower }}"
26+
namespace: "{{ .Release.Namespace }}"
27+
labels:
28+
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
29+
app.kubernetes.io/instance: {{ .Release.Name | quote }}
30+
app.kubernetes.io/version: {{ .Chart.AppVersion }}
31+
helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
32+
annotations:
33+
"helm.sh/hook": post-install,post-upgrade
34+
"helm.sh/hook-weight": "-4"
35+
"helm.sh/hook-delete-policy": hook-succeeded,hook-failed
36+
spec:
37+
template:
38+
metadata:
39+
name: "{{ .Release.Name }}"
40+
labels:
41+
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
42+
app.kubernetes.io/instance: {{ .Release.Name | quote }}
43+
helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
44+
spec:
45+
restartPolicy: Never
46+
serviceAccountName: kubefed-config-hook
47+
automountServiceAccountToken: true
48+
containers:
49+
- name: post-install-job
50+
image: "bitnami/kubectl:1.17.16"
51+
command: ["/bin/bash"]
52+
args: ["/opt/scripts/setup.sh"]
53+
volumeMounts:
54+
- name: "scripts"
55+
mountPath: "/opt/scripts"
56+
volumes:
57+
- name: "scripts"
58+
configMap:
59+
name: "{{ .Release.Name }}-kubefed-config-hook"
60+
---
61+
apiVersion: rbac.authorization.k8s.io/v1
62+
kind: Role
63+
metadata:
64+
name: kubefed-config-hook
65+
namespace: {{ .Release.Namespace }}
66+
annotations:
67+
"helm.sh/hook": post-install, post-upgrade
68+
"helm.sh/hook-weight": "-5"
69+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
70+
rules:
71+
- apiGroups: ["apps"]
72+
resources: ["deployments"]
73+
verbs: ["patch","get"]
74+
- apiGroups: ["core.kubefed.io"]
75+
resources: ["kubefedconfigs"]
76+
verbs: ["patch","get"]
77+
---
78+
apiVersion: rbac.authorization.k8s.io/v1
79+
kind: RoleBinding
80+
metadata:
81+
name: kubefed-config-hook
82+
namespace: {{ .Release.Namespace }}
83+
annotations:
84+
"helm.sh/hook": post-install, post-upgrade
85+
"helm.sh/hook-weight": "-5"
86+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
87+
roleRef:
88+
apiGroup: rbac.authorization.k8s.io
89+
kind: Role
90+
name: kubefed-config-hook
91+
subjects:
92+
- kind: ServiceAccount
93+
name: kubefed-config-hook
94+
---
95+
apiVersion: v1
96+
kind: ServiceAccount
97+
metadata:
98+
name: kubefed-config-hook
99+
namespace: {{ .Release.Namespace }}
100+
annotations:
101+
"helm.sh/hook": post-install, post-upgrade
102+
"helm.sh/hook-weight": "-5"
103+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation

‎scripts/deploy-kubefed.sh

-21
Original file line numberDiff line numberDiff line change
@@ -92,27 +92,6 @@ function kubefed-admission-webhook-ready() {
9292
[[ "${readyReplicas}" -ge "1" ]]
9393
}
9494

95-
function deployment-image-as-expected() {
96-
local namespace="${1}"
97-
local deployment="${2}"
98-
local container="${3}"
99-
local expected_image="${4}"
100-
101-
local deployed_image
102-
deployed_image="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')"
103-
[[ "${deployed_image}" == "${expected_image}" ]]
104-
}
105-
106-
function check-command-installed() {
107-
local cmdName="${1}"
108-
109-
command -v "${cmdName}" >/dev/null 2>&1 ||
110-
{
111-
echo "${cmdName} command not found. Please download dependencies using ${BASH_SOURCE%/*}/download-binaries.sh and install it in your PATH." >&2
112-
exit 1
113-
}
114-
}
115-
11695
NS="${KUBEFED_NAMESPACE:-kube-federation-system}"
11796
IMAGE_NAME="${1:-}"
11897
NAMESPACED="${NAMESPACED:-}"

‎scripts/pre-commit.sh

+46-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ E2E_TEST_CMD="${TEMP_DIR}/e2e-${PLATFORM} ${COMMON_TEST_ARGS}"
3838
# given control plane scope.
3939
IN_MEMORY_E2E_TEST_CMD="go test -v -timeout 900s -race ./test/e2e -args ${COMMON_TEST_ARGS} -in-memory-controllers=true -limited-scope-in-memory-controllers=false"
4040

41+
KUBEFED_UPGRADE_TEST_NS="upgrade-test"
42+
4143
function build-binaries() {
4244
${MAKE_CMD} hyperfed
4345
${MAKE_CMD} controller
@@ -62,6 +64,43 @@ function run-e2e-tests() {
6264
${E2E_TEST_CMD}
6365
}
6466

67+
function run-e2e-upgrade-test() {
68+
HOST_CLUSTER="$(kubectl config current-context)"
69+
70+
echo "Adding a repo to install an older kubefed version"
71+
helm repo add kubefed-charts https://raw.githubusercontent.com/kubernetes-sigs/kubefed/master/charts
72+
helm repo update
73+
74+
# Get the previous version prior to our latest stable version
75+
KUBEFED_UPGRADE_TEST_VERSION=$(helm search repo kubefed-charts/kubefed --versions | awk '{print $2}' | head -3 | tail -1)
76+
77+
echo "Installing an older kubefed version v${KUBEFED_UPGRADE_TEST_VERSION}"
78+
helm install kubefed kubefed-charts/kubefed --namespace ${KUBEFED_UPGRADE_TEST_NS} --version=v${KUBEFED_UPGRADE_TEST_VERSION} --create-namespace --wait
79+
80+
deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-admission-webhook admission-webhook "quay.io/kubernetes-multicluster/kubefed:v${KUBEFED_UPGRADE_TEST_VERSION}"
81+
deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-controller-manager controller-manager "quay.io/kubernetes-multicluster/kubefed:v${KUBEFED_UPGRADE_TEST_VERSION}"
82+
83+
echo "Upgrading kubefed to current version"
84+
IMAGE_NAME="local/kubefed:e2e"
85+
local repository=${IMAGE_NAME%/*}
86+
local image_tag=${IMAGE_NAME##*/}
87+
local image=${image_tag%:*}
88+
local tag=${image_tag#*:}
89+
90+
helm upgrade -i kubefed charts/kubefed --namespace ${KUBEFED_UPGRADE_TEST_NS} \
91+
--set controllermanager.controller.repository=${repository} \
92+
--set controllermanager.controller.image=${image} \
93+
--set controllermanager.controller.tag=${tag} \
94+
--set controllermanager.webhook.repository=${repository} \
95+
--set controllermanager.webhook.image=${image} \
96+
--set controllermanager.webhook.tag=${tag} \
97+
--set controllermanager.featureGates.CrossClusterServiceDiscovery=Enabled,controllermanager.featureGates.FederatedIngress=Enabled,controllermanager.featureGates.RawResourceStatusCollection=Enabled \
98+
--wait
99+
100+
deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-admission-webhook admission-webhook "local/kubefed:e2e"
101+
deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-controller-manager controller-manager "local/kubefed:e2e"
102+
}
103+
65104
function run-e2e-tests-with-in-memory-controllers() {
66105
${IN_MEMORY_E2E_TEST_CMD}
67106
}
@@ -138,7 +177,7 @@ echo "Downloading e2e test dependencies"
138177

139178
KIND_TAG="v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b" ./scripts/create-clusters.sh
140179

141-
declare -a join_cluster_list=()
180+
declare -a join_cluster_list=()
142181
if [[ -z "${JOIN_CLUSTERS}" ]]; then
143182
for i in $(seq 2 "${NUM_CLUSTERS}"); do
144183
join_cluster_list+=("cluster${i}")
@@ -178,3 +217,9 @@ run-namespaced-e2e-tests
178217

179218
echo "Deleting namespace-scoped kubefed"
180219
KUBEFED_NAMESPACE=foo NAMESPACED=y DELETE_CLUSTER_RESOURCE=y ./scripts/delete-kubefed.sh
220+
221+
echo "Running e2e upgrade test"
222+
run-e2e-upgrade-test
223+
224+
echo "Deleting kubefed"
225+
KUBEFED_NAMESPACE=${KUBEFED_UPGRADE_TEST_NS} DELETE_CLUSTER_RESOURCE=y ./scripts/delete-kubefed.sh

‎scripts/util.sh

+21
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,24 @@ function util::wait-for-condition() {
9898
fi
9999
}
100100
readonly -f util::wait-for-condition
101+
102+
function deployment-image-as-expected() {
103+
local namespace="${1}"
104+
local deployment="${2}"
105+
local container="${3}"
106+
local expected_image="${4}"
107+
108+
local deployed_image
109+
deployed_image="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')"
110+
[[ "${deployed_image}" == "${expected_image}" ]]
111+
}
112+
113+
function check-command-installed() {
114+
local cmdName="${1}"
115+
116+
command -v "${cmdName}" >/dev/null 2>&1 ||
117+
{
118+
echo "${cmdName} command not found. Please download dependencies using ${BASH_SOURCE%/*}/download-binaries.sh and install it in your PATH." >&2
119+
exit 1
120+
}
121+
}

0 commit comments

Comments
 (0)