Skip to content

Commit 86d5e88

Browse files
ingvagabundingvagabund
committed
Allow to deploy kubemark via cvo with kustomize
1 parent cb677b3 commit 86d5e88

8 files changed

+269
-0
lines changed

‎Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ build-integration: ## Build integration test binary
5252
mkdir -p bin
5353
$(DOCKER_CMD) go build $(GOGCFLAGS) -o bin/integration github.com/openshift/machine-api-operator/test/integration
5454

55+
.PHONY: test-e2e
5556
test-e2e: ## Run openshift specific e2e test
5657
go test -timeout 60m \
5758
-v ./vendor/github.com/openshift/cluster-api-actuator-pkg/pkg/e2e \
@@ -60,6 +61,12 @@ test-e2e: ## Run openshift specific e2e test
6061
-ginkgo.v \
6162
-args -v 5 -logtostderr true
6263

64+
.PHONY: deploy-kubemark
65+
deploy-kubemark:
66+
kustomize build config | kubectl apply -f -
67+
kustomize build | kubectl apply -f -
68+
kubectl apply -f config/kubemark-install-config.yaml
69+
6370
.PHONY: test
6471
test: ## Run tests
6572
@echo -e "\033[32mTesting...\033[0m"

‎README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,38 @@ However you can run it in a vanilla Kubernetes cluster by precreating some asset
129129
- Then you can run it as a [deployment](install/0000_50_machine-api-operator_08_deployment.yaml)
130130
- You should then be able to deploy a [cluster](test/integration/manifests/cluster.yaml) and a [machineSet](test/integration/manifests/machineset.yaml) object
131131

132+
## Machine API operator with Kubemark over Kubernetes
133+
134+
INFO: For development and testing purposes only
135+
136+
1. Deploy MAO over Kubernetes:
137+
```sh
138+
$ kustomize build | kubectl apply -f -
139+
```
140+
141+
2. Deploy [Kubemark actuator](https://github.com/openshift/cluster-api-provider-kubemark) prerequisities:
142+
```sh
143+
$ kustomize build config | kubectl apply -f -
144+
```
145+
146+
3. Create `cluster-config-v1` configmap to tell the MAO to deploy `kubemark` provider:
147+
```yaml
148+
apiVersion: v1
149+
kind: ConfigMap
150+
metadata:
151+
name: cluster-config-v1
152+
namespace: kube-system
153+
data:
154+
install-config: |-
155+
platform:
156+
kubemark: {}
157+
```
158+
159+
The file is already present under `config/kubemark-install-config.yaml` so it's sufficient to run:
160+
```sh
161+
$ kubectl apply -f config/kubemark-install-config.yaml
162+
```
163+
132164
## CI & tests
133165

134166
Run unit test:

‎config/0000_00_cluster-version-operator_01_clusteroperator.crd.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
kind: CustomResourceDefinition
2+
apiVersion: apiextensions.k8s.io/v1beta1
3+
metadata:
4+
name: clusteroperators.config.openshift.io
5+
spec:
6+
additionalPrinterColumns:
7+
- JSONPath: .status.version
8+
description: The version the operator is at.
9+
name: Version
10+
type: string
11+
- JSONPath: .status.conditions[?(@.type=="Available")].status
12+
description: Whether the operator is running and stable.
13+
name: Available
14+
type: string
15+
- JSONPath: .status.conditions[?(@.type=="Progressing")].status
16+
description: Whether the operator is processing changes.
17+
name: Progressing
18+
type: string
19+
- JSONPath: .status.conditions[?(@.type=="Failing")].status
20+
description: Whether the operator is failing changes.
21+
name: Failing
22+
type: string
23+
- JSONPath: .status.conditions[?(@.type=="Available")].lastTransitionTime
24+
description: The time the operator's Available status last changed.
25+
name: Since
26+
type: date
27+
group: config.openshift.io
28+
names:
29+
kind: ClusterOperator
30+
listKind: ClusterOperatorList
31+
plural: clusteroperators
32+
singular: clusteroperator
33+
shortNames:
34+
- co
35+
scope: Cluster
36+
subresources:
37+
status: {}
38+
version: v1
39+
versions:
40+
- name: v1
41+
served: true
42+
storage: true

‎config/kubemark-install-config.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cluster-config-v1
5+
namespace: kube-system
6+
data:
7+
install-config: |-
8+
platform:
9+
kubemark: {}

‎config/kubemark.yaml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
---
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: deleteunreadynodes
6+
namespace: kube-system
7+
data:
8+
entrypoint.sh: |-
9+
#!/bin/bash
10+
while true; do
11+
echo "Checking NotReady nodes"
12+
for node in $(kubectl get nodes -o json | jq '.items[].metadata.name' --raw-output); do
13+
echo "Checking node $node"
14+
15+
taint=$(kubectl get nodes $node -o json | jq '.spec | select(.taints!=null) | .taints[] | select(.key=="kubemark") | select (.!=null) | select(.value=="true")' | wc -l)
16+
if [ $taint -eq 0 ]; then
17+
echo "Skipping $node, no 'kubemark' taint found"
18+
continue
19+
fi
20+
21+
status=$(kubectl get node $node -o json | jq '.status.conditions[] | select(.type=="Ready") | .status' --raw-output)
22+
if [ $status != "Unknown" ]; then
23+
continue
24+
fi
25+
26+
# Delete node
27+
echo "Deleting node $node"
28+
kubectl delete node $node
29+
done
30+
sleep 10s
31+
done
32+
---
33+
apiVersion: apps/v1
34+
kind: Deployment
35+
metadata:
36+
name: machineapi-kubemark-controllers
37+
namespace: kube-system
38+
labels:
39+
api: machineapi
40+
k8s-app: kubemark
41+
spec:
42+
selector:
43+
matchLabels:
44+
api: machineapi
45+
k8s-app: kubemark
46+
replicas: 1
47+
template:
48+
metadata:
49+
labels:
50+
api: machineapi
51+
k8s-app: kubemark
52+
spec:
53+
nodeSelector:
54+
node-role.kubernetes.io/master: ""
55+
tolerations:
56+
- effect: NoSchedule
57+
key: node-role.kubernetes.io/master
58+
- key: CriticalAddonsOnly
59+
operator: Exists
60+
- effect: NoExecute
61+
key: node.alpha.kubernetes.io/notReady
62+
operator: Exists
63+
- effect: NoExecute
64+
key: node.alpha.kubernetes.io/unreachable
65+
operator: Exists
66+
containers:
67+
- name: unready-nodes-gb
68+
image: gofed/kubemark-machine-controllers:v1.0
69+
command:
70+
- /bin/entrypoint.sh
71+
volumeMounts:
72+
- name: deleteunreadynodes
73+
mountPath: /bin/entrypoint.sh
74+
readOnly: true
75+
subPath: entrypoint.sh
76+
volumes:
77+
- name: deleteunreadynodes
78+
configMap:
79+
defaultMode: 0700
80+
name: deleteunreadynodes

‎config/kubemark_rbac.yaml

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
---
2+
apiVersion: v1
3+
kind: Namespace
4+
metadata:
5+
name: kubemark-actuator
6+
---
7+
apiVersion: v1
8+
kind: ServiceAccount
9+
metadata:
10+
name: kubemark
11+
namespace: kubemark-actuator
12+
---
13+
apiVersion: rbac.authorization.k8s.io/v1
14+
kind: ClusterRole
15+
metadata:
16+
name: kubemark-actuator-role
17+
rules:
18+
- apiGroups:
19+
- ""
20+
resources:
21+
- nodes
22+
verbs:
23+
- create
24+
- get
25+
- list
26+
- watch
27+
- apiGroups:
28+
- ""
29+
resources:
30+
- nodes/status
31+
verbs:
32+
- patch
33+
- apiGroups:
34+
- ""
35+
resources:
36+
- services
37+
- secrets
38+
verbs:
39+
- list
40+
- watch
41+
- get
42+
- apiGroups:
43+
- ""
44+
resources:
45+
- pods
46+
verbs:
47+
- list
48+
- watch
49+
- get
50+
- delete
51+
- apiGroups:
52+
- ""
53+
resources:
54+
- pods/status
55+
verbs:
56+
- patch
57+
---
58+
apiVersion: rbac.authorization.k8s.io/v1
59+
kind: ClusterRoleBinding
60+
metadata:
61+
name: kubemark-actuator-rolebinding
62+
roleRef:
63+
apiGroup: rbac.authorization.k8s.io
64+
kind: ClusterRole
65+
name: kubemark-actuator-role
66+
subjects:
67+
- kind: ServiceAccount
68+
name: kubemark
69+
namespace: kubemark-actuator

‎config/kustomization.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
resources:
2+
# Install kubemark RBAC rules and namespace so mao can properly deploy kubemark actuator if configured
3+
# Pulled from https://github.com/openshift/cluster-api-provider-kubemark/blob/master/config/rbac/kubemark_rbac.yaml
4+
- kubemark_rbac.yaml
5+
# Pulled from https://github.com/openshift/cluster-api-provider-kubemark/blob/master/config/controllers/kubemark.yaml
6+
- kubemark.yaml

‎kustomization.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Adds namespace to all resources.
2+
namespace: kube-system
3+
4+
# Each entry in this list must resolve to an existing
5+
# resource definition in YAML. These are the resource
6+
# files that kustomize reads, modifies and emits as a
7+
# YAML string, with resources separated by document
8+
# markers ("---").
9+
resources:
10+
# Install CVO clusteroperator CRD required by mao (so it can report its status)
11+
# Pulled from https://github.com/openshift/cluster-version-operator/blob/master/install/0000_00_cluster-version-operator_01_clusteroperator.crd.yaml
12+
# and updated (delete depricated clusteroperators.operatorstatus.openshift.io CRD)
13+
- config/0000_00_cluster-version-operator_01_clusteroperator.crd.yaml
14+
# Install mao namespaces, CRDS and other resources to properly deploy machine API stack
15+
- install/0000_30_machine-api-operator_00_namespace.yaml
16+
- install/0000_30_machine-api-operator_01_images.configmap.yaml
17+
- install/0000_30_machine-api-operator_02_machine.crd.yaml
18+
- install/0000_30_machine-api-operator_03_machineset.crd.yaml
19+
- install/0000_30_machine-api-operator_04_machinedeployment.crd.yaml
20+
- install/0000_30_machine-api-operator_05_cluster.crd.yaml
21+
- install/0000_30_machine-api-operator_06_machineclass.crd.yaml
22+
- install/0000_30_machine-api-operator_07_machinehealthcheck.crd.yaml
23+
- install/0000_30_machine-api-operator_08_rbac.yaml
24+
- install/0000_30_machine-api-operator_09_deployment.yaml

0 commit comments

Comments
 (0)