Skip to content

Commit 275fc71

Browse files
benfiolabenfiola
committed
feat: external cert support
* Add cert-manager to dev manifests * Add cert-manager resources to dev manifests * Deploy cert-manager as part of 'apply-manifests' Makefile target * Generate cert-manager resources as part of 'apply-manifests' Makefile target * Update operator 'getTenantClientInfo' to correctly set 'secure' flag based upon external cert settings * Update operator 'getTenantClientInfo' to build cert pool with 'operator-ca-tls' prefixed secrets * Update operator 'getTenantClientInfo' to build cert pool with tenant spec 'externalCaCertSecret' secrets
1 parent 8d3dd53 commit 275fc71

File tree

9 files changed

+135
-33
lines changed

9 files changed

+135
-33
lines changed

‎Makefile

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ ifeq ($(ALTARCH),amd64)
1919
ALTARCH = x86_64
2020
endif
2121

22+
CERT_MANAGER_MANIFEST = $(ASSETS)/cert-manager.yaml
23+
CERT_MANAGER_MANIFEST_SRC = $(DEV)/manifests/cert-manager
24+
CERT_MANAGER_RESOURCES_MANIFEST = $(ASSETS)/cert-manager-resources.yaml
25+
CERT_MANAGER_RESOURCES_MANIFEST_SRC = $(DEV)/manifests/cert-manager-resources
2226
CONTROLLER_GEN = $(ASSETS)/controller-gen
2327
CONTROLLER_GEN_URL = https://github.com/kubernetes-sigs/controller-tools/releases/download/v$(CONTROLLER_GEN_VERSION)/controller-gen-$(OS)-$(ARCH)
2428
CRANE = $(ASSETS)/crane
@@ -218,7 +222,9 @@ $(MINIKUBE): | $(ASSETS)
218222
# MANIFESTS
219223

220224
.PHONY: apply-manifests
221-
apply-manifests: $(CRDS_MANIFEST) $(KUBECTL) $(MINIO_OPERATOR_MANIFEST) $(MINIO_TENANT_MANIFEST) $(OPENLDAP_MANIFEST)
225+
apply-manifests: $(CERT_MANAGER_MANIFEST) $(CERT_MANAGER_RESOURCES_MANIFEST) $(CRDS_MANIFEST) $(KUBECTL) $(MINIO_OPERATOR_MANIFEST) $(MINIO_TENANT_MANIFEST) $(OPENLDAP_MANIFEST)
226+
# deploy cert-manager
227+
$(KUBECTL_CMD) apply -f $(CERT_MANAGER_MANIFEST)
222228
# deploy minio operator
223229
$(KUBECTL_CMD) apply -f $(MINIO_OPERATOR_MANIFEST)
224230
# deploy minio tenant
@@ -228,6 +234,14 @@ apply-manifests: $(CRDS_MANIFEST) $(KUBECTL) $(MINIO_OPERATOR_MANIFEST) $(MINIO_
228234
# deploy crds
229235
$(KUBECTL_CMD) apply -f $(CRDS_MANIFEST)
230236

237+
$(CERT_MANAGER_MANIFEST): $(KUBECTL) $(HELM) | $(ASSETS)
238+
# generate cert-manager manifest
239+
$(KUSTOMIZE_CMD) $(CERT_MANAGER_MANIFEST_SRC) > $(CERT_MANAGER_MANIFEST)
240+
241+
$(CERT_MANAGER_RESOURCES_MANIFEST): $(KUBECTL) $(HELM) | $(ASSETS)
242+
# generate cert-manager resources manifest
243+
$(KUSTOMIZE_CMD) $(CERT_MANAGER_RESOURCES_MANIFEST_SRC) > $(CERT_MANAGER_RESOURCES_MANIFEST)
244+
231245
$(CRDS_MANIFEST): generate $(KUBECTL) $(HELM) | $(ASSETS)
232246
# generate crds manifest
233247
$(KUSTOMIZE_CMD) $(CRDS_MANIFEST_SRC) > $(CRDS_MANIFEST)

‎README.md

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,18 @@ The following arguments/environment variables configure the operator:
5151

5252
The operator requires the a service account with the following RBAC settings:
5353

54-
| Resource | Verbs | Why |
55-
| -------------------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------- |
56-
| minio.min.io/v2/Tenant | Get | Used to discover MinIO tenants |
57-
| v1/ConfigMap | Get | Used to obtain the CA bundle used to generate a MinIO tenant's TLS certificates (- for HTTP client cert validation) |
58-
| v1/Secret | Get | Used to fetch a MinIO tenant's configuration (which is stored as a secret) |
59-
| v1/Services | Get | Used to determine a MinIO tenant's internal endpoint |
60-
| bfiola.dev/v1/MinioBucket | Get, Watch, List, Update | Required for the operator to manage minio bucket resources |
61-
| bfiola.dev/v1/MinioGroup | Get, Watch, List, Update | Required for the operator to manage minio group resources |
62-
| bfiola.dev/v1/MinioGroupBinding | Get, Watch, List, Update | Required for the operator to manage minio group membership |
63-
| bfiola.dev/v1/MinioPolicy | Get, Watch, List, Update | Required for the operator to manage minio policies |
64-
| bfiola.dev/v1/MinioPolicyBinding | Get, Watch, List, Update | Required for the operator to manage minio policy attachments |
65-
| bfiola.dev/v1/MinioUser | Get, Watch, List, Update | Required for the operator to manage minio user resources |
54+
| Resource | Verbs | Why |
55+
| -------------------------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
56+
| minio.min.io/v2/Tenant | Get | Used to discover MinIO tenants |
57+
| v1/ConfigMap | Get | Used to fetch TLS certificate data to perform TLS verification when connecting to MinIO tenants |
58+
| v1/Secret | Get, List | Used to fetch a MinIO tenant's configuration (which is stored as a secret) and fetch TLS certificate data to perform TLS verification when connecting to MinIO tenants |
59+
| v1/Services | Get | Used to determine a MinIO tenant's internal endpoint |
60+
| bfiola.dev/v1/MinioBucket | Get, Watch, List, Update | Required for the operator to manage minio bucket resources |
61+
| bfiola.dev/v1/MinioGroup | Get, Watch, List, Update | Required for the operator to manage minio group resources |
62+
| bfiola.dev/v1/MinioGroupBinding | Get, Watch, List, Update | Required for the operator to manage minio group membership |
63+
| bfiola.dev/v1/MinioPolicy | Get, Watch, List, Update | Required for the operator to manage minio policies |
64+
| bfiola.dev/v1/MinioPolicyBinding | Get, Watch, List, Update | Required for the operator to manage minio policy attachments |
65+
| bfiola.dev/v1/MinioUser | Get, Watch, List, Update | Required for the operator to manage minio user resources |
6666

6767
## Limitations
6868

‎dev/manifests/cert-manager-resources/kustomization.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
resources:
2+
- ./root-issuer.yaml
3+
- ./tenant-ca.yaml
4+
- ./tenant-issuer.yaml
5+
- ./tenant-cert.yaml

‎dev/manifests/cert-manager-resources/root-issuer.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: ClusterIssuer
3+
metadata:
4+
name: cm-root
5+
spec:
6+
selfSigned: {}

‎dev/manifests/cert-manager-resources/tenant-ca.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: cm-tenant-ca
5+
spec:
6+
isCA: true
7+
commonName: cm-tenant
8+
secretName: cm-tenant-ca-tls
9+
duration: 70128h
10+
privateKey:
11+
algorithm: ECDSA
12+
size: 256
13+
issuerRef:
14+
name: cm-root
15+
kind: ClusterIssuer
16+
group: cert-manager.io

‎dev/manifests/cert-manager-resources/tenant-cert.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: cm-tenant-cert
5+
spec:
6+
dnsNames:
7+
- minio.default
8+
- minio.default.svc
9+
- minio.default.svc.cluster.local
10+
- "*.tenant-hl.default.svc.cluster.local"
11+
- "*.default.svc.cluster.local"
12+
- "*.tenant.minio.default.svc.cluster.local"
13+
secretName: cm-tenant-tls
14+
issuerRef:
15+
name: cm-tenant-issuer

‎dev/manifests/cert-manager-resources/tenant-issuer.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
name: cm-tenant-issuer
5+
spec:
6+
ca:
7+
secretName: cm-tenant-ca-tls

‎dev/manifests/cert-manager/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
resources:
2+
- https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml

‎internal/operator/operator.go

Lines changed: 57 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"crypto/tls"
2424
"crypto/x509"
2525
"encoding/json"
26-
"encoding/pem"
2726
"fmt"
2827
"io"
2928
"log/slog"
@@ -209,7 +208,7 @@ type minioTenantClientInfo struct {
209208

210209
// +kubebuilder:rbac:groups=core,resources=services,verbs=get
211210
// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get
212-
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get
211+
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list
213212
// +kubebuilder:rbac:groups=minio.min.io,resources=tenants,verbs=get
214213

215214
// Returns [minioTenantClientInfo] for a [miniov2.Tenant] referenced by [v1.ResourceRef].
@@ -227,9 +226,11 @@ func getMinioTenantClientInfo(ctx context.Context, c client.Client, rr v1.Resour
227226
}
228227

229228
// determine if tenant secure
230-
s := true
231-
if t.Spec.RequestAutoCert != nil {
232-
s = *t.Spec.RequestAutoCert
229+
s := false
230+
if t.Spec.RequestAutoCert != nil && *t.Spec.RequestAutoCert {
231+
s = true
232+
} else if len(t.Spec.ExternalCertSecret) > 0 {
233+
s = true
233234
}
234235

235236
// obtain access + secret key
@@ -281,25 +282,61 @@ func getMinioTenantClientInfo(ctx context.Context, c client.Client, rr v1.Resour
281282
return nil, fmt.Errorf("endpoint for minio service %s/%s not found", tsvc.Namespace, tsvc.Name)
282283
}
283284

284-
// obtain ca bundle
285-
krccm := &corev1.ConfigMap{}
286-
err = c.Get(ctx, types.NamespacedName{Namespace: rr.Namespace, Name: "kube-root-ca.crt"}, krccm)
285+
// create cert pool for tls verification
286+
cbp, err := x509.SystemCertPool()
287287
if err != nil {
288-
return nil, err
288+
cbp = x509.NewCertPool()
289+
}
290+
ks := []string{"public.crt", "tls.crt", "ca.crt"}
291+
// pod CA
292+
pcacm := &corev1.ConfigMap{}
293+
err = c.Get(ctx, types.NamespacedName{Namespace: rr.Namespace, Name: "kube-root-ca.crt"}, pcacm)
294+
if err == nil {
295+
for _, k := range ks {
296+
cas, ok := pcacm.Data[k]
297+
if !ok {
298+
continue
299+
}
300+
ca := []byte(cas)
301+
cbp.AppendCertsFromPEM(ca)
302+
}
289303
}
290-
cb, ok := krccm.Data["ca.crt"]
291-
if !ok {
292-
return nil, fmt.Errorf("kube root ca for %s not found", rr.Namespace)
304+
// CAs from secrets prefixed with 'operator-ca-tls'
305+
sl := &corev1.SecretList{}
306+
err = c.List(ctx, sl, client.InNamespace(rr.Namespace))
307+
if err == nil {
308+
for _, s := range sl.Items {
309+
if !strings.HasPrefix(s.Name, "operator-ca-tls") {
310+
continue
311+
}
312+
for _, k := range ks {
313+
cas, ok := s.Data[k]
314+
if !ok {
315+
continue
316+
}
317+
ca := []byte(cas)
318+
cbp.AppendCertsFromPEM(ca)
319+
}
320+
}
293321
}
294-
295-
// create cert pool with ca bundle
296-
cbp := x509.NewCertPool()
297-
cbb, _ := pem.Decode([]byte(cb))
298-
cbc, err := x509.ParseCertificate(cbb.Bytes)
299-
if err != nil {
300-
return nil, err
322+
// CAs from configured external sources
323+
for _, eca := range t.Spec.ExternalCaCertSecret {
324+
if eca.Type == "kubernetes.io/tls" {
325+
s := &corev1.Secret{}
326+
err := c.Get(ctx, types.NamespacedName{Namespace: t.Namespace, Name: eca.Name}, s)
327+
if err != nil {
328+
continue
329+
}
330+
for _, k := range ks {
331+
cas, ok := s.Data[k]
332+
if !ok {
333+
continue
334+
}
335+
ca := []byte(cas)
336+
cbp.AppendCertsFromPEM(ca)
337+
}
338+
}
301339
}
302-
cbp.AddCert(cbc)
303340

304341
return &minioTenantClientInfo{
305342
AccessKey: ak,

0 commit comments

Comments
 (0)