diff --git a/Makefile b/Makefile index 33e25322f..0c528d0cc 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./pkg/... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./pkg/... ./api/... -coverprofile cover.out ##@ Build diff --git a/PROJECT b/PROJECT index e8b7d8da4..73a48bc52 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,8 @@ resources: kind: JobSet path: sigs.k8s.io/jobset/api/v1alpha1 version: v1alpha1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/README.md b/README.md index ba2f0d289..279a6f685 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,19 @@ JobSet: An API for managing a group of Jobs as a unit. # Installation -To install the CRD and deploy the controller on the cluster selected on your `~/.kubeconfig`, run the following commands: +### Prerequisites +[cert-manager](https://cert-manager.io/) is required to create certificates for the webhook. To install +it on your cluster, run the following command: +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml +``` +See more details about [cert-manager installation](https://cert-manager.io/docs/installation/). + +To install the JobSet CRD and deploy the controller on the cluster selected on your `~/.kubeconfig`, run the following commands: ``` git clone https://github.com/kubernetes-sigs/jobset.git cd jobset + IMAGE_REGISTRY=/ make image-push deploy ``` diff --git a/api/v1alpha1/jobset_webhook.go b/api/v1alpha1/jobset_webhook.go new file mode 100644 index 000000000..1dc2120ad --- /dev/null +++ b/api/v1alpha1/jobset_webhook.go @@ -0,0 +1,65 @@ +/* +Copyright 2023 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + batchv1 "k8s.io/api/batch/v1" +) + +func (r *JobSet) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-batch-x-k8s-io-v1alpha1-jobset,mutating=true,failurePolicy=fail,sideEffects=None,groups=batch.x-k8s.io,resources=jobsets,verbs=create;update,versions=v1alpha1,name=mjobset.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &JobSet{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *JobSet) Default() { + // Default job completion mode to indexed. + for i, rjob := range r.Spec.Jobs { + if rjob.Template.Spec.CompletionMode == nil { + r.Spec.Jobs[i].Template.Spec.CompletionMode = completionModePtr(batchv1.IndexedCompletion) + } + } +} + +//+kubebuilder:webhook:path=/validate-batch-x-k8s-io-v1alpha1-jobset,mutating=false,failurePolicy=fail,sideEffects=None,groups=batch.x-k8s.io,resources=jobsets,verbs=create;update,versions=v1alpha1,name=vjobset.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &JobSet{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *JobSet) ValidateCreate() error { + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *JobSet) ValidateUpdate(old runtime.Object) error { + return nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *JobSet) ValidateDelete() error { + return nil +} + +func completionModePtr(mode batchv1.CompletionMode) *batchv1.CompletionMode { + return &mode +} diff --git a/api/v1alpha1/jobset_webhook_test.go b/api/v1alpha1/jobset_webhook_test.go new file mode 100644 index 000000000..b67b78140 --- /dev/null +++ b/api/v1alpha1/jobset_webhook_test.go @@ -0,0 +1,82 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + batchv1 "k8s.io/api/batch/v1" +) + +func TestJobSetDefaulting(t *testing.T) { + testCases := []struct { + name string + js *JobSet + want *JobSet + }{ + { + name: "job completion mode is unset", + js: &JobSet{ + Spec: JobSetSpec{ + Jobs: []ReplicatedJob{ + { + Template: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{}, + }, + }, + }, + }, + }, + want: &JobSet{ + Spec: JobSetSpec{ + Jobs: []ReplicatedJob{ + { + Template: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + CompletionMode: completionModePtr(batchv1.IndexedCompletion), + }, + }, + }, + }, + }, + }, + }, + { + name: "job completion mode is set to non-indexed", + js: &JobSet{ + Spec: JobSetSpec{ + Jobs: []ReplicatedJob{ + { + Template: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + }, + }, + }, + }, + }, + }, + want: &JobSet{ + Spec: JobSetSpec{ + Jobs: []ReplicatedJob{ + { + Template: batchv1.JobTemplateSpec{ + Spec: batchv1.JobSpec{ + CompletionMode: completionModePtr(batchv1.NonIndexedCompletion), + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.js.Default() + if diff := cmp.Diff(tc.want, tc.js); diff != "" { + t.Errorf("unexpected jobset defaulting: (-want/+got): %s", diff) + } + }) + } +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 42a8c2c0e..81c22c29e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -20,7 +20,7 @@ package v1alpha1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 000000000..294bbc270 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,39 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: issuer + app.kubernetes.io/instance: selfsigned-issuer + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..bebea5a59 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..e631f7773 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d1ea91e84..19471dc05 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,12 +8,12 @@ resources: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_jobsets.yaml +- patches/webhook_in_jobsets.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_jobsets.yaml +- patches/cainjection_in_jobsets.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index c3baaa2db..4e82ca98a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,9 +18,9 @@ bases: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -34,39 +34,39 @@ patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- manager_webhook_patch.yaml +- manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml +- webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution vars: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..738de350b --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..2a9035112 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,29 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84c..3ac5f0b21 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: gcr.io/danielvm-gke-dev2/jobset + newTag: f250948-dirty diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..25e21e3c9 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..ae3ef9cd6 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-batch-x-k8s-io-v1alpha1-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - batch.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-batch-x-k8s-io-v1alpha1-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - batch.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..758586ecc --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,20 @@ + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/main.go b/main.go index 61a83f2a1..551f555ad 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - jobsetv1alpha1 "sigs.k8s.io/jobset/api/v1alpha1" + jobset "sigs.k8s.io/jobset/api/v1alpha1" "sigs.k8s.io/jobset/pkg/controllers" //+kubebuilder:scaffold:imports ) @@ -44,7 +44,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(jobsetv1alpha1.AddToScheme(scheme)) + utilruntime.Must(jobset.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -93,6 +93,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "JobSet") os.Exit(1) } + if err = (&jobset.JobSet{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "JobSet") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/test/integration/jobset_controller_test.go b/test/integration/controller/jobset_controller_test.go similarity index 98% rename from test/integration/jobset_controller_test.go rename to test/integration/controller/jobset_controller_test.go index 62bf319ff..779bc5d46 100644 --- a/test/integration/jobset_controller_test.go +++ b/test/integration/controller/jobset_controller_test.go @@ -13,7 +13,7 @@ // limitations under the License. // */ -package test +package controllertest import ( "context" @@ -80,7 +80,7 @@ var _ = ginkgo.Describe("JobSet validation", func() { updates []*jobSetUpdate } - ginkgo.DescribeTable("validation on jobset creation and updates", + ginkgo.DescribeTable("JobSet validation during creation and updates", func(tc *testCase) { ctx := context.Background() @@ -123,7 +123,9 @@ var _ = ginkgo.Describe("JobSet validation", func() { makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { return testing.MakeJobSet("js-hostnames-non-indexed", ns.Name). ReplicatedJob(testing.MakeReplicatedJob("test-job"). - Job(testing.MakeJobTemplate("test-job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Job(testing.MakeJobTemplate("test-job", ns.Name). + PodSpec(testing.TestPodSpec). + CompletionMode(batchv1.NonIndexedCompletion).Obj()). EnableDNSHostnames(true). Obj()) }, diff --git a/test/integration/suite_test.go b/test/integration/controller/suite_test.go similarity index 96% rename from test/integration/suite_test.go rename to test/integration/controller/suite_test.go index d44f8116e..bd770f8a1 100644 --- a/test/integration/suite_test.go +++ b/test/integration/controller/suite_test.go @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package test +package controllertest import ( "context" @@ -59,7 +59,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } diff --git a/test/integration/webhook/jobset_webhook_test.go b/test/integration/webhook/jobset_webhook_test.go new file mode 100644 index 000000000..ea9b17e4f --- /dev/null +++ b/test/integration/webhook/jobset_webhook_test.go @@ -0,0 +1,120 @@ +// /* +// Copyright 2023 The Kubernetes Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ + +package webhooktest + +import ( + "context" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + jobset "sigs.k8s.io/jobset/api/v1alpha1" + "sigs.k8s.io/jobset/pkg/util/testing" +) + +const ( + timeout = 10 * time.Second + interval = time.Millisecond * 250 +) + +var _ = ginkgo.Describe("jobset webhook defaulting", func() { + + // Each test runs in a separate namespace. + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + // Create test namespace before each test. + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-ns-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + + // Wait for namespace to exist before proceeding with test. + gomega.Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Namespace, Name: ns.Name}, ns) + if err != nil { + return false + } + return true + }, timeout, interval).Should(gomega.BeTrue()) + }) + + ginkgo.AfterEach(func() { + // Delete test namespace after each test. + gomega.Expect(k8sClient.Delete(ctx, ns)).To(gomega.Succeed()) + }) + + type testCase struct { + makeJobSet func(ns *corev1.Namespace) *testing.JobSetWrapper + defaultsApplied func(*jobset.JobSet) bool + } + + ginkgo.DescribeTable("defaulting on jobset creation", + func(tc *testCase) { + ctx := context.Background() + + // Create JobSet. + ginkgo.By("creating jobset") + js := tc.makeJobSet(ns).Obj() + + // Verify jobset created successfully. + ginkgo.By("checking that jobset creation succeeds") + gomega.Expect(k8sClient.Create(ctx, js)).Should(gomega.Succeed()) + + // We'll need to retry getting this newly created jobset, given that creation may not immediately happen. + var fetchedJS jobset.JobSet + gomega.Eventually(k8sClient.Get(ctx, types.NamespacedName{Name: js.Name, Namespace: js.Namespace}, &fetchedJS), timeout, interval).Should(gomega.Succeed()) + + // Check defaulting. + gomega.Expect(tc.defaultsApplied(&fetchedJS)).Should(gomega.Equal(true)) + }, + ginkgo.Entry("job.spec.completionMode defaults to indexed if unset", &testCase{ + makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { + return testing.MakeJobSet("completionmode-unset", ns.Name). + ReplicatedJob(testing.MakeReplicatedJob("test-job"). + Job(testing.MakeJobTemplate("test-job", ns.Name). + PodSpec(testing.TestPodSpec).Obj()). + EnableDNSHostnames(true). + Obj()) + }, + defaultsApplied: func(js *jobset.JobSet) bool { + completionMode := js.Spec.Jobs[0].Template.Spec.CompletionMode + return completionMode != nil && *completionMode == batchv1.IndexedCompletion + }, + }), + ginkgo.Entry("job.spec.completionMode unchanged if already set", &testCase{ + makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { + return testing.MakeJobSet("completionmode-nonindexed", ns.Name). + ReplicatedJob(testing.MakeReplicatedJob("test-job"). + Job(testing.MakeJobTemplate("test-job", ns.Name). + CompletionMode(batchv1.NonIndexedCompletion). + PodSpec(testing.TestPodSpec).Obj()). + Obj()) + }, + defaultsApplied: func(js *jobset.JobSet) bool { + completionMode := js.Spec.Jobs[0].Template.Spec.CompletionMode + return completionMode != nil && *completionMode == batchv1.NonIndexedCompletion + }, + }), + ) // end of DescribeTable +}) // end of Describe diff --git a/test/integration/webhook/suite_test.go b/test/integration/webhook/suite_test.go new file mode 100644 index 000000000..3224b4c25 --- /dev/null +++ b/test/integration/webhook/suite_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2023 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooktest + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + //+kubebuilder:scaffold:imports + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + jobset "sigs.k8s.io/jobset/api/v1alpha1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + // scheme := runtime.NewScheme() + err = jobset.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + LeaderElection: false, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&jobset.JobSet{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + conn.Close() + return nil + }).Should(Succeed()) + +}) + +var _ = AfterSuite(func() { + // https://github.com/kubernetes-sigs/controller-runtime/issues/1571 + cancel() + By("tearing down the test environment, but I do nothing here.") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +})