Skip to content

Commit bd9aa47

Browse files
authored
test: scaffold e2e tests (#189)
## Description This PR re-scaffolds the project using `kubebuilder alpha generate`. This is a follow-up to #185. - Updates a few things that were missed in the last PR wrt. metrics service - Initialize e2e tests. For now, we are only testing basic functionality. The tests should be expanded to cover validator-specific use-cases. --------- Signed-off-by: Artur Shad Nik <[email protected]>
1 parent c83531a commit bd9aa47

15 files changed

+769
-31
lines changed

.golangci.yml

+9
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ issues:
66
# don't skip warning about doc comments
77
# don't exclude the default set of lint
88
exclude-use-default: false
9+
# restore some of the defaults
10+
# (fill in the rest as needed)
11+
exclude-dirs:
12+
- tests
913
exclude-files:
1014
- ".*_test\\.go"
1115

@@ -32,3 +36,8 @@ linters:
3236
- unconvert
3337
- unparam
3438
- unused
39+
40+
linters-settings:
41+
revive:
42+
rules:
43+
- name: comment-spacings

api/v1alpha1/maasvalidator_types.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,8 @@ type Resource struct {
192192
// MaasValidatorStatus defines the observed state of MaasValidator
193193
type MaasValidatorStatus struct{}
194194

195-
//+kubebuilder:object:root=true
196-
//+kubebuilder:subresource:status
195+
// +kubebuilder:object:root=true
196+
// +kubebuilder:subresource:status
197197

198198
// MaasValidator is the Schema for the maasvalidators API
199199
type MaasValidator struct {
@@ -219,7 +219,7 @@ func (v MaasValidator) ResultCount() int {
219219
return v.Spec.ResultCount()
220220
}
221221

222-
//+kubebuilder:object:root=true
222+
// +kubebuilder:object:root=true
223223

224224
// MaasValidatorList contains a list of MaasValidator
225225
type MaasValidatorList struct {

cmd/main.go

+16-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright 2023.
2+
Copyright 2024.
33
44
Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
// Package main initializes a MaasValidator controller.
1817
package main
1918

2019
import (
@@ -34,11 +33,11 @@ import (
3433
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3534
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
3635
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
36+
"sigs.k8s.io/controller-runtime/pkg/webhook"
3737

3838
validationv1alpha1 "github.com/validator-labs/validator-plugin-maas/api/v1alpha1"
3939
"github.com/validator-labs/validator-plugin-maas/internal/controller"
40-
validatorv1alpha1 "github.com/validator-labs/validator/api/v1alpha1"
41-
//+kubebuilder:scaffold:imports
40+
// +kubebuilder:scaffold:imports
4241
)
4342

4443
var (
@@ -48,9 +47,9 @@ var (
4847

4948
func init() {
5049
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
51-
utilruntime.Must(validatorv1alpha1.AddToScheme(scheme))
50+
5251
utilruntime.Must(validationv1alpha1.AddToScheme(scheme))
53-
//+kubebuilder:scaffold:scheme
52+
// +kubebuilder:scaffold:scheme
5453
}
5554

5655
func main() {
@@ -60,17 +59,16 @@ func main() {
6059
var secureMetrics bool
6160
var enableHTTP2 bool
6261
var tlsOpts []func(*tls.Config)
62+
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
63+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
6364
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
6465
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
6566
"Enable leader election for controller manager. "+
6667
"Enabling this will ensure there is only one active controller manager.")
67-
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
68-
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
6968
flag.BoolVar(&secureMetrics, "metrics-secure", true,
7069
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
7170
flag.BoolVar(&enableHTTP2, "enable-http2", false,
7271
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
73-
7472
opts := zap.Options{
7573
Development: true,
7674
}
@@ -93,6 +91,11 @@ func main() {
9391
if !enableHTTP2 {
9492
tlsOpts = append(tlsOpts, disableHTTP2)
9593
}
94+
95+
webhookServer := webhook.NewServer(webhook.Options{
96+
TLSOpts: tlsOpts,
97+
})
98+
9699
// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
97100
// More info:
98101
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
@@ -116,18 +119,19 @@ func main() {
116119

117120
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
118121
// you can uncomment the following lines to use the certificate managed by cert-manager.
122+
119123
// metricsServerOptions.CertDir = "/tmp/k8s-metrics-server/metrics-certs"
120124
// metricsServerOptions.CertName = "tls.crt"
121125
// metricsServerOptions.KeyName = "tls.key"
122-
123126
}
124127

125128
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
126129
Scheme: scheme,
127130
Metrics: metricsServerOptions,
131+
WebhookServer: webhookServer,
128132
HealthProbeBindAddress: probeAddr,
129133
LeaderElection: enableLeaderElection,
130-
LeaderElectionID: "ecaf1259.spectrocloud.labs",
134+
LeaderElectionID: "dee7189b.spectrocloud.labs",
131135
// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
132136
// when the Manager ends. This requires the binary to immediately end when the
133137
// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
@@ -153,7 +157,7 @@ func main() {
153157
setupLog.Error(err, "unable to create controller", "controller", "MaasValidator")
154158
os.Exit(1)
155159
}
156-
//+kubebuilder:scaffold:builder
160+
// +kubebuilder:scaffold:builder
157161

158162
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
159163
setupLog.Error(err, "unable to set up health check")

config/default/manager_config_patch.yaml

-10
This file was deleted.

config/default/metrics_service.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
labels:
5+
control-plane: controller-manager
6+
app.kubernetes.io/name: validator-plugin-maas
7+
app.kubernetes.io/managed-by: kustomize
8+
name: controller-manager-metrics-service
9+
namespace: system
10+
spec:
11+
ports:
12+
- name: https
13+
port: 8443
14+
protocol: TCP
15+
targetPort: 8443
16+
selector:
17+
control-plane: controller-manager

config/manager/manager.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ spec:
7272
- --leader-elect
7373
- --health-probe-bind-address=:8081
7474
image: controller:latest
75+
imagePullPolicy: IfNotPresent
7576
name: manager
7677
securityContext:
7778
allowPrivilegeEscalation: false
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# This NetworkPolicy allows ingress traffic
2+
# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those
3+
# namespaces are able to gathering data from the metrics endpoint.
4+
apiVersion: networking.k8s.io/v1
5+
kind: NetworkPolicy
6+
metadata:
7+
labels:
8+
app.kubernetes.io/name: validator-plugin-maas
9+
app.kubernetes.io/managed-by: kustomize
10+
name: allow-metrics-traffic
11+
namespace: system
12+
spec:
13+
podSelector:
14+
matchLabels:
15+
control-plane: controller-manager
16+
policyTypes:
17+
- Ingress
18+
ingress:
19+
# This allows ingress traffic from any namespace with the label metrics: enabled
20+
- from:
21+
- namespaceSelector:
22+
matchLabels:
23+
metrics: enabled # Only from namespaces with this label
24+
ports:
25+
- port: 8443
26+
protocol: TCP
+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
resources:
2+
- allow-metrics-traffic.yaml

config/samples/kustomization.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
## Append samples of your project ##
2+
resources:
3+
- validation_v1alpha1_maasvalidator.yaml
4+
# +kubebuilder:scaffold:manifestskustomizesamples

internal/controller/maasvalidator_controller.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ type MaasValidatorReconciler struct {
5050
Scheme *runtime.Scheme
5151
}
5252

53-
//+kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators,verbs=get;list;watch;create;update;patch;delete
54-
//+kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators/status,verbs=get;update;patch
55-
//+kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators/finalizers,verbs=update
53+
// +kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators,verbs=get;list;watch;create;update;patch;delete
54+
// +kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators/status,verbs=get;update;patch
55+
// +kubebuilder:rbac:groups=validation.spectrocloud.labs,resources=maasvalidators/finalizers,verbs=update
5656

5757
// Reconcile reconciles each rule found in each MaasValidator in the cluster and creates ValidationResults accordingly
5858
func (r *MaasValidatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

internal/controller/maasvalidator_controller_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
"k8s.io/apimachinery/pkg/types"
1515

1616
// . "github.com/onsi/gomega"
17-
//+kubebuilder:scaffold:imports
17+
// +kubebuilder:scaffold:imports
1818
"github.com/validator-labs/validator-plugin-maas/api/v1alpha1"
1919
"github.com/validator-labs/validator-plugin-maas/pkg/validate"
2020
vapi "github.com/validator-labs/validator/api/v1alpha1"

internal/controller/suite_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ import (
4545
v1alpha1 "github.com/validator-labs/validator-plugin-maas/api/v1alpha1"
4646
vapi "github.com/validator-labs/validator/api/v1alpha1"
4747
"github.com/validator-labs/validator/pkg/util"
48-
//+kubebuilder:scaffold:imports
48+
// +kubebuilder:scaffold:imports
4949
)
5050

5151
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -115,7 +115,7 @@ var _ = BeforeSuite(func() {
115115
err = vapi.AddToScheme(scheme.Scheme)
116116
Expect(err).NotTo(HaveOccurred())
117117

118-
//+kubebuilder:scaffold:scheme
118+
// +kubebuilder:scaffold:scheme
119119

120120
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
121121
Expect(err).NotTo(HaveOccurred())

tests/e2e/e2e_suite_test.go

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"os/exec"
23+
"testing"
24+
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
28+
"github.com/validator-labs/validator-plugin-maas/tests/utils"
29+
)
30+
31+
var (
32+
// Optional Environment Variables:
33+
// - PROMETHEUS_INSTALL_SKIP=true: Skips Prometheus Operator installation during test setup.
34+
// - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup.
35+
// These variables are useful if Prometheus or CertManager is already installed, avoiding
36+
// re-installation and conflicts.
37+
skipPrometheusInstall = os.Getenv("PROMETHEUS_INSTALL_SKIP") == "true"
38+
skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true"
39+
// isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster
40+
isPrometheusOperatorAlreadyInstalled = false
41+
// isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster
42+
isCertManagerAlreadyInstalled = false
43+
44+
// projectImage is the name of the image which will be build and loaded
45+
// with the code source changes to be tested.
46+
projectImage = "quay.io/validator-labs/validator-plugin-maas:latest"
47+
)
48+
49+
// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
50+
// temporary environment to validate project changes with the the purposed to be used in CI jobs.
51+
// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs
52+
// CertManager and Prometheus.
53+
func TestE2E(t *testing.T) {
54+
RegisterFailHandler(Fail)
55+
_, _ = fmt.Fprintf(GinkgoWriter, "Starting validator-plugin-maas integration test suite\n")
56+
RunSpecs(t, "e2e suite")
57+
}
58+
59+
var _ = BeforeSuite(func() {
60+
By("Ensure that Prometheus is enabled")
61+
_ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#")
62+
63+
By("generating files")
64+
cmd := exec.Command("make", "generate")
65+
_, err := utils.Run(cmd)
66+
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make generate")
67+
68+
By("generating manifests")
69+
cmd = exec.Command("make", "manifests")
70+
_, err = utils.Run(cmd)
71+
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make manifests")
72+
73+
By("building the manager(Operator) image")
74+
cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
75+
_, err = utils.Run(cmd)
76+
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")
77+
78+
// TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is
79+
// built and available before running the tests. Also, remove the following block.
80+
By("loading the manager(Operator) image on Kind")
81+
err = utils.LoadImageToKindClusterWithName(projectImage)
82+
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")
83+
84+
// The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
85+
// To prevent errors when tests run in environments with Prometheus or CertManager already installed,
86+
// we check for their presence before execution.
87+
// Setup Prometheus and CertManager before the suite if not skipped and if not already installed
88+
if !skipPrometheusInstall {
89+
By("checking if prometheus is installed already")
90+
isPrometheusOperatorAlreadyInstalled = utils.IsPrometheusCRDsInstalled()
91+
if !isPrometheusOperatorAlreadyInstalled {
92+
_, _ = fmt.Fprintf(GinkgoWriter, "Installing Prometheus Operator...\n")
93+
Expect(utils.InstallPrometheusOperator()).To(Succeed(), "Failed to install Prometheus Operator")
94+
} else {
95+
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: Prometheus Operator is already installed. Skipping installation...\n")
96+
}
97+
}
98+
if !skipCertManagerInstall {
99+
By("checking if cert manager is installed already")
100+
isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled()
101+
if !isCertManagerAlreadyInstalled {
102+
_, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n")
103+
Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager")
104+
} else {
105+
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n")
106+
}
107+
}
108+
})
109+
110+
var _ = AfterSuite(func() {
111+
// Teardown Prometheus and CertManager after the suite if not skipped and if they were not already installed
112+
if !skipPrometheusInstall && !isPrometheusOperatorAlreadyInstalled {
113+
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Prometheus Operator...\n")
114+
utils.UninstallPrometheusOperator()
115+
}
116+
if !skipCertManagerInstall && !isCertManagerAlreadyInstalled {
117+
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
118+
utils.UninstallCertManager()
119+
}
120+
})

0 commit comments

Comments
 (0)