Add validation for pod/service monitors for TargetAllocator and skip invalid ones (open-telemetry#2328)

* fix for adding validation

* removing unused references

* adding tests

* adding some changes

* cleaning up

* adding change log

* running goimports and adding return value check code for namespace informer

* fixing lint error

* fixing tests and comment

* adding permissions for e2e tests

* adding cluster roles instead of roles

* updaintg readme

* fixing comments

* adding contant to same block

* fixing lint errors

* running go import

* adding namespaces since that is required for informer

* adding extected warnings

* addressing comments

* adding test for namespace label update

* fixing goimports

* making namespaceselectores as labelselectors

Author: rashmichandrashekar

.chloggen/target-allocator-use-recommended-prometheus-operator-interfaces.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: target allocator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use recommended interfaces(resource selector) by the prometheus-operator for watching CRs.
+
+# One or more tracking issues related to the change
+issues: [2309]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: The target allocator now requires get/list/watch permissions for namespaces. Update your RBAC permissions for the attached role, if necessary.

apis/v1alpha1/collector_webhook.go

@@ -47,7 +47,7 @@ var (
 			Verbs:     []string{"*"},
 		}, {
 			APIGroups: []string{""},
-			Resources: []string{"nodes", "nodes/metrics", "services", "endpoints", "pods"},
+			Resources: []string{"nodes", "nodes/metrics", "services", "endpoints", "pods", "namespaces"},
 			Verbs:     []string{"get", "list", "watch"},
 		}, {
 			APIGroups: []string{""},

apis/v1alpha1/collector_webhook_test.go

@@ -563,6 +563,7 @@ func TestOTELColValidatingWebhook(t *testing.T) {
 				"missing the following rules for nodes/metrics: [get,list,watch]",
 				"missing the following rules for services: [get,list,watch]",
 				"missing the following rules for endpoints: [get,list,watch]",
+				"missing the following rules for namespaces: [get,list,watch]",
 				"missing the following rules for networking.k8s.io/ingresses: [get,list,watch]",
 				"missing the following rules for nodes: [get,list,watch]",
 				"missing the following rules for pods: [get,list,watch]",

cmd/otel-allocator/README.md

@@ -124,7 +124,7 @@ to collector instance pods by default.
 
 
 ### RBAC
-The ServiceAccount that the TargetAllocator runs as, has to have access to the CRs. A role like this will provide that
+The ServiceAccount that the TargetAllocator runs as, has to have access to the CRs and the namespaces to watch for the pod and service monitors. A role like this will provide that
 access.
 ```yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -139,6 +139,10 @@ rules:
   - podmonitors
   verbs:
   - '*'
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs: ["get", "list", "watch"]
 ```
 In addition, the TargetAllocator needs the same permissions as a Prometheus instance would to find the matching targets
 from the CR instances.

cmd/otel-allocator/config/config.go

@@ -44,17 +44,19 @@ const (
 )
 
 type Config struct {
-	ListenAddr             string                `yaml:"listen_addr,omitempty"`
-	KubeConfigFilePath     string                `yaml:"kube_config_file_path,omitempty"`
-	ClusterConfig          *rest.Config          `yaml:"-"`
-	RootLogger             logr.Logger           `yaml:"-"`
-	CollectorSelector      *metav1.LabelSelector `yaml:"collector_selector,omitempty"`
-	PromConfig             *promconfig.Config    `yaml:"config"`
-	AllocationStrategy     string                `yaml:"allocation_strategy,omitempty"`
-	FilterStrategy         string                `yaml:"filter_strategy,omitempty"`
-	PrometheusCR           PrometheusCRConfig    `yaml:"prometheus_cr,omitempty"`
-	PodMonitorSelector     map[string]string     `yaml:"pod_monitor_selector,omitempty"`
-	ServiceMonitorSelector map[string]string     `yaml:"service_monitor_selector,omitempty"`
+	ListenAddr                      string                `yaml:"listen_addr,omitempty"`
+	KubeConfigFilePath              string                `yaml:"kube_config_file_path,omitempty"`
+	ClusterConfig                   *rest.Config          `yaml:"-"`
+	RootLogger                      logr.Logger           `yaml:"-"`
+	CollectorSelector               *metav1.LabelSelector `yaml:"collector_selector,omitempty"`
+	PromConfig                      *promconfig.Config    `yaml:"config"`
+	AllocationStrategy              string                `yaml:"allocation_strategy,omitempty"`
+	FilterStrategy                  string                `yaml:"filter_strategy,omitempty"`
+	PrometheusCR                    PrometheusCRConfig    `yaml:"prometheus_cr,omitempty"`
+	PodMonitorSelector              map[string]string     `yaml:"pod_monitor_selector,omitempty"`
+	ServiceMonitorSelector          map[string]string     `yaml:"service_monitor_selector,omitempty"`
+	ServiceMonitorNamespaceSelector *metav1.LabelSelector `yaml:"service_monitor_namespace_selector,omitempty"`
+	PodMonitorNamespaceSelector     *metav1.LabelSelector `yaml:"pod_monitor_namespace_selector,omitempty"`
 }
 
 type PrometheusCRConfig struct {

cmd/otel-allocator/main.go

@@ -102,7 +102,7 @@ func main() {
 	defer close(interrupts)
 
 	if cfg.PrometheusCR.Enabled {
-		promWatcher, err = allocatorWatcher.NewPrometheusCRWatcher(setupLog.WithName("prometheus-cr-watcher"), *cfg)
+		promWatcher, err = allocatorWatcher.NewPrometheusCRWatcher(ctx, setupLog.WithName("prometheus-cr-watcher"), *cfg)
 		if err != nil {
 			setupLog.Error(err, "Can't start the prometheus watcher")
 			os.Exit(1)