apis: add CRD + informer for Telemetry & ExtensionService (openservicemesh#5184)

- Adds CRD for Telemetry and ExtensionService APIs

- Adds informer clients

- Adds an API to lookup the Telemetry config for a proxy

- Minor changes to the API definition

Part of openservicemesh#5136

Signed-off-by: Shashank Ram <[email protected]>

Author: shashankram

charts/osm/templates/osm-rbac.yaml

@@ -41,7 +41,7 @@ rules:
     resources: ["customresourcedefinitions"]
     verbs: ["get", "list", "watch", "create", "update", "patch"]
   - apiGroups: ["config.openservicemesh.io"]
-    resources: ["meshconfigs", "meshrootcertificates"]
+    resources: ["meshconfigs", "meshrootcertificates", "extensionservices"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
   - apiGroups: ["config.openservicemesh.io"]
     resources: ["meshrootcertificates/status"]
@@ -58,10 +58,10 @@ rules:
 
   # OSM's custom policy API
   - apiGroups: ["policy.openservicemesh.io"]
-    resources: ["egresses", "ingressbackends", "retries", "upstreamtrafficsettings"]
+    resources: ["egresses", "ingressbackends", "retries", "upstreamtrafficsettings", "telemetries"]
     verbs: ["list", "get", "watch"]
   - apiGroups: ["policy.openservicemesh.io"]
-    resources: ["ingressbackends/status", "upstreamtrafficsettings/status"]
+    resources: ["ingressbackends/status", "upstreamtrafficsettings/status", "telemetry/status"]
     verbs: ["update"]
 
   # Used for interacting with cert-manager CertificateRequest resources.

cmd/osm-bootstrap/crds/config_extensionservice.yaml

@@ -0,0 +1,63 @@
+# Custom Resource Definition (CRD) for OSM's ExtensionService specification.
+#
+# Copyright Open Service Mesh authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: extensionservices.config.openservicemesh.io
+  labels:
+    app.kubernetes.io/name : "openservicemesh.io"
+spec:
+  group: config.openservicemesh.io
+  scope: Namespaced
+  names:
+    kind: ExtensionService
+    listKind: ExtensionServiceList
+    shortNames:
+      - extsvc
+    singular: extensionservice
+    plural: extensionservices
+  conversion:
+    strategy: None
+  versions:
+    - name: v1alpha2
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              required:
+                - host
+                - port
+              properties:
+                host:
+                  description: Hostname of the service.
+                  type: string
+                  minLength: 1
+                port:
+                  description: Port of the service.
+                  type: integer
+                  minimum: 1
+                  maximum: 65535
+                protocol:
+                  description: Protocol of the service.
+                  type: string
+                connectTimeout:
+                  description: Timeout for connecting to the service.
+                  type: string

cmd/osm-bootstrap/crds/policy_telemetry.yaml

@@ -0,0 +1,101 @@
+# Custom Resource Definition (CRD) for OSM's Telemetry API.
+#
+# Copyright Open Service Mesh authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: telemetries.policy.openservicemesh.io
+  labels:
+    app.kubernetes.io/name : "openservicemesh.io"
+spec:
+  group: policy.openservicemesh.io
+  scope: Namespaced
+  names:
+    kind: Telemetry
+    listKind: TelemetryList
+    shortNames:
+      - telemetry
+    singular: telemetry
+    plural: telemetries
+  conversion:
+    strategy: None
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+      - description: Current status of the Telemetry policy.
+        jsonPath: .status.currentStatus
+        name: Status
+        type: string
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                selector:
+                  description: selector (optional) defines the pod label selector for pods the Telemetry
+                    configuration is applicable to. It selects pods with matching label keys
+                    and values. If not specified, the configuration applies to all pods
+                    in the Telemetry resource's namespace.
+                  type: object
+                  additionalProperties: true
+                accessLog:
+                  description: accessLog (optional) defines the Envoy access log configuration.
+                  type: object
+                  properties:
+                    format:
+                      description: format (optional) defines the Envoy access log format.
+                        The format can either be unstructured or structured (e.g. JSON).
+                        Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage#format-strings
+                        regarding how a format string can be specified.
+                      type: string
+                      minLength: 1
+                    openTelemetry:
+                      description: openTelemetry (optional) defines the OpenTelemetry configuration used to export the
+                        Envoy access logs to an OpenTelemetry collector.
+                      type: object
+                      required:
+                        - extensionService
+                      properties:
+                        extensionService:
+                          description: extensionService defines the reference to ExtensionService resource
+                            corresponding to the OpenTelemetry collector the access log should be exported to.
+                          type: object
+                          required:
+                            - namespace
+                            - name
+                          properties:
+                            namespace:
+                              description: Namespace of the ExtensionService resource.
+                              type: string
+                              minLength: 1
+                            name:
+                              description: Name of the ExtensionService resource.
+                              type: string
+                              minLength: 1
+                        attributes:
+                          description: attributes (optional) defines key-value pairs as additional metadata corresponding access log record.
+                          type: object
+                          additionalProperties: true
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+      subresources:
+        # status enables the status subresource
+        status: {}

pkg/apis/policy/v1alpha1/telemetry.go

@@ -45,7 +45,8 @@ type EnvoyAccessLogConfig struct {
 	// The format can either be unstructured or structured (e.g. JSON).
 	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage#format-strings
 	// regarding how a format string can be specified.
-	Format string `json:"format"`
+	// +optional
+	Format string `json:"format,omitempty"`
 
 	// OpenTelemetry defines the OpenTelemetry configuration used to export the
 	// Envoy access logs to an OpenTelemetry collector.
@@ -56,8 +57,8 @@ type EnvoyAccessLogConfig struct {
 // EnvoyAccessLogOpenTelemetryConfig defines the Envoy access log OpenTelemetry
 // configuration.
 type EnvoyAccessLogOpenTelemetryConfig struct {
-	// ExtensionService defines the references to ExtensionService resource
-	// corresponding to the OpenTelemetry collector.
+	// ExtensionService defines the referenence to ExtensionService resource
+	// corresponding to the OpenTelemetry collector the access log should be exported to.
 	ExtensionService ExtensionServiceRef `json:"extensionService"`
 
 	// Attributes defines key-value pairs as additional metadata corresponding access log record.

pkg/catalog/mock_catalog_generated.go

@@ -3,26 +3,26 @@ package k8s
 import (
 	"context"
 
+	smiAccess "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/access/v1alpha3"
+	smiSpecs "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/specs/v1alpha4"
+	smiSplit "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/split/v1alpha2"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 
-	smiAccess "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/access/v1alpha3"
-	smiSpecs "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/specs/v1alpha4"
-	smiSplit "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/split/v1alpha2"
-
 	configv1alpha2 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha2"
 	policyv1alpha1 "github.com/openservicemesh/osm/pkg/apis/policy/v1alpha1"
 	configv1alpha2Client "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned"
 	policyv1alpha1Client "github.com/openservicemesh/osm/pkg/gen/client/policy/clientset/versioned"
-	"github.com/openservicemesh/osm/pkg/models"
 
 	"github.com/openservicemesh/osm/pkg/constants"
 	"github.com/openservicemesh/osm/pkg/errcode"
 	"github.com/openservicemesh/osm/pkg/k8s/informers"
 	"github.com/openservicemesh/osm/pkg/messaging"
+	"github.com/openservicemesh/osm/pkg/models"
 )
 
 // NewClient returns a new kubernetes.Controller which means to provide access to locally-cached k8s resources
@@ -47,10 +47,12 @@ func NewClient(osmNamespace, meshConfigName string, informerCollection *informer
 		Endpoints:              c.initEndpointMonitor,
 		MeshConfig:             c.initMeshConfigMonitor,
 		MeshRootCertificate:    c.initMRCMonitor,
+		ExtensionService:       c.initExtensionServiceMonitor,
 		Egress:                 c.initEgressMonitor,
 		IngressBackend:         c.initIngressBackendMonitor,
 		Retry:                  c.initRetryMonitor,
 		UpstreamTrafficSetting: c.initUpstreamTrafficSettingMonitor,
+		Telemetry:              c.initTelemetryMonitor,
 		TrafficSplit:           c.initTrafficSplitMonitor,
 		HTTPRouteGroup:         c.initHTTPRouteGroupMonitor,
 		TCPRoute:               c.initTCPRouteMonitor,
@@ -60,8 +62,8 @@ func NewClient(osmNamespace, meshConfigName string, informerCollection *informer
 	// If specific informers are not selected to be initialized, initialize all informers
 	if len(selectInformers) == 0 {
 		selectInformers = []InformerKey{
-			Namespaces, Services, ServiceAccounts, Pods, Endpoints, MeshConfig, MeshRootCertificate,
-			Egress, IngressBackend, Retry, UpstreamTrafficSetting, TrafficSplit, HTTPRouteGroup, TCPRoute,
+			Namespaces, Services, ServiceAccounts, Pods, Endpoints, MeshConfig, MeshRootCertificate, ExtensionService,
+			Egress, IngressBackend, Retry, UpstreamTrafficSetting, Telemetry, TrafficSplit, HTTPRouteGroup, TCPRoute,
 			TrafficTarget}
 	}
 
@@ -87,6 +89,10 @@ func (c *Client) initMRCMonitor() {
 	c.informers.AddEventHandler(informers.InformerKeyMeshRootCertificate, GetEventHandlerFuncs(nil, c.msgBroker))
 }
 
+func (c *Client) initExtensionServiceMonitor() {
+	c.informers.AddEventHandler(informers.InformerKeyExtensionService, GetEventHandlerFuncs(nil, c.msgBroker))
+}
+
 func (c *Client) initEgressMonitor() {
 	c.informers.AddEventHandler(informers.InformerKeyEgress, GetEventHandlerFuncs(c.shouldObserve, c.msgBroker))
 }
@@ -103,6 +109,10 @@ func (c *Client) initUpstreamTrafficSettingMonitor() {
 	c.informers.AddEventHandler(informers.InformerKeyUpstreamTrafficSetting, GetEventHandlerFuncs(c.shouldObserve, c.msgBroker))
 }
 
+func (c *Client) initTelemetryMonitor() {
+	c.informers.AddEventHandler(informers.InformerKeyTelemetry, GetEventHandlerFuncs(c.shouldObserve, c.msgBroker))
+}
+
 // Function to filter K8s meta Objects by OSM's isMonitoredNamespace
 func (c *Client) shouldObserve(obj interface{}) bool {
 	object, ok := obj.(metav1.Object)
@@ -593,3 +603,52 @@ func (c *Client) ListTrafficTargets() []*smiAccess.TrafficTarget {
 	}
 	return trafficTargets
 }
+
+// GetTelemetryPolicy returns the Telemetry policy for the given proxy instance.
+// It returns the most specific match if multiple matching policies exist, in the following
+// order of preference: 1. selector match, 2. namespace match, 3. global match
+func (c *Client) GetTelemetryPolicy(proxy *models.Proxy) *policyv1alpha1.Telemetry {
+	pod, _ := c.GetPodForProxy(proxy)
+	if pod == nil {
+		return nil
+	}
+
+	var policy *policyv1alpha1.Telemetry
+
+	for _, resource := range c.informers.List(informers.InformerKeyTelemetry) {
+		t := resource.(*policyv1alpha1.Telemetry)
+
+		// If there is a global policy and a more specific policy hasn't been
+		// found yet, consider the global policy as a candidate
+		if policy == nil && t.Namespace == c.osmNamespace {
+			policy = t
+			continue
+		}
+
+		if !c.IsMonitoredNamespace(t.Namespace) {
+			continue
+		}
+
+		// If the policy matches the namespace of the proxy's pod,
+		// consider this policy to be a candidate, but continue
+		// to look for a more specific policy that matches the pod
+		// based on a selector
+		if t.Namespace == pod.Namespace {
+			policy = t
+		}
+
+		// Look for a more specific match based on pod selector on the Telemetry resource.
+		// If we find a Telemetry resource that matches the pod's selector, this is
+		// the best match for this proxy.
+		selector := t.Spec.Selector
+		if len(selector) == 0 {
+			continue
+		}
+		sel := labels.Set(selector).AsSelector()
+		if sel.Matches(labels.Set(pod.Labels)) {
+			return t
+		}
+	}
+
+	return policy
+}