From 61dffa009eeb23ce5d2a482422b8fd97e0238030 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 11 May 2021 11:33:54 -0700 Subject: [PATCH 01/15] Add namespace tagger to k8s-tagger --- processor/k8sprocessor/client_test.go | 34 +++-- processor/k8sprocessor/config.go | 11 +- processor/k8sprocessor/config_test.go | 2 +- processor/k8sprocessor/go.sum | 41 ++++++ processor/k8sprocessor/kube/client.go | 144 +++++++++++++++++-- processor/k8sprocessor/kube/client_test.go | 144 ++++++++++++++++++- processor/k8sprocessor/kube/fake_informer.go | 25 ++++ processor/k8sprocessor/kube/informer.go | 33 +++++ processor/k8sprocessor/kube/informer_test.go | 39 +++++ processor/k8sprocessor/kube/kube.go | 40 ++++-- processor/k8sprocessor/options.go | 18 ++- processor/k8sprocessor/options_test.go | 62 +++++++- processor/k8sprocessor/processor.go | 37 ++++- processor/k8sprocessor/processor_test.go | 2 +- processor/k8sprocessor/testdata/config.yaml | 7 + 15 files changed, 582 insertions(+), 57 deletions(-) diff --git a/processor/k8sprocessor/client_test.go b/processor/k8sprocessor/client_test.go index ecfd5b2f341ef..d6e6a0adc1cf4 100644 --- a/processor/k8sprocessor/client_test.go +++ b/processor/k8sprocessor/client_test.go @@ -27,12 +27,14 @@ import ( // fakeClient is used as a replacement for WatchClient in test cases. type fakeClient struct { - Pods map[kube.PodIdentifier]*kube.Pod - Rules kube.ExtractionRules - Filters kube.Filters - Associations []kube.Association - Informer cache.SharedInformer - StopCh chan struct{} + Pods map[kube.PodIdentifier]*kube.Pod + Rules kube.ExtractionRules + Filters kube.Filters + Associations []kube.Association + Informer cache.SharedInformer + NamespaceInformer cache.SharedInformer + Namespaces map[string]*kube.Namespace + StopCh chan struct{} } func selectors() (labels.Selector, fields.Selector) { @@ -41,17 +43,18 @@ func selectors() (labels.Selector, fields.Selector) { } // newFakeClient instantiates a new FakeClient object and satisfies the ClientProvider type -func newFakeClient(_ *zap.Logger, apiCfg k8sconfig.APIConfig, rules kube.ExtractionRules, filters kube.Filters, associations []kube.Association, _ kube.APIClientsetProvider, _ kube.InformerProvider) (kube.Client, error) { +func newFakeClient(_ *zap.Logger, apiCfg k8sconfig.APIConfig, rules kube.ExtractionRules, filters kube.Filters, associations []kube.Association, _ kube.APIClientsetProvider, _ kube.InformerProvider, _ kube.InformerProviderNamespace) (kube.Client, error) { cs := fake.NewSimpleClientset() ls, fs := selectors() return &fakeClient{ - Pods: map[kube.PodIdentifier]*kube.Pod{}, - Rules: rules, - Filters: filters, - Associations: associations, - Informer: kube.NewFakeInformer(cs, "", ls, fs), - StopCh: make(chan struct{}), + Pods: map[kube.PodIdentifier]*kube.Pod{}, + Rules: rules, + Filters: filters, + Associations: associations, + Informer: kube.NewFakeInformer(cs, "", ls, fs), + NamespaceInformer: kube.NewFakeInformer(cs, "", ls, fs), + StopCh: make(chan struct{}), }, nil } @@ -62,6 +65,11 @@ func (f *fakeClient) GetPod(identifier kube.PodIdentifier) (*kube.Pod, bool) { return p, ok } +func (f *fakeClient) GetNamespace(namespace string) (*kube.Namespace, bool) { + ns, ok := f.Namespaces[namespace] + return ns, ok +} + // Start is a noop for FakeClient. func (f *fakeClient) Start() { if f.Informer != nil { diff --git a/processor/k8sprocessor/config.go b/processor/k8sprocessor/config.go index 09817fa67c313..120928e0532c4 100644 --- a/processor/k8sprocessor/config.go +++ b/processor/k8sprocessor/config.go @@ -52,12 +52,16 @@ func (cfg *Config) Validate() error { // ExtractConfig section allows specifying extraction rules to extract // data from k8s pod specs. type ExtractConfig struct { - // Metadata allows to extract pod metadata from a list of metadata fields. + // Metadata allows to extract pod/namespace metadata from a list of metadata fields. // The field accepts a list of strings. // // Metadata fields supported right now are, - // k8s.namespace.name, k8s.pod.name, k8s.pod.uid, k8s.deployment.name, k8s.cluster.name, + // k8s.pod.name, k8s.pod.uid, k8s.deployment.name, k8s.cluster.name, // k8s.node.name and k8s.pod.start_time + // + // Metadata fields supported for namespaces right now are, + // k8s.namespace.name, k8s.namespace.uid and k8s.namespace.start_time + // // Specifying anything other than these values will result in an error. // By default all of the fields are extracted and added to spans and metrics. Metadata []string `mapstructure:"metadata"` @@ -113,6 +117,9 @@ type FieldExtractConfig struct { TagName string `mapstructure:"tag_name"` Key string `mapstructure:"key"` Regex string `mapstructure:"regex"` + // From represents the source of the labels/annotations. + // Allowed values are "pod" and "namespace". + From string `mapstructure:"from"` } // FilterConfig section allows specifying filters to filter diff --git a/processor/k8sprocessor/config_test.go b/processor/k8sprocessor/config_test.go index 0451ad61f3ad8..9cdcf8d04b9a0 100644 --- a/processor/k8sprocessor/config_test.go +++ b/processor/k8sprocessor/config_test.go @@ -57,7 +57,7 @@ func TestLoadConfig(t *testing.T) { APIConfig: k8sconfig.APIConfig{AuthType: k8sconfig.AuthTypeKubeConfig}, Passthrough: false, Extract: ExtractConfig{ - Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time"}, + Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time", "k8s.namespace.uid", "k8s.namespace.start_time"}, Annotations: []FieldExtractConfig{ {TagName: "a1", Key: "annotation-one"}, {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)"}, diff --git a/processor/k8sprocessor/go.sum b/processor/k8sprocessor/go.sum index d3db2d833f608..1748e823eb57c 100644 --- a/processor/k8sprocessor/go.sum +++ b/processor/k8sprocessor/go.sum @@ -45,6 +45,7 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go v52.5.0+incompatible h1:/NLBWHCnIHtZyLPc1P7WIqi4Te4CC23kIQyK3Ep/7lA= github.com/Azure/azure-sdk-for-go v52.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -56,6 +57,7 @@ github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZ github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= @@ -74,6 +76,7 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= +github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= @@ -87,9 +90,11 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/sarama v1.29.0 h1:ARid8o8oieau9XrHI55f/L3EoRAhm9px6sonbD7yuUE= github.com/Shopify/sarama v1.29.0/go.mod h1:2QpgD79wpdAESqNQMxNc0KYMkycd4slxGdV3TWSVqrU= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -184,6 +189,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -201,6 +207,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.58.0 h1:Iy8ULTvgCAxH8dlxZ54qRYpm5uTEb2deUqijywLH7Lo= github.com/digitalocean/godo v1.58.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -239,14 +246,17 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.7.3/go.mod h1:V1d2J5pfxYH6EjBAgSK7YNXcXlTWxUHdE1sVDXkjnig= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -360,6 +370,7 @@ github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= @@ -439,6 +450,7 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -492,13 +504,16 @@ github.com/gophercloud/gophercloud v0.16.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -506,6 +521,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -521,7 +537,9 @@ github.com/hashicorp/consul/api v1.8.1 h1:BOEQaMWoGMhmQ29fC26bi0qb7/rId9JzZP2V0X github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.7.0 h1:H6R9d008jDcHPQPAqPNuydAshJ4v5/8URdFnUvK/+sc= github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= @@ -533,8 +551,10 @@ github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.4.0/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -542,6 +562,7 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -560,6 +581,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= @@ -575,6 +597,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.4/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -594,6 +617,7 @@ github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= @@ -604,6 +628,7 @@ github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyX github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -705,6 +730,7 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -719,6 +745,7 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -727,6 +754,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -749,6 +777,7 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= @@ -791,6 +820,7 @@ github.com/openzipkin/zipkin-go v0.2.5 h1:UwtQQx2pyPIgWYHRg+epgdx1/HnBQTgN3/oIYE github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -897,6 +927,7 @@ github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44 h1:3egqo0Vut6daANFm7tOXdNAa8v5/uLU+sgCJrc88Meo= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec v0.0.0-20200203094520-d13bb6d2420c/go.mod h1:gp0gaHj0WlmPh9BdsTmo1aq6C27yIPWdxCKGFGdVKBE= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -933,6 +964,7 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -954,6 +986,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -975,6 +1008,7 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -990,6 +1024,7 @@ github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c h1:Wm21TPasVdeOUTg1m/uNkRdMuvI+jIeYfTIwq98Z2V0= github.com/xdg-go/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:FV1RpvYFmF8wnKtr3ArzkC0b+tAySCbw8eP7QSIvLKM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/scram v1.0.3 h1:nTadYh2Fs4BK2xdldEa2g5bbaZp0/+1nJMMPtPxS/to= github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= @@ -1034,6 +1069,7 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -1097,6 +1133,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1107,6 +1144,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1189,6 +1227,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1361,6 +1400,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1537,6 +1577,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index 537600e25e951..405941c38db72 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -36,14 +36,15 @@ import ( // WatchClient is the main interface provided by this package to a kubernetes cluster. type WatchClient struct { - m sync.RWMutex - deleteMut sync.Mutex - logger *zap.Logger - kc kubernetes.Interface - informer cache.SharedInformer - deploymentRegex *regexp.Regexp - deleteQueue []deleteRequest - stopCh chan struct{} + m sync.RWMutex + deleteMut sync.Mutex + logger *zap.Logger + kc kubernetes.Interface + informer cache.SharedInformer + namespaceInformer cache.SharedInformer + deploymentRegex *regexp.Regexp + deleteQueue []deleteRequest + stopCh chan struct{} // A map containing Pod related data, used to associate them with resources. // Key can be either an IP address or Pod UID @@ -51,6 +52,10 @@ type WatchClient struct { Rules ExtractionRules Filters Filters Associations []Association + + // A map containing Namespace related data, used to associate them with resources. + // Key is namespace name + Namespaces map[string]*Namespace } // Extract deployment name from the pod name. Pod name is created using @@ -58,7 +63,7 @@ type WatchClient struct { var dRegex = regexp.MustCompile(`^(.*)-[0-9a-zA-Z]*-[0-9a-zA-Z]*$`) // New initializes a new k8s Client. -func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, filters Filters, associations []Association, newClientSet APIClientsetProvider, newInformer InformerProvider) (Client, error) { +func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, filters Filters, associations []Association, newClientSet APIClientsetProvider, newInformer InformerProvider, newNamespaceInformer InformerProviderNamespace) (Client, error) { c := &WatchClient{ logger: logger, Rules: rules, @@ -70,6 +75,7 @@ func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, go c.deleteLoop(time.Second*30, defaultPodDeleteGracePeriod) c.Pods = map[PodIdentifier]*Pod{} + c.Namespaces = map[string]*Namespace{} if newClientSet == nil { newClientSet = k8sconfig.MakeClient } @@ -93,7 +99,12 @@ func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, newInformer = newSharedInformer } + if newNamespaceInformer == nil { + newNamespaceInformer = newNamespaceSharedInformer + } + c.informer = newInformer(c.kc, c.Filters.Namespace, labelSelector, fieldSelector) + c.namespaceInformer = newNamespaceInformer(c.kc) return c, err } @@ -104,7 +115,13 @@ func (c *WatchClient) Start() { UpdateFunc: c.handlePodUpdate, DeleteFunc: c.handlePodDelete, }) - c.informer.Run(c.stopCh) + go c.informer.Run(c.stopCh) + c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.handleNamespaceAdd, + UpdateFunc: c.handleNamespaceUpdate, + DeleteFunc: c.handleNamespaceDelete, + }) + c.namespaceInformer.Run(c.stopCh) } // Stop signals the the k8s watcher/informer to stop watching for new events. @@ -146,6 +163,30 @@ func (c *WatchClient) handlePodDelete(obj interface{}) { observability.RecordPodTableSize(int64(podTableSize)) } +func (c *WatchClient) handleNamespaceAdd(obj interface{}) { + if namespace, ok := obj.(*api_v1.Namespace); ok { + c.addOrUpdateNamespace(namespace) + } else { + c.logger.Error("object received was not of type api_v1.Namespace", zap.Any("received", obj)) + } +} + +func (c *WatchClient) handleNamespaceUpdate(old, new interface{}) { + if namespace, ok := new.(*api_v1.Namespace); ok { + c.addOrUpdateNamespace(namespace) + } else { + c.logger.Error("object received was not of type api_v1.Namespace", zap.Any("received", new)) + } +} + +func (c *WatchClient) handleNamespaceDelete(obj interface{}) { + if namespace, ok := obj.(*api_v1.Namespace); ok { + c.forgetNamespace(namespace) + } else { + c.logger.Error("object received was not of type api_v1.Namespace", zap.Any("received", obj)) + } +} + func (c *WatchClient) deleteLoop(interval time.Duration, gracePeriod time.Duration) { // This loop runs after N seconds and deletes pods from cache. // It iterates over the delete queue and deletes all that aren't @@ -201,6 +242,17 @@ func (c *WatchClient) GetPod(identifier PodIdentifier) (*Pod, bool) { return nil, false } +// GetNamespace takes a namespace and returns the namespace object the namespace is associated with. +func (c *WatchClient) GetNamespace(namespace string) (*Namespace, bool) { + c.m.RLock() + ns, ok := c.Namespaces[namespace] + c.m.RUnlock() + if ok { + return ns, ok + } + return nil, false +} + func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { tags := map[string]string{} if c.Rules.PodName { @@ -243,14 +295,51 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Labels { - if v, ok := pod.Labels[r.Key]; ok { - tags[r.Name] = c.extractField(v, r) + if r.From == "pod" { + if v, ok := pod.Labels[r.Key]; ok { + tags[r.Name] = c.extractField(v, r) + } + } + } + + for _, r := range c.Rules.Annotations { + if r.From == "pod" { + if v, ok := pod.Annotations[r.Key]; ok { + tags[r.Name] = c.extractField(v, r) + } + } + } + return tags +} + +func (c *WatchClient) extractNamespaceAttributes(namespace *api_v1.Namespace) map[string]string { + tags := map[string]string{} + + if c.Rules.NamespaceUID { + uid := namespace.GetUID() + tags[tagNamespaceUID] = string(uid) + } + + if c.Rules.NamespaceStartTime { + ts := namespace.GetCreationTimestamp() + if !ts.IsZero() { + tags[tagNamespaceStartTime] = ts.String() + } + } + + for _, r := range c.Rules.Labels { + if r.From == "namespace" { + if v, ok := namespace.Labels[r.Key]; ok { + tags[r.Name] = c.extractField(v, r) + } } } for _, r := range c.Rules.Annotations { - if v, ok := pod.Annotations[r.Key]; ok { - tags[r.Name] = c.extractField(v, r) + if r.From == "namespace" { + if v, ok := namespace.Annotations[r.Key]; ok { + tags[r.Name] = c.extractField(v, r) + } } } return tags @@ -273,6 +362,7 @@ func (c *WatchClient) extractField(v string, r FieldExtractionRule) string { func (c *WatchClient) addOrUpdatePod(pod *api_v1.Pod) { newPod := &Pod{ Name: pod.Name, + Namespace: pod.GetNamespace(), Address: pod.Status.PodIP, PodUID: string(pod.UID), StartTime: pod.Status.StartTime, @@ -386,3 +476,29 @@ func selectorsFromFilters(filters Filters) (labels.Selector, fields.Selector, er } return labelSelector, fields.AndSelectors(selectors...), nil } + +func (c *WatchClient) addOrUpdateNamespace(namespace *api_v1.Namespace) { + newNamespace := &Namespace{ + Name: namespace.Name, + NamespaceUID: string(namespace.UID), + StartTime: namespace.GetCreationTimestamp(), + } + + newNamespace.Attributes = c.extractNamespaceAttributes(namespace) + + c.m.Lock() + defer c.m.Unlock() + + if namespace.Name != "" { + c.Namespaces[namespace.Name] = newNamespace + } +} + +func (c *WatchClient) forgetNamespace(namespace *api_v1.Namespace) { + c.m.Lock() + defer c.m.Unlock() + + if ns, ok := c.Namespaces[namespace.Name]; ok { + delete(c.Namespaces, ns.Name) + } +} diff --git a/processor/k8sprocessor/kube/client_test.go b/processor/k8sprocessor/kube/client_test.go index c9cb322583f78..89af37e0a701c 100644 --- a/processor/k8sprocessor/kube/client_test.go +++ b/processor/k8sprocessor/kube/client_test.go @@ -86,13 +86,38 @@ func podAddAndUpdateTest(t *testing.T, c *WatchClient, handler func(obj interfac } +func namespaceAddAndUpdateTest(t *testing.T, c *WatchClient, handler func(obj interface{})) { + assert.Equal(t, len(c.Namespaces), 0) + + namespace := &api_v1.Namespace{} + handler(namespace) + assert.Equal(t, len(c.Namespaces), 0) + + namespace = &api_v1.Namespace{} + namespace.Name = "namespaceA" + handler(namespace) + assert.Equal(t, len(c.Namespaces), 1) + got := c.Namespaces["namespaceA"] + assert.Equal(t, got.Name, "namespaceA") + assert.Equal(t, got.NamespaceUID, "") + + namespace = &api_v1.Namespace{} + namespace.Name = "namespaceB" + namespace.UID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + handler(namespace) + assert.Equal(t, len(c.Namespaces), 2) + got = c.Namespaces["namespaceB"] + assert.Equal(t, got.Name, "namespaceB") + assert.Equal(t, got.NamespaceUID, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") +} + func TestDefaultClientset(t *testing.T) { - c, err := New(zap.NewNop(), k8sconfig.APIConfig{}, ExtractionRules{}, Filters{}, []Association{}, nil, nil) + c, err := New(zap.NewNop(), k8sconfig.APIConfig{}, ExtractionRules{}, Filters{}, []Association{}, nil, nil, nil) assert.Error(t, err) assert.Equal(t, "invalid authType for kubernetes: ", err.Error()) assert.Nil(t, c) - c, err = New(zap.NewNop(), k8sconfig.APIConfig{}, ExtractionRules{}, Filters{}, []Association{}, newFakeAPIClientset, nil) + c, err = New(zap.NewNop(), k8sconfig.APIConfig{}, ExtractionRules{}, Filters{}, []Association{}, newFakeAPIClientset, nil, nil) assert.NoError(t, err) assert.NotNil(t, c) } @@ -106,6 +131,7 @@ func TestBadFilters(t *testing.T) { []Association{}, newFakeAPIClientset, NewFakeInformer, + NewFakeNamespaceInformer, ) assert.Error(t, err) assert.Nil(t, c) @@ -141,7 +167,7 @@ func TestConstructorErrors(t *testing.T) { gotAPIConfig = c return nil, fmt.Errorf("error creating k8s client") } - c, err := New(zap.NewNop(), apiCfg, er, ff, []Association{}, clientProvider, NewFakeInformer) + c, err := New(zap.NewNop(), apiCfg, er, ff, []Association{}, clientProvider, NewFakeInformer, NewFakeNamespaceInformer) assert.Nil(t, c) assert.Error(t, err) assert.Equal(t, err.Error(), "error creating k8s client") @@ -154,6 +180,11 @@ func TestPodAdd(t *testing.T) { podAddAndUpdateTest(t, c, c.handlePodAdd) } +func TestNamespaceAdd(t *testing.T) { + c, _ := newTestClient(t) + namespaceAddAndUpdateTest(t, c, c.handleNamespaceAdd) +} + func TestPodHostNetwork(t *testing.T) { c, _ := newTestClient(t) assert.Equal(t, 0, len(c.Pods)) @@ -205,6 +236,14 @@ func TestPodUpdate(t *testing.T) { }) } +func TestNamespaceUpdate(t *testing.T) { + c, _ := newTestClient(t) + namespaceAddAndUpdateTest(t, c, func(obj interface{}) { + // first argument (old namespace) is not used right now + c.handleNamespaceUpdate(&api_v1.Namespace{}, obj) + }) +} + func TestPodDelete(t *testing.T) { c, _ := newTestClient(t) podAddAndUpdateTest(t, c, c.handlePodAdd) @@ -266,6 +305,24 @@ func TestPodDelete(t *testing.T) { assert.False(t, deleteRequest.ts.After(time.Now())) } +func TestNamespaceDelete(t *testing.T) { + c, _ := newTestClient(t) + namespaceAddAndUpdateTest(t, c, c.handleNamespaceAdd) + assert.Equal(t, len(c.Namespaces), 2) + assert.Equal(t, c.Namespaces["namespaceA"].Name, "namespaceA") + + // delete empty namespace + c.handleNamespaceDelete(&api_v1.Namespace{}) + + // delete non-existent namespace + namespace := &api_v1.Namespace{} + namespace.Name = "namespaceC" + c.handleNamespaceDelete(namespace) + assert.Equal(t, len(c.Namespaces), 2) + got := c.Namespaces["namespaceA"] + assert.Equal(t, got.Name, "namespaceA") +} + func TestDeleteQueue(t *testing.T) { c, _ := newTestClient(t) podAddAndUpdateTest(t, c, c.handlePodAdd) @@ -401,7 +458,7 @@ func TestExtractionRules(t *testing.T) { "k8s.node.name": "node1", "k8s.pod.name": "auth-service-abc12-xyz3", "k8s.pod.uid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", - "k8s.pod.startTime": pod.GetCreationTimestamp().String(), + "k8s.pod.start_time": pod.GetCreationTimestamp().String(), }, }, { name: "labels", @@ -409,15 +466,18 @@ func TestExtractionRules(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", + From: "pod", }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", + From: "pod", }, { Name: "l2", Key: "label2", Regex: regexp.MustCompile(`k5=(?P[^\s]+)`), + From: "pod", }, }, }, @@ -445,6 +505,80 @@ func TestExtractionRules(t *testing.T) { } } +func TestNamespaceExtractionRules(t *testing.T) { + c, _ := newTestClientWithRulesAndFilters(t, ExtractionRules{}, Filters{}) + + namespace := &api_v1.Namespace{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "auth-service-namespace-abc12-xyz3", + UID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + CreationTimestamp: meta_v1.Now(), + Labels: map[string]string{ + "label1": "lv1", + }, + Annotations: map[string]string{ + "annotation1": "av1", + }, + }, + } + + testCases := []struct { + name string + rules ExtractionRules + attributes map[string]string + }{{ + name: "no-rules", + rules: ExtractionRules{}, + attributes: nil, + }, { + name: "metadata", + rules: ExtractionRules{ + NamespaceUID: true, + NamespaceStartTime: true, + }, + attributes: map[string]string{ + "k8s.namespace.uid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "k8s.namespace.start_time": namespace.GetCreationTimestamp().String(), + }, + }, { + name: "labels", + rules: ExtractionRules{ + Annotations: []FieldExtractionRule{{ + Name: "a1", + Key: "annotation1", + From: "namespace", + }, + }, + Labels: []FieldExtractionRule{{ + Name: "l1", + Key: "label1", + From: "namespace", + }, + }, + }, + attributes: map[string]string{ + "l1": "lv1", + "a1": "av1", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c.Rules = tc.rules + c.handleNamespaceAdd(namespace) + p, ok := c.GetNamespace(namespace.Name) + require.True(t, ok) + + assert.Equal(t, len(tc.attributes), len(p.Attributes)) + for k, v := range tc.attributes { + got, ok := p.Attributes[k] + assert.True(t, ok) + assert.Equal(t, v, got) + } + }) + } +} + func TestFilters(t *testing.T) { testCases := []struct { name string @@ -675,7 +809,7 @@ func Test_selectorsFromFilters(t *testing.T) { func newTestClientWithRulesAndFilters(t *testing.T, e ExtractionRules, f Filters) (*WatchClient, *observer.ObservedLogs) { observedLogger, logs := observer.New(zapcore.WarnLevel) logger := zap.New(observedLogger) - c, err := New(logger, k8sconfig.APIConfig{}, e, f, []Association{}, newFakeAPIClientset, NewFakeInformer) + c, err := New(logger, k8sconfig.APIConfig{}, e, f, []Association{}, newFakeAPIClientset, NewFakeInformer, NewFakeNamespaceInformer) require.NoError(t, err) return c.(*WatchClient), logs } diff --git a/processor/k8sprocessor/kube/fake_informer.go b/processor/k8sprocessor/kube/fake_informer.go index 0a5c03fdc3225..bdf0d97a3dad2 100644 --- a/processor/k8sprocessor/kube/fake_informer.go +++ b/processor/k8sprocessor/kube/fake_informer.go @@ -59,6 +59,31 @@ func (f *FakeInformer) GetController() cache.Controller { return f.FakeController } +type FakeNamespaceInformer struct { + *FakeController +} + +func NewFakeNamespaceInformer( + _ kubernetes.Interface, +) cache.SharedInformer { + return &FakeInformer{ + FakeController: &FakeController{}, + } +} + +func (f *FakeNamespaceInformer) AddEventHandler(handler cache.ResourceEventHandler) {} + +func (f *FakeNamespaceInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, period time.Duration) { +} + +func (f *FakeNamespaceInformer) GetStore() cache.Store { + return cache.NewStore(func(obj interface{}) (string, error) { return "", nil }) +} + +func (f *FakeNamespaceInformer) GetController() cache.Controller { + return f.FakeController +} + type FakeController struct { sync.Mutex stopped bool diff --git a/processor/k8sprocessor/kube/informer.go b/processor/k8sprocessor/kube/informer.go index 1ecfcd652653b..8352d65a65f44 100644 --- a/processor/k8sprocessor/kube/informer.go +++ b/processor/k8sprocessor/kube/informer.go @@ -36,6 +36,12 @@ type InformerProvider func( fieldSelector fields.Selector, ) cache.SharedInformer +// InformerProviderNamespace defines a function type that returns a new SharedInformer. It is used to +// allow passing custom shared informers to the watch client for fetching namespace objects. +type InformerProviderNamespace func( + client kubernetes.Interface, +) cache.SharedInformer + func newSharedInformer( client kubernetes.Interface, namespace string, @@ -69,3 +75,30 @@ func informerWatchFuncWithSelectors(client kubernetes.Interface, namespace strin return client.CoreV1().Pods(namespace).Watch(context.Background(), opts) } } + +func newNamespaceSharedInformer( + client kubernetes.Interface, +) cache.SharedInformer { + informer := cache.NewSharedInformer( + &cache.ListWatch{ + ListFunc: namespaceInformerListFunc(client), + WatchFunc: namespaceInformerWatchFunc(client), + }, + &api_v1.Namespace{}, + watchSyncPeriod, + ) + return informer +} + +func namespaceInformerListFunc(client kubernetes.Interface) cache.ListFunc { + return func(opts metav1.ListOptions) (runtime.Object, error) { + return client.CoreV1().Namespaces().List(context.Background(), opts) + } + +} + +func namespaceInformerWatchFunc(client kubernetes.Interface) cache.WatchFunc { + return func(opts metav1.ListOptions) (watch.Interface, error) { + return client.CoreV1().Namespaces().Watch(context.Background(), opts) + } +} diff --git a/processor/k8sprocessor/kube/informer_test.go b/processor/k8sprocessor/kube/informer_test.go index d4272d919b9a0..7920ca3c21ad3 100644 --- a/processor/k8sprocessor/kube/informer_test.go +++ b/processor/k8sprocessor/kube/informer_test.go @@ -37,6 +37,13 @@ func Test_newSharedInformer(t *testing.T) { assert.NotNil(t, informer) } +func Test_newSharedNamespaceInformer(t *testing.T) { + client, err := newFakeAPIClientset(k8sconfig.APIConfig{}) + require.NoError(t, err) + informer := newNamespaceSharedInformer(client) + assert.NotNil(t, informer) +} + func Test_informerListFuncWithSelectors(t *testing.T) { ls, fs, err := selectorsFromFilters(Filters{ Fields: []FieldFilter{ @@ -64,6 +71,16 @@ func Test_informerListFuncWithSelectors(t *testing.T) { assert.NotNil(t, obj) } +func Test_namespaceInformerListFunc(t *testing.T) { + c, err := newFakeAPIClientset(k8sconfig.APIConfig{}) + assert.NoError(t, err) + listFunc := namespaceInformerListFunc(c) + opts := metav1.ListOptions{} + obj, err := listFunc(opts) + assert.NoError(t, err) + assert.NotNil(t, obj) +} + func Test_informerWatchFuncWithSelectors(t *testing.T) { ls, fs, err := selectorsFromFilters(Filters{ Fields: []FieldFilter{ @@ -91,6 +108,16 @@ func Test_informerWatchFuncWithSelectors(t *testing.T) { assert.NotNil(t, obj) } +func Test_namespaceInformerWatchFunc(t *testing.T) { + c, err := newFakeAPIClientset(k8sconfig.APIConfig{}) + assert.NoError(t, err) + watchFunc := namespaceInformerWatchFunc(c) + opts := metav1.ListOptions{} + obj, err := watchFunc(opts) + assert.NoError(t, err) + assert.NotNil(t, obj) +} + func Test_fakeInformer(t *testing.T) { // nothing real to test here. just to make coverage happy c, err := newFakeAPIClientset(k8sconfig.APIConfig{}) @@ -102,3 +129,15 @@ func Test_fakeInformer(t *testing.T) { store := i.GetStore() store.Add(api_v1.Pod{}) } + +func Test_fakeNamespaceInformer(t *testing.T) { + // nothing real to test here. just to make coverage happy + c, err := newFakeAPIClientset(k8sconfig.APIConfig{}) + assert.NoError(t, err) + i := NewFakeNamespaceInformer(c) + i.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{}, time.Second) + i.HasSynced() + i.LastSyncResourceVersion() + store := i.GetStore() + store.Add(api_v1.Namespace{}) +} diff --git a/processor/k8sprocessor/kube/kube.go b/processor/k8sprocessor/kube/kube.go index e2d3bef50597d..0404895697769 100644 --- a/processor/k8sprocessor/kube/kube.go +++ b/processor/k8sprocessor/kube/kube.go @@ -30,8 +30,10 @@ const ( podNodeField = "spec.nodeName" ignoreAnnotation string = "opentelemetry.io/k8s-processor/ignore" - tagNodeName = "k8s.node.name" - tagStartTime = "k8s.pod.startTime" + tagNodeName = "k8s.node.name" + tagStartTime = "k8s.pod.start_time" + tagNamespaceStartTime = "k8s.namespace.start_time" + tagNamespaceUID = "k8s.namespace.uid" ) // PodIdentifier is a custom type to represent IP Address or Pod UID @@ -50,12 +52,13 @@ var ( // Client defines the main interface that allows querying pods by metadata. type Client interface { GetPod(PodIdentifier) (*Pod, bool) + GetNamespace(string) (*Namespace, bool) Start() Stop() } // ClientProvider defines a func type that returns a new Client. -type ClientProvider func(*zap.Logger, k8sconfig.APIConfig, ExtractionRules, Filters, []Association, APIClientsetProvider, InformerProvider) (Client, error) +type ClientProvider func(*zap.Logger, k8sconfig.APIConfig, ExtractionRules, Filters, []Association, APIClientsetProvider, InformerProvider, InformerProviderNamespace) (Client, error) // APIClientsetProvider defines a func type that initializes and return a new kubernetes // Clientset object. @@ -69,10 +72,20 @@ type Pod struct { Attributes map[string]string StartTime *metav1.Time Ignore bool + Namespace string DeletedAt time.Time } +// Namespace represents a kubernetes namespace. +type Namespace struct { + Name string + NamespaceUID string + Attributes map[string]string + StartTime metav1.Time + DeletedAt time.Time +} + type deleteRequest struct { // id is identifier (IP address or Pod UID) of pod to remove from pods map id PodIdentifier @@ -108,13 +121,15 @@ type FieldFilter struct { // ExtractionRules is used to specify the information that needs to be extracted // from pods and added to the spans as tags. type ExtractionRules struct { - Deployment bool - Namespace bool - PodName bool - PodUID bool - Node bool - Cluster bool - StartTime bool + Deployment bool + Namespace bool + PodName bool + PodUID bool + Node bool + Cluster bool + StartTime bool + NamespaceUID bool + NamespaceStartTime bool Annotations []FieldExtractionRule Labels []FieldExtractionRule @@ -130,6 +145,11 @@ type FieldExtractionRule struct { // Regex is a regular expression used to extract a sub-part of a field value. // Full value is extracted when no regexp is provided. Regex *regexp.Regexp + // From determines the kubernetes object the field should be retrieved from. + // Currently only two values are supported, + // - pod + // - namespace + From string } // Associations represent a list of rules for Pod metadata associations with resources diff --git a/processor/k8sprocessor/options.go b/processor/k8sprocessor/options.go index a29d01cc0b33f..e37aecd9f30ec 100644 --- a/processor/k8sprocessor/options.go +++ b/processor/k8sprocessor/options.go @@ -40,7 +40,9 @@ const ( metadataCluster = "cluster" metadataNode = "node" // Will be removed when new fields get merged to https://github.com/open-telemetry/opentelemetry-collector/blob/main/translator/conventions/opentelemetry.go - metadataPodStartTime = "k8s.pod.start_time" + metadataPodStartTime = "k8s.pod.start_time" + metadataNamespaceUID = "k8s.namespace.uid" + metadataNamespaceStartTime = "k8s.namespace.start_time" ) // Option represents a configuration option that can be passes. @@ -78,6 +80,8 @@ func WithExtractMetadata(fields ...string) Option { conventions.AttributeK8sDeployment, conventions.AttributeK8sCluster, conventions.AttributeK8sNodeName, + metadataNamespaceUID, + metadataNamespaceStartTime, } } for _, field := range fields { @@ -99,6 +103,10 @@ func WithExtractMetadata(fields ...string) Option { p.rules.Cluster = true case metadataNode, conventions.AttributeK8sNodeName: p.rules.Node = true + case metadataNamespaceUID: + p.rules.NamespaceUID = true + case metadataNamespaceStartTime: + p.rules.NamespaceStartTime = true default: return fmt.Errorf("\"%s\" is not a supported metadata field", field) } @@ -136,7 +144,11 @@ func extractFieldRules(fieldType string, fields ...FieldExtractConfig) ([]kube.F for _, a := range fields { name := a.TagName if name == "" { - name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) + if a.From == "pod" { + name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) + } else if a.From == "namespace" { + name = fmt.Sprintf("k8s.namespace.%s.%s", fieldType, a.Key) + } } var r *regexp.Regexp @@ -153,7 +165,7 @@ func extractFieldRules(fieldType string, fields ...FieldExtractConfig) ([]kube.F } rules = append(rules, kube.FieldExtractionRule{ - Name: name, Key: a.Key, Regex: r, + Name: name, Key: a.Key, Regex: r, From: a.From, }) } return rules, nil diff --git a/processor/k8sprocessor/options_test.go b/processor/k8sprocessor/options_test.go index 53770f14ae725..2b041e254b881 100644 --- a/processor/k8sprocessor/options_test.go +++ b/processor/k8sprocessor/options_test.go @@ -90,6 +90,7 @@ func TestWithExtractAnnotations(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "[", + From: "pod", }, }, []kube.FieldExtractionRule{}, @@ -102,6 +103,7 @@ func TestWithExtractAnnotations(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "field=(?P.+)", + From: "pod", }, }, []kube.FieldExtractionRule{ @@ -109,6 +111,25 @@ func TestWithExtractAnnotations(t *testing.T) { Name: "tag1", Key: "key1", Regex: regexp.MustCompile(`field=(?P.+)`), + From: "pod", + }, + }, + "", + }, + { + "basic-namespace", + []FieldExtractConfig{ + { + TagName: "tag1", + Key: "key1", + From: "namespace", + }, + }, + []kube.FieldExtractionRule{ + { + Name: "tag1", + Key: "key1", + From: "namespace", }, }, "", @@ -153,6 +174,7 @@ func TestWithExtractLabels(t *testing.T) { TagName: "t1", Key: "k1", Regex: "[", + From: "pod", }}, []kube.FieldExtractionRule{}, "error parsing regexp: missing closing ]: `[`", @@ -164,6 +186,7 @@ func TestWithExtractLabels(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "field=(?P.+)", + From: "pod", }, }, []kube.FieldExtractionRule{ @@ -171,6 +194,25 @@ func TestWithExtractLabels(t *testing.T) { Name: "tag1", Key: "key1", Regex: regexp.MustCompile(`field=(?P.+)`), + From: "pod", + }, + }, + "", + }, + { + "basic-namespace", + []FieldExtractConfig{ + { + TagName: "tag1", + Key: "key1", + From: "namespace", + }, + }, + []kube.FieldExtractionRule{ + { + Name: "tag1", + Key: "key1", + From: "namespace", }, }, "", @@ -206,13 +248,15 @@ func TestWithExtractMetadata(t *testing.T) { assert.True(t, p.rules.Deployment) assert.True(t, p.rules.Cluster) assert.True(t, p.rules.Node) + assert.True(t, p.rules.NamespaceUID) + assert.True(t, p.rules.NamespaceStartTime) p = &kubernetesprocessor{} err := WithExtractMetadata("randomfield")(p) assert.Error(t, err) assert.Equal(t, err.Error(), `"randomfield" is not a supported metadata field`) - assert.NoError(t, WithExtractMetadata("namespace", "cluster")(p)) + assert.NoError(t, WithExtractMetadata("namespace", "cluster", "k8s.namespace.uid")(p)) assert.True(t, p.rules.Namespace) assert.True(t, p.rules.Cluster) assert.False(t, p.rules.PodName) @@ -231,6 +275,22 @@ func TestWithExtractMetadata(t *testing.T) { assert.False(t, p.rules.StartTime) assert.False(t, p.rules.Deployment) assert.False(t, p.rules.Node) + + p = &kubernetesprocessor{} + assert.NoError(t, WithExtractMetadata()(p)) + assert.True(t, p.rules.Namespace) + assert.True(t, p.rules.NamespaceUID) + assert.True(t, p.rules.NamespaceStartTime) + + p = &kubernetesprocessor{} + err = WithExtractMetadata("randomfield")(p) + assert.Error(t, err) + assert.Equal(t, err.Error(), `"randomfield" is not a supported metadata field`) + + assert.NoError(t, WithExtractMetadata("k8s.namespace.name", "k8s.namespace.uid")(p)) + assert.True(t, p.rules.Namespace) + assert.True(t, p.rules.NamespaceUID) + assert.False(t, p.rules.NamespaceStartTime) } func TestWithFilterLabels(t *testing.T) { diff --git a/processor/k8sprocessor/processor.go b/processor/k8sprocessor/processor.go index f0abb297e8371..dcc3bdea435bc 100644 --- a/processor/k8sprocessor/processor.go +++ b/processor/k8sprocessor/processor.go @@ -19,6 +19,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer/pdata" + "go.opentelemetry.io/collector/translator/conventions" "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig" @@ -45,7 +46,7 @@ func (kp *kubernetesprocessor) initKubeClient(logger *zap.Logger, kubeClient kub kubeClient = kube.New } if !kp.passthroughMode { - kc, err := kubeClient(logger, kp.apiConfig, kp.rules, kp.filters, kp.podAssociations, nil, nil) + kc, err := kubeClient(logger, kp.apiConfig, kp.rules, kp.filters, kp.podAssociations, nil, nil, nil) if err != nil { return err } @@ -102,16 +103,30 @@ func (kp *kubernetesprocessor) ProcessLogs(ctx context.Context, ld pdata.Logs) ( func (kp *kubernetesprocessor) processResource(ctx context.Context, resource pdata.Resource) { podIdentifierKey, podIdentifierValue := extractPodID(ctx, resource.Attributes(), kp.podAssociations) - if podIdentifierKey == "" { - return + if podIdentifierKey != "" { + resource.Attributes().InsertString(podIdentifierKey, string(podIdentifierValue)) + } + namespace := stringAttributeFromMap(resource.Attributes(), conventions.AttributeK8sNamespace) + if namespace != "" { + resource.Attributes().InsertString(conventions.AttributeK8sNamespace, namespace) } - resource.Attributes().InsertString(podIdentifierKey, string(podIdentifierValue)) + if kp.passthroughMode { return } - attrsToAdd := kp.getAttributesForPod(podIdentifierValue) - for key, val := range attrsToAdd { - resource.Attributes().InsertString(key, val) + + if podIdentifierKey != "" { + attrsToAdd := kp.getAttributesForPod(podIdentifierValue) + for key, val := range attrsToAdd { + resource.Attributes().InsertString(key, val) + } + } + + if namespace != "" { + attrsToAdd := kp.getAttributesForPodsNamespace(namespace) + for key, val := range attrsToAdd { + resource.Attributes().InsertString(key, val) + } } } @@ -122,3 +137,11 @@ func (kp *kubernetesprocessor) getAttributesForPod(identifier kube.PodIdentifier } return pod.Attributes } + +func (kp *kubernetesprocessor) getAttributesForPodsNamespace(namespace string) map[string]string { + ns, ok := kp.kc.GetNamespace(namespace) + if !ok { + return nil + } + return ns.Attributes +} diff --git a/processor/k8sprocessor/processor_test.go b/processor/k8sprocessor/processor_test.go index cf038b9ce690d..353885bf730fa 100644 --- a/processor/k8sprocessor/processor_test.go +++ b/processor/k8sprocessor/processor_test.go @@ -217,7 +217,7 @@ func TestProcessorBadConfig(t *testing.T) { } func TestProcessorBadClientProvider(t *testing.T) { - clientProvider := func(_ *zap.Logger, _ k8sconfig.APIConfig, _ kube.ExtractionRules, _ kube.Filters, _ []kube.Association, _ kube.APIClientsetProvider, _ kube.InformerProvider) (kube.Client, error) { + clientProvider := func(_ *zap.Logger, _ k8sconfig.APIConfig, _ kube.ExtractionRules, _ kube.Filters, _ []kube.Association, _ kube.APIClientsetProvider, _ kube.InformerProvider, _ kube.InformerProviderNamespace) (kube.Client, error) { return nil, fmt.Errorf("bad client error") } diff --git a/processor/k8sprocessor/testdata/config.yaml b/processor/k8sprocessor/testdata/config.yaml index c7178c77ee422..987838f1f501c 100644 --- a/processor/k8sprocessor/testdata/config.yaml +++ b/processor/k8sprocessor/testdata/config.yaml @@ -24,19 +24,26 @@ processors: # - namespace # - node # - startTime + - # extract the following well-known metadata fields from namespaces + - k8s.namespace.uid + - k8s.namespace.start_time annotations: - tag_name: a1 # extracts value of annotation with key `annotation-one` and inserts it as a tag with key `a1` key: annotation-one + from: pod - tag_name: a2 # extracts value of annotation with key `annotation-two` with regexp and inserts it as a tag with key `a2` key: annotation-two regex: field=(?P.+) + from: pod labels: - tag_name: l1 # extracts value of label with key `label1` and inserts it as a tag with key `l1` key: label1 + from: pod - tag_name: l2 # extracts value of label with key `label1` with regexp and inserts it as a tag with key `l2` key: label2 regex: field=(?P.+) + from: pod filter: namespace: ns2 # only look for pods running in ns2 namespace From e522382abd2fbdf7691e9f3e45fbf4059ee46629 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 8 Jun 2021 17:57:04 -0700 Subject: [PATCH 02/15] cleanup tests --- processor/k8sprocessor/config_test.go | 8 ++++---- processor/k8sprocessor/options_test.go | 8 +++++++- processor/k8sprocessor/testdata/config.yaml | 19 +++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/processor/k8sprocessor/config_test.go b/processor/k8sprocessor/config_test.go index 9cdcf8d04b9a0..9108933525eeb 100644 --- a/processor/k8sprocessor/config_test.go +++ b/processor/k8sprocessor/config_test.go @@ -59,12 +59,12 @@ func TestLoadConfig(t *testing.T) { Extract: ExtractConfig{ Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time", "k8s.namespace.uid", "k8s.namespace.start_time"}, Annotations: []FieldExtractConfig{ - {TagName: "a1", Key: "annotation-one"}, - {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)"}, + {TagName: "a1", Key: "annotation-one", From: "pod"}, + {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)", From: "pod"}, }, Labels: []FieldExtractConfig{ - {TagName: "l1", Key: "label1"}, - {TagName: "l2", Key: "label2", Regex: "field=(?P.+)"}, + {TagName: "l1", Key: "label1", From: "pod"}, + {TagName: "l2", Key: "label2", Regex: "field=(?P.+)", From: "pod"}, }, }, Filter: FilterConfig{ diff --git a/processor/k8sprocessor/options_test.go b/processor/k8sprocessor/options_test.go index 2b041e254b881..033e0223caa5b 100644 --- a/processor/k8sprocessor/options_test.go +++ b/processor/k8sprocessor/options_test.go @@ -571,13 +571,15 @@ func Test_extractFieldRules(t *testing.T) { "default", args{"labels", []FieldExtractConfig{ { - Key: "key", + Key: "key", + From: "pod", }, }}, []kube.FieldExtractionRule{ { Name: "k8s.pod.labels.key", Key: "key", + From: "pod", }, }, false, @@ -588,12 +590,14 @@ func Test_extractFieldRules(t *testing.T) { { TagName: "name", Key: "key", + From: "pod", }, }}, []kube.FieldExtractionRule{ { Name: "name", Key: "key", + From: "pod", }, }, false, @@ -605,6 +609,7 @@ func Test_extractFieldRules(t *testing.T) { TagName: "name", Key: "key", Regex: "^h$", + From: "pod", }, }}, []kube.FieldExtractionRule{}, @@ -617,6 +622,7 @@ func Test_extractFieldRules(t *testing.T) { TagName: "name", Key: "key", Regex: "[", + From: "pod", }, }}, []kube.FieldExtractionRule{}, diff --git a/processor/k8sprocessor/testdata/config.yaml b/processor/k8sprocessor/testdata/config.yaml index 987838f1f501c..44892fb4f2c8c 100644 --- a/processor/k8sprocessor/testdata/config.yaml +++ b/processor/k8sprocessor/testdata/config.yaml @@ -8,7 +8,15 @@ processors: auth_type: "kubeConfig" extract: metadata: - # extract the following well-known metadata fields + # the following metadata fields configuration options are deprecated + # - podName + # - podUID + # - deployment + # - cluster + # - namespace + # - node + # - startTime + # extract the following well-known metadata fields from pods and namespaces - k8s.pod.name - k8s.pod.uid - k8s.deployment.name @@ -16,15 +24,6 @@ processors: - k8s.namespace.name - k8s.node.name - k8s.pod.start_time - # the following metadata fields configuration options are deprecated -# - podName -# - podUID -# - deployment -# - cluster -# - namespace -# - node -# - startTime - - # extract the following well-known metadata fields from namespaces - k8s.namespace.uid - k8s.namespace.start_time From 6a20a924243271a943a227fea3ef596269f95510 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Tue, 8 Jun 2021 18:21:36 -0700 Subject: [PATCH 03/15] fix test --- processor/k8sprocessor/kube/client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/processor/k8sprocessor/kube/client_test.go b/processor/k8sprocessor/kube/client_test.go index 89af37e0a701c..9276ba14b6f3e 100644 --- a/processor/k8sprocessor/kube/client_test.go +++ b/processor/k8sprocessor/kube/client_test.go @@ -152,6 +152,7 @@ func TestClientStartStop(t *testing.T) { }() c.Stop() <-done + time.Sleep(time.Second) assert.True(t, fctr.HasStopped()) } From 087ba86fe2efb05ff2848ff4ae8a2a4a7a9452c3 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 9 Jun 2021 13:57:51 -0700 Subject: [PATCH 04/15] review cleanup --- processor/k8sprocessor/config.go | 5 +- processor/k8sprocessor/config_test.go | 2 +- processor/k8sprocessor/kube/client.go | 49 +++++++------ processor/k8sprocessor/kube/client_test.go | 79 +++++++++++++++++---- processor/k8sprocessor/kube/kube.go | 20 +++--- processor/k8sprocessor/options.go | 10 +-- processor/k8sprocessor/options_test.go | 27 ------- processor/k8sprocessor/testdata/config.yaml | 2 - 8 files changed, 102 insertions(+), 92 deletions(-) diff --git a/processor/k8sprocessor/config.go b/processor/k8sprocessor/config.go index 120928e0532c4..d54f542e452ea 100644 --- a/processor/k8sprocessor/config.go +++ b/processor/k8sprocessor/config.go @@ -57,10 +57,7 @@ type ExtractConfig struct { // // Metadata fields supported right now are, // k8s.pod.name, k8s.pod.uid, k8s.deployment.name, k8s.cluster.name, - // k8s.node.name and k8s.pod.start_time - // - // Metadata fields supported for namespaces right now are, - // k8s.namespace.name, k8s.namespace.uid and k8s.namespace.start_time + // k8s.node.name, k8s.namespace.name and k8s.pod.start_time // // Specifying anything other than these values will result in an error. // By default all of the fields are extracted and added to spans and metrics. diff --git a/processor/k8sprocessor/config_test.go b/processor/k8sprocessor/config_test.go index 9108933525eeb..36a6f932713c2 100644 --- a/processor/k8sprocessor/config_test.go +++ b/processor/k8sprocessor/config_test.go @@ -57,7 +57,7 @@ func TestLoadConfig(t *testing.T) { APIConfig: k8sconfig.APIConfig{AuthType: k8sconfig.AuthTypeKubeConfig}, Passthrough: false, Extract: ExtractConfig{ - Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time", "k8s.namespace.uid", "k8s.namespace.start_time"}, + Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time"}, Annotations: []FieldExtractConfig{ {TagName: "a1", Key: "annotation-one", From: "pod"}, {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)", From: "pod"}, diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index 405941c38db72..9f14dc0389dd7 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -104,7 +104,9 @@ func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, } c.informer = newInformer(c.kc, c.Filters.Namespace, labelSelector, fieldSelector) - c.namespaceInformer = newNamespaceInformer(c.kc) + if c.extractNamespaceLabelsAnnotations() { + c.namespaceInformer = newNamespaceInformer(c.kc) + } return c, err } @@ -121,7 +123,7 @@ func (c *WatchClient) Start() { UpdateFunc: c.handleNamespaceUpdate, DeleteFunc: c.handleNamespaceDelete, }) - c.namespaceInformer.Run(c.stopCh) + go c.namespaceInformer.Run(c.stopCh) } // Stop signals the the k8s watcher/informer to stop watching for new events. @@ -295,7 +297,7 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Labels { - if r.From == "pod" { + if r.From == metadataFromPod { if v, ok := pod.Labels[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -303,7 +305,7 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Annotations { - if r.From == "pod" { + if r.From == metadataFromPod { if v, ok := pod.Annotations[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -315,20 +317,8 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { func (c *WatchClient) extractNamespaceAttributes(namespace *api_v1.Namespace) map[string]string { tags := map[string]string{} - if c.Rules.NamespaceUID { - uid := namespace.GetUID() - tags[tagNamespaceUID] = string(uid) - } - - if c.Rules.NamespaceStartTime { - ts := namespace.GetCreationTimestamp() - if !ts.IsZero() { - tags[tagNamespaceStartTime] = ts.String() - } - } - for _, r := range c.Rules.Labels { - if r.From == "namespace" { + if r.From == metadataFromNamespace { if v, ok := namespace.Labels[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -336,7 +326,7 @@ func (c *WatchClient) extractNamespaceAttributes(namespace *api_v1.Namespace) ma } for _, r := range c.Rules.Annotations { - if r.From == "namespace" { + if r.From == metadataFromNamespace { if v, ok := namespace.Annotations[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -483,22 +473,35 @@ func (c *WatchClient) addOrUpdateNamespace(namespace *api_v1.Namespace) { NamespaceUID: string(namespace.UID), StartTime: namespace.GetCreationTimestamp(), } - newNamespace.Attributes = c.extractNamespaceAttributes(namespace) c.m.Lock() - defer c.m.Unlock() - if namespace.Name != "" { c.Namespaces[namespace.Name] = newNamespace } + c.m.Unlock() } func (c *WatchClient) forgetNamespace(namespace *api_v1.Namespace) { c.m.Lock() - defer c.m.Unlock() - if ns, ok := c.Namespaces[namespace.Name]; ok { delete(c.Namespaces, ns.Name) } + c.m.Unlock() +} + +func (c *WatchClient) extractNamespaceLabelsAnnotations() bool { + for _, r := range c.Rules.Labels { + if r.From == metadataFromNamespace { + return true + } + } + + for _, r := range c.Rules.Annotations { + if r.From == metadataFromNamespace { + return true + } + } + + return false } diff --git a/processor/k8sprocessor/kube/client_test.go b/processor/k8sprocessor/kube/client_test.go index 9276ba14b6f3e..0fc60a2161b15 100644 --- a/processor/k8sprocessor/kube/client_test.go +++ b/processor/k8sprocessor/kube/client_test.go @@ -467,18 +467,18 @@ func TestExtractionRules(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: "pod", + From: metadataFromPod, }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: "pod", + From: metadataFromPod, }, { Name: "l2", Key: "label2", Regex: regexp.MustCompile(`k5=(?P[^\s]+)`), - From: "pod", + From: metadataFromPod, }, }, }, @@ -531,29 +531,19 @@ func TestNamespaceExtractionRules(t *testing.T) { name: "no-rules", rules: ExtractionRules{}, attributes: nil, - }, { - name: "metadata", - rules: ExtractionRules{ - NamespaceUID: true, - NamespaceStartTime: true, - }, - attributes: map[string]string{ - "k8s.namespace.uid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", - "k8s.namespace.start_time": namespace.GetCreationTimestamp().String(), - }, }, { name: "labels", rules: ExtractionRules{ Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: "namespace", + From: metadataFromNamespace, }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: "namespace", + From: metadataFromNamespace, }, }, }, @@ -807,6 +797,65 @@ func Test_selectorsFromFilters(t *testing.T) { } } +func TestExtractNamespaceLabelsAnnotations(t *testing.T) { + c, _ := newTestClientWithRulesAndFilters(t, ExtractionRules{}, Filters{}) + testCases := []struct { + name string + shouldExtractNamespace bool + rules ExtractionRules + }{{ + name: "empty-rules", + shouldExtractNamespace: false, + rules: ExtractionRules{}, + }, { + name: "pod-rules", + shouldExtractNamespace: false, + rules: ExtractionRules{ + Annotations: []FieldExtractionRule{{ + Name: "a1", + Key: "annotation1", + From: metadataFromPod, + }, + }, + Labels: []FieldExtractionRule{{ + Name: "l1", + Key: "label1", + From: metadataFromPod, + }, + }, + }, + }, { + name: "namespace-rules-only-annotations", + shouldExtractNamespace: true, + rules: ExtractionRules{ + Annotations: []FieldExtractionRule{{ + Name: "a1", + Key: "annotation1", + From: metadataFromNamespace, + }, + }, + }, + }, { + name: "namespace-rules-only-labels", + shouldExtractNamespace: true, + rules: ExtractionRules{ + Labels: []FieldExtractionRule{{ + Name: "l1", + Key: "label1", + From: metadataFromNamespace, + }, + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c.Rules = tc.rules + assert.Equal(t, tc.shouldExtractNamespace, c.extractNamespaceLabelsAnnotations()) + }) + } +} + func newTestClientWithRulesAndFilters(t *testing.T, e ExtractionRules, f Filters) (*WatchClient, *observer.ObservedLogs) { observedLogger, logs := observer.New(zapcore.WarnLevel) logger := zap.New(observedLogger) diff --git a/processor/k8sprocessor/kube/kube.go b/processor/k8sprocessor/kube/kube.go index 0404895697769..f70e7a95fd3b1 100644 --- a/processor/k8sprocessor/kube/kube.go +++ b/processor/k8sprocessor/kube/kube.go @@ -32,8 +32,8 @@ const ( tagNodeName = "k8s.node.name" tagStartTime = "k8s.pod.start_time" - tagNamespaceStartTime = "k8s.namespace.start_time" - tagNamespaceUID = "k8s.namespace.uid" + metadataFromPod = "pod" + metadataFromNamespace = "namespace" ) // PodIdentifier is a custom type to represent IP Address or Pod UID @@ -121,15 +121,13 @@ type FieldFilter struct { // ExtractionRules is used to specify the information that needs to be extracted // from pods and added to the spans as tags. type ExtractionRules struct { - Deployment bool - Namespace bool - PodName bool - PodUID bool - Node bool - Cluster bool - StartTime bool - NamespaceUID bool - NamespaceStartTime bool + Deployment bool + Namespace bool + PodName bool + PodUID bool + Node bool + Cluster bool + StartTime bool Annotations []FieldExtractionRule Labels []FieldExtractionRule diff --git a/processor/k8sprocessor/options.go b/processor/k8sprocessor/options.go index e37aecd9f30ec..fc1185dc553ad 100644 --- a/processor/k8sprocessor/options.go +++ b/processor/k8sprocessor/options.go @@ -40,9 +40,7 @@ const ( metadataCluster = "cluster" metadataNode = "node" // Will be removed when new fields get merged to https://github.com/open-telemetry/opentelemetry-collector/blob/main/translator/conventions/opentelemetry.go - metadataPodStartTime = "k8s.pod.start_time" - metadataNamespaceUID = "k8s.namespace.uid" - metadataNamespaceStartTime = "k8s.namespace.start_time" + metadataPodStartTime = "k8s.pod.start_time" ) // Option represents a configuration option that can be passes. @@ -80,8 +78,6 @@ func WithExtractMetadata(fields ...string) Option { conventions.AttributeK8sDeployment, conventions.AttributeK8sCluster, conventions.AttributeK8sNodeName, - metadataNamespaceUID, - metadataNamespaceStartTime, } } for _, field := range fields { @@ -103,10 +99,6 @@ func WithExtractMetadata(fields ...string) Option { p.rules.Cluster = true case metadataNode, conventions.AttributeK8sNodeName: p.rules.Node = true - case metadataNamespaceUID: - p.rules.NamespaceUID = true - case metadataNamespaceStartTime: - p.rules.NamespaceStartTime = true default: return fmt.Errorf("\"%s\" is not a supported metadata field", field) } diff --git a/processor/k8sprocessor/options_test.go b/processor/k8sprocessor/options_test.go index 033e0223caa5b..fa08fb78da822 100644 --- a/processor/k8sprocessor/options_test.go +++ b/processor/k8sprocessor/options_test.go @@ -248,23 +248,12 @@ func TestWithExtractMetadata(t *testing.T) { assert.True(t, p.rules.Deployment) assert.True(t, p.rules.Cluster) assert.True(t, p.rules.Node) - assert.True(t, p.rules.NamespaceUID) - assert.True(t, p.rules.NamespaceStartTime) p = &kubernetesprocessor{} err := WithExtractMetadata("randomfield")(p) assert.Error(t, err) assert.Equal(t, err.Error(), `"randomfield" is not a supported metadata field`) - assert.NoError(t, WithExtractMetadata("namespace", "cluster", "k8s.namespace.uid")(p)) - assert.True(t, p.rules.Namespace) - assert.True(t, p.rules.Cluster) - assert.False(t, p.rules.PodName) - assert.False(t, p.rules.PodUID) - assert.False(t, p.rules.StartTime) - assert.False(t, p.rules.Deployment) - assert.False(t, p.rules.Node) - p = &kubernetesprocessor{} assert.NoError(t, WithExtractMetadata(conventions.AttributeK8sNamespace, conventions.AttributeK8sPod, conventions.AttributeK8sPodUID)(p)) @@ -275,22 +264,6 @@ func TestWithExtractMetadata(t *testing.T) { assert.False(t, p.rules.StartTime) assert.False(t, p.rules.Deployment) assert.False(t, p.rules.Node) - - p = &kubernetesprocessor{} - assert.NoError(t, WithExtractMetadata()(p)) - assert.True(t, p.rules.Namespace) - assert.True(t, p.rules.NamespaceUID) - assert.True(t, p.rules.NamespaceStartTime) - - p = &kubernetesprocessor{} - err = WithExtractMetadata("randomfield")(p) - assert.Error(t, err) - assert.Equal(t, err.Error(), `"randomfield" is not a supported metadata field`) - - assert.NoError(t, WithExtractMetadata("k8s.namespace.name", "k8s.namespace.uid")(p)) - assert.True(t, p.rules.Namespace) - assert.True(t, p.rules.NamespaceUID) - assert.False(t, p.rules.NamespaceStartTime) } func TestWithFilterLabels(t *testing.T) { diff --git a/processor/k8sprocessor/testdata/config.yaml b/processor/k8sprocessor/testdata/config.yaml index 44892fb4f2c8c..89e95e5129536 100644 --- a/processor/k8sprocessor/testdata/config.yaml +++ b/processor/k8sprocessor/testdata/config.yaml @@ -24,8 +24,6 @@ processors: - k8s.namespace.name - k8s.node.name - k8s.pod.start_time - - k8s.namespace.uid - - k8s.namespace.start_time annotations: - tag_name: a1 # extracts value of annotation with key `annotation-one` and inserts it as a tag with key `a1` From ece4edfa22682bbecb83bfbff6b9bc6f29598f5a Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 9 Jun 2021 14:16:06 -0700 Subject: [PATCH 05/15] fix test --- processor/k8sprocessor/kube/client.go | 14 ++++++++------ processor/k8sprocessor/options_test.go | 1 - 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index 9f14dc0389dd7..066bf3e9051ee 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -118,12 +118,14 @@ func (c *WatchClient) Start() { DeleteFunc: c.handlePodDelete, }) go c.informer.Run(c.stopCh) - c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: c.handleNamespaceAdd, - UpdateFunc: c.handleNamespaceUpdate, - DeleteFunc: c.handleNamespaceDelete, - }) - go c.namespaceInformer.Run(c.stopCh) + if c.extractNamespaceLabelsAnnotations() { + c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.handleNamespaceAdd, + UpdateFunc: c.handleNamespaceUpdate, + DeleteFunc: c.handleNamespaceDelete, + }) + go c.namespaceInformer.Run(c.stopCh) + } } // Stop signals the the k8s watcher/informer to stop watching for new events. diff --git a/processor/k8sprocessor/options_test.go b/processor/k8sprocessor/options_test.go index fa08fb78da822..4bd354b13c204 100644 --- a/processor/k8sprocessor/options_test.go +++ b/processor/k8sprocessor/options_test.go @@ -255,7 +255,6 @@ func TestWithExtractMetadata(t *testing.T) { assert.Equal(t, err.Error(), `"randomfield" is not a supported metadata field`) p = &kubernetesprocessor{} - assert.NoError(t, WithExtractMetadata(conventions.AttributeK8sNamespace, conventions.AttributeK8sPod, conventions.AttributeK8sPodUID)(p)) assert.True(t, p.rules.Namespace) assert.False(t, p.rules.Cluster) From b71ab1cd1fbd3317462dcc4077814ee21f5bb4f4 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Wed, 9 Jun 2021 16:35:38 -0700 Subject: [PATCH 06/15] cleanup --- processor/k8sprocessor/kube/client.go | 12 ++++---- processor/k8sprocessor/kube/client_test.go | 18 ++++++------ processor/k8sprocessor/kube/kube.go | 4 +-- processor/k8sprocessor/options_test.go | 32 +++++++++++----------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index 066bf3e9051ee..8b9aa0e38af2f 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -299,7 +299,7 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Labels { - if r.From == metadataFromPod { + if r.From == MetadataFromPod { if v, ok := pod.Labels[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -307,7 +307,7 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Annotations { - if r.From == metadataFromPod { + if r.From == MetadataFromPod { if v, ok := pod.Annotations[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -320,7 +320,7 @@ func (c *WatchClient) extractNamespaceAttributes(namespace *api_v1.Namespace) ma tags := map[string]string{} for _, r := range c.Rules.Labels { - if r.From == metadataFromNamespace { + if r.From == MetadataFromNamespace { if v, ok := namespace.Labels[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -328,7 +328,7 @@ func (c *WatchClient) extractNamespaceAttributes(namespace *api_v1.Namespace) ma } for _, r := range c.Rules.Annotations { - if r.From == metadataFromNamespace { + if r.From == MetadataFromNamespace { if v, ok := namespace.Annotations[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -494,13 +494,13 @@ func (c *WatchClient) forgetNamespace(namespace *api_v1.Namespace) { func (c *WatchClient) extractNamespaceLabelsAnnotations() bool { for _, r := range c.Rules.Labels { - if r.From == metadataFromNamespace { + if r.From == MetadataFromNamespace { return true } } for _, r := range c.Rules.Annotations { - if r.From == metadataFromNamespace { + if r.From == MetadataFromNamespace { return true } } diff --git a/processor/k8sprocessor/kube/client_test.go b/processor/k8sprocessor/kube/client_test.go index 0fc60a2161b15..ffefe3ced02cb 100644 --- a/processor/k8sprocessor/kube/client_test.go +++ b/processor/k8sprocessor/kube/client_test.go @@ -467,18 +467,18 @@ func TestExtractionRules(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: metadataFromPod, + From: MetadataFromPod, }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: metadataFromPod, + From: MetadataFromPod, }, { Name: "l2", Key: "label2", Regex: regexp.MustCompile(`k5=(?P[^\s]+)`), - From: metadataFromPod, + From: MetadataFromPod, }, }, }, @@ -537,13 +537,13 @@ func TestNamespaceExtractionRules(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: metadataFromNamespace, + From: MetadataFromNamespace, }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: metadataFromNamespace, + From: MetadataFromNamespace, }, }, }, @@ -814,13 +814,13 @@ func TestExtractNamespaceLabelsAnnotations(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: metadataFromPod, + From: MetadataFromPod, }, }, Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: metadataFromPod, + From: MetadataFromPod, }, }, }, @@ -831,7 +831,7 @@ func TestExtractNamespaceLabelsAnnotations(t *testing.T) { Annotations: []FieldExtractionRule{{ Name: "a1", Key: "annotation1", - From: metadataFromNamespace, + From: MetadataFromNamespace, }, }, }, @@ -842,7 +842,7 @@ func TestExtractNamespaceLabelsAnnotations(t *testing.T) { Labels: []FieldExtractionRule{{ Name: "l1", Key: "label1", - From: metadataFromNamespace, + From: MetadataFromNamespace, }, }, }, diff --git a/processor/k8sprocessor/kube/kube.go b/processor/k8sprocessor/kube/kube.go index f70e7a95fd3b1..a99a28ed3a47a 100644 --- a/processor/k8sprocessor/kube/kube.go +++ b/processor/k8sprocessor/kube/kube.go @@ -32,8 +32,8 @@ const ( tagNodeName = "k8s.node.name" tagStartTime = "k8s.pod.start_time" - metadataFromPod = "pod" - metadataFromNamespace = "namespace" + MetadataFromPod = "pod" + MetadataFromNamespace = "namespace" ) // PodIdentifier is a custom type to represent IP Address or Pod UID diff --git a/processor/k8sprocessor/options_test.go b/processor/k8sprocessor/options_test.go index 4bd354b13c204..ac286044edc23 100644 --- a/processor/k8sprocessor/options_test.go +++ b/processor/k8sprocessor/options_test.go @@ -90,7 +90,7 @@ func TestWithExtractAnnotations(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "[", - From: "pod", + From: kube.MetadataFromPod, }, }, []kube.FieldExtractionRule{}, @@ -103,7 +103,7 @@ func TestWithExtractAnnotations(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "field=(?P.+)", - From: "pod", + From: kube.MetadataFromPod, }, }, []kube.FieldExtractionRule{ @@ -111,7 +111,7 @@ func TestWithExtractAnnotations(t *testing.T) { Name: "tag1", Key: "key1", Regex: regexp.MustCompile(`field=(?P.+)`), - From: "pod", + From: kube.MetadataFromPod, }, }, "", @@ -122,14 +122,14 @@ func TestWithExtractAnnotations(t *testing.T) { { TagName: "tag1", Key: "key1", - From: "namespace", + From: kube.MetadataFromNamespace, }, }, []kube.FieldExtractionRule{ { Name: "tag1", Key: "key1", - From: "namespace", + From: kube.MetadataFromNamespace, }, }, "", @@ -174,7 +174,7 @@ func TestWithExtractLabels(t *testing.T) { TagName: "t1", Key: "k1", Regex: "[", - From: "pod", + From: kube.MetadataFromPod, }}, []kube.FieldExtractionRule{}, "error parsing regexp: missing closing ]: `[`", @@ -186,7 +186,7 @@ func TestWithExtractLabels(t *testing.T) { TagName: "tag1", Key: "key1", Regex: "field=(?P.+)", - From: "pod", + From: kube.MetadataFromPod, }, }, []kube.FieldExtractionRule{ @@ -194,7 +194,7 @@ func TestWithExtractLabels(t *testing.T) { Name: "tag1", Key: "key1", Regex: regexp.MustCompile(`field=(?P.+)`), - From: "pod", + From: kube.MetadataFromPod, }, }, "", @@ -205,14 +205,14 @@ func TestWithExtractLabels(t *testing.T) { { TagName: "tag1", Key: "key1", - From: "namespace", + From: kube.MetadataFromNamespace, }, }, []kube.FieldExtractionRule{ { Name: "tag1", Key: "key1", - From: "namespace", + From: kube.MetadataFromNamespace, }, }, "", @@ -544,14 +544,14 @@ func Test_extractFieldRules(t *testing.T) { args{"labels", []FieldExtractConfig{ { Key: "key", - From: "pod", + From: kube.MetadataFromPod, }, }}, []kube.FieldExtractionRule{ { Name: "k8s.pod.labels.key", Key: "key", - From: "pod", + From: kube.MetadataFromPod, }, }, false, @@ -562,14 +562,14 @@ func Test_extractFieldRules(t *testing.T) { { TagName: "name", Key: "key", - From: "pod", + From: kube.MetadataFromPod, }, }}, []kube.FieldExtractionRule{ { Name: "name", Key: "key", - From: "pod", + From: kube.MetadataFromPod, }, }, false, @@ -581,7 +581,7 @@ func Test_extractFieldRules(t *testing.T) { TagName: "name", Key: "key", Regex: "^h$", - From: "pod", + From: kube.MetadataFromPod, }, }}, []kube.FieldExtractionRule{}, @@ -594,7 +594,7 @@ func Test_extractFieldRules(t *testing.T) { TagName: "name", Key: "key", Regex: "[", - From: "pod", + From: kube.MetadataFromPod, }, }}, []kube.FieldExtractionRule{}, From 2fd8cd07ea423a0e9eec025a09c52502777ee5cf Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 10 Jun 2021 12:00:07 -0700 Subject: [PATCH 07/15] review cleanup --- processor/k8sprocessor/kube/client.go | 36 ++++++------- processor/k8sprocessor/kube/fake_informer.go | 45 ++++++++++++++++ processor/k8sprocessor/kube/kube.go | 9 ++-- .../observability/observability.go | 51 ++++++++++++++++--- 4 files changed, 114 insertions(+), 27 deletions(-) diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index 8b9aa0e38af2f..e823e84f6631a 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -106,6 +106,8 @@ func New(logger *zap.Logger, apiCfg k8sconfig.APIConfig, rules ExtractionRules, c.informer = newInformer(c.kc, c.Filters.Namespace, labelSelector, fieldSelector) if c.extractNamespaceLabelsAnnotations() { c.namespaceInformer = newNamespaceInformer(c.kc) + } else { + c.namespaceInformer = NewNoOpInformer(c.kc) } return c, err } @@ -118,14 +120,12 @@ func (c *WatchClient) Start() { DeleteFunc: c.handlePodDelete, }) go c.informer.Run(c.stopCh) - if c.extractNamespaceLabelsAnnotations() { - c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: c.handleNamespaceAdd, - UpdateFunc: c.handleNamespaceUpdate, - DeleteFunc: c.handleNamespaceDelete, - }) - go c.namespaceInformer.Run(c.stopCh) - } + c.namespaceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.handleNamespaceAdd, + UpdateFunc: c.handleNamespaceUpdate, + DeleteFunc: c.handleNamespaceDelete, + }) + go c.namespaceInformer.Run(c.stopCh) } // Stop signals the the k8s watcher/informer to stop watching for new events. @@ -168,6 +168,7 @@ func (c *WatchClient) handlePodDelete(obj interface{}) { } func (c *WatchClient) handleNamespaceAdd(obj interface{}) { + observability.RecordNamespaceAdded() if namespace, ok := obj.(*api_v1.Namespace); ok { c.addOrUpdateNamespace(namespace) } else { @@ -176,6 +177,7 @@ func (c *WatchClient) handleNamespaceAdd(obj interface{}) { } func (c *WatchClient) handleNamespaceUpdate(old, new interface{}) { + observability.RecordNamespaceUpdated() if namespace, ok := new.(*api_v1.Namespace); ok { c.addOrUpdateNamespace(namespace) } else { @@ -184,8 +186,16 @@ func (c *WatchClient) handleNamespaceUpdate(old, new interface{}) { } func (c *WatchClient) handleNamespaceDelete(obj interface{}) { + observability.RecordNamespaceDeleted() if namespace, ok := obj.(*api_v1.Namespace); ok { - c.forgetNamespace(namespace) + c.m.Lock() + if ns, ok := c.Namespaces[namespace.Name]; ok { + // When a namespace is deleted all the pods(and other k8s objects in that namespace) in that namespace are deleted before it. + // So we wont have any spans that might need namespace annotations and labels. + // Thats why we dont need an implementation for deleteQueue and gracePeriod for namespaces. + delete(c.Namespaces, ns.Name) + } + c.m.Unlock() } else { c.logger.Error("object received was not of type api_v1.Namespace", zap.Any("received", obj)) } @@ -484,14 +494,6 @@ func (c *WatchClient) addOrUpdateNamespace(namespace *api_v1.Namespace) { c.m.Unlock() } -func (c *WatchClient) forgetNamespace(namespace *api_v1.Namespace) { - c.m.Lock() - if ns, ok := c.Namespaces[namespace.Name]; ok { - delete(c.Namespaces, ns.Name) - } - c.m.Unlock() -} - func (c *WatchClient) extractNamespaceLabelsAnnotations() bool { for _, r := range c.Rules.Labels { if r.From == MetadataFromNamespace { diff --git a/processor/k8sprocessor/kube/fake_informer.go b/processor/k8sprocessor/kube/fake_informer.go index bdf0d97a3dad2..f2fd4ff5f8a6c 100644 --- a/processor/k8sprocessor/kube/fake_informer.go +++ b/processor/k8sprocessor/kube/fake_informer.go @@ -113,3 +113,48 @@ func (c *FakeController) LastSyncResourceVersion() string { func (f *FakeInformer) SetWatchErrorHandler(cache.WatchErrorHandler) error { return nil } + +type NoOpInformer struct { + *NoOpController +} + +func NewNoOpInformer( + _ kubernetes.Interface, +) cache.SharedInformer { + return &NoOpInformer{ + NoOpController: &NoOpController{}, + } +} + +func (f *NoOpInformer) AddEventHandler(handler cache.ResourceEventHandler) {} + +func (f *NoOpInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, period time.Duration) { +} + +func (f *NoOpInformer) GetStore() cache.Store { + return cache.NewStore(func(obj interface{}) (string, error) { return "", nil }) +} + +func (f *NoOpInformer) GetController() cache.Controller { + return f.NoOpController +} + +type NoOpController struct{} + +func (c *NoOpController) HasSynced() bool { + return true +} + +func (c *NoOpController) Run(stopCh <-chan struct{}) {} + +func (c *NoOpController) HasStopped() bool { + return true +} + +func (c *NoOpController) LastSyncResourceVersion() string { + return "" +} + +func (f *NoOpController) SetWatchErrorHandler(cache.WatchErrorHandler) error { + return nil +} diff --git a/processor/k8sprocessor/kube/kube.go b/processor/k8sprocessor/kube/kube.go index a99a28ed3a47a..a252b3dfa6d3f 100644 --- a/processor/k8sprocessor/kube/kube.go +++ b/processor/k8sprocessor/kube/kube.go @@ -29,10 +29,11 @@ import ( const ( podNodeField = "spec.nodeName" ignoreAnnotation string = "opentelemetry.io/k8s-processor/ignore" - - tagNodeName = "k8s.node.name" - tagStartTime = "k8s.pod.start_time" - MetadataFromPod = "pod" + tagNodeName = "k8s.node.name" + tagStartTime = "k8s.pod.start_time" + // Used to specify to extract metadata/labels/annotations from pod + MetadataFromPod = "pod" + // Used to specify to extract metadata/labels/annotations from namespace MetadataFromNamespace = "namespace" ) diff --git a/processor/k8sprocessor/observability/observability.go b/processor/k8sprocessor/observability/observability.go index e639cec823a1c..3d3d6b6a8ee00 100644 --- a/processor/k8sprocessor/observability/observability.go +++ b/processor/k8sprocessor/observability/observability.go @@ -35,12 +35,14 @@ func init() { } var ( - mPodsUpdated = stats.Int64("otelsvc/k8s/pod_updated", "Number of pod update events received", "1") - mPodsAdded = stats.Int64("otelsvc/k8s/pod_added", "Number of pod add events received", "1") - mPodsDeleted = stats.Int64("otelsvc/k8s/pod_deleted", "Number of pod delete events received", "1") - mPodTableSize = stats.Int64("otelsvc/k8s/pod_table_size", "Size of table containing pod info", "1") - - mIPLookupMiss = stats.Int64("otelsvc/k8s/ip_lookup_miss", "Number of times pod by IP lookup failed.", "1") + mPodsUpdated = stats.Int64("otelsvc/k8s/pod_updated", "Number of pod update events received", "1") + mPodsAdded = stats.Int64("otelsvc/k8s/pod_added", "Number of pod add events received", "1") + mPodsDeleted = stats.Int64("otelsvc/k8s/pod_deleted", "Number of pod delete events received", "1") + mPodTableSize = stats.Int64("otelsvc/k8s/pod_table_size", "Size of table containing pod info", "1") + mIPLookupMiss = stats.Int64("otelsvc/k8s/ip_lookup_miss", "Number of times pod by IP lookup failed.", "1") + mNamespacesUpdated = stats.Int64("otelsvc/k8s/namespace_updated", "Number of namespace update events received", "1") + mNamespacesAdded = stats.Int64("otelsvc/k8s/namespace_added", "Number of namespace add events received", "1") + mNamespacesDeleted = stats.Int64("otelsvc/k8s/namespace_deleted", "Number of namespace delete events received", "1") ) var viewPodsUpdated = &view.View{ @@ -70,6 +72,7 @@ var viewIPLookupMiss = &view.View{ Measure: mIPLookupMiss, Aggregation: view.Sum(), } + var viewPodTableSize = &view.View{ Name: mPodTableSize.Name(), Description: mPodTableSize.Description(), @@ -77,6 +80,27 @@ var viewPodTableSize = &view.View{ Aggregation: view.LastValue(), } +var viewNamespacesUpdated = &view.View{ + Name: mNamespacesUpdated.Name(), + Description: mNamespacesUpdated.Description(), + Measure: mNamespacesUpdated, + Aggregation: view.Sum(), +} + +var viewNamespacesAdded = &view.View{ + Name: mNamespacesAdded.Name(), + Description: mNamespacesAdded.Description(), + Measure: mNamespacesAdded, + Aggregation: view.Sum(), +} + +var viewNamespacesDeleted = &view.View{ + Name: mNamespacesDeleted.Name(), + Description: mNamespacesDeleted.Description(), + Measure: mNamespacesDeleted, + Aggregation: view.Sum(), +} + // RecordPodUpdated increments the metric that records pod update events received. func RecordPodUpdated() { stats.Record(context.Background(), mPodsUpdated.M(int64(1))) @@ -101,3 +125,18 @@ func RecordIPLookupMiss() { func RecordPodTableSize(podTableSize int64) { stats.Record(context.Background(), mPodTableSize.M(podTableSize)) } + +// RecordNamespaceUpdated increments the metric that records namespace update events received. +func RecordNamespaceUpdated() { + stats.Record(context.Background(), mNamespacesUpdated.M(int64(1))) +} + +// RecordNamespaceAdded increments the metric that records namespace add events receiver. +func RecordNamespaceAdded() { + stats.Record(context.Background(), mNamespacesAdded.M(int64(1))) +} + +// RecordNamespaceDeleted increments the metric that records namespace events deleted. +func RecordNamespaceDeleted() { + stats.Record(context.Background(), mNamespacesDeleted.M(int64(1))) +} From 172117ce9fcfc84a85dd27dedb0b65d5e1b8dc1a Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 10 Jun 2021 12:50:24 -0700 Subject: [PATCH 08/15] fix lint --- processor/k8sprocessor/kube/fake_informer.go | 2 +- processor/k8sprocessor/kube/kube.go | 4 ++-- .../k8sprocessor/observability/observability.go | 3 +++ .../k8sprocessor/observability/observability_test.go | 12 ++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/processor/k8sprocessor/kube/fake_informer.go b/processor/k8sprocessor/kube/fake_informer.go index f2fd4ff5f8a6c..c98a0797b8100 100644 --- a/processor/k8sprocessor/kube/fake_informer.go +++ b/processor/k8sprocessor/kube/fake_informer.go @@ -155,6 +155,6 @@ func (c *NoOpController) LastSyncResourceVersion() string { return "" } -func (f *NoOpController) SetWatchErrorHandler(cache.WatchErrorHandler) error { +func (c *NoOpController) SetWatchErrorHandler(cache.WatchErrorHandler) error { return nil } diff --git a/processor/k8sprocessor/kube/kube.go b/processor/k8sprocessor/kube/kube.go index a252b3dfa6d3f..721070fb1ccf4 100644 --- a/processor/k8sprocessor/kube/kube.go +++ b/processor/k8sprocessor/kube/kube.go @@ -31,9 +31,9 @@ const ( ignoreAnnotation string = "opentelemetry.io/k8s-processor/ignore" tagNodeName = "k8s.node.name" tagStartTime = "k8s.pod.start_time" - // Used to specify to extract metadata/labels/annotations from pod + // MetadataFromPod is used to specify to extract metadata/labels/annotations from pod MetadataFromPod = "pod" - // Used to specify to extract metadata/labels/annotations from namespace + // MetadataFromNamespace is used to specify to extract metadata/labels/annotations from namespace MetadataFromNamespace = "namespace" ) diff --git a/processor/k8sprocessor/observability/observability.go b/processor/k8sprocessor/observability/observability.go index 3d3d6b6a8ee00..efda52eb7317d 100644 --- a/processor/k8sprocessor/observability/observability.go +++ b/processor/k8sprocessor/observability/observability.go @@ -31,6 +31,9 @@ func init() { viewPodsDeleted, viewIPLookupMiss, viewPodTableSize, + viewNamespacesAdded, + viewNamespacesUpdated, + viewNamespacesDeleted, ) } diff --git a/processor/k8sprocessor/observability/observability_test.go b/processor/k8sprocessor/observability/observability_test.go index a37fbec42dfb2..7e56c06aab960 100644 --- a/processor/k8sprocessor/observability/observability_test.go +++ b/processor/k8sprocessor/observability/observability_test.go @@ -81,6 +81,18 @@ func TestMetrics(t *testing.T) { "otelsvc/k8s/ip_lookup_miss", RecordIPLookupMiss, }, + { + "otelsvc/k8s/namespace_added", + RecordNamespaceAdded, + }, + { + "otelsvc/k8s/namespace_updated", + RecordNamespaceUpdated, + }, + { + "otelsvc/k8s/namespace_deleted", + RecordNamespaceDeleted, + }, } var ( From bec83e34fc5a7aeda4d60b6a354221bd19d2bdf7 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Thu, 10 Jun 2021 17:12:02 -0700 Subject: [PATCH 09/15] review cleanup --- processor/k8sprocessor/config.go | 2 +- processor/k8sprocessor/kube/fake_informer.go | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/processor/k8sprocessor/config.go b/processor/k8sprocessor/config.go index d54f542e452ea..ee2287bbf0347 100644 --- a/processor/k8sprocessor/config.go +++ b/processor/k8sprocessor/config.go @@ -115,7 +115,7 @@ type FieldExtractConfig struct { Key string `mapstructure:"key"` Regex string `mapstructure:"regex"` // From represents the source of the labels/annotations. - // Allowed values are "pod" and "namespace". + // Allowed values are "pod" and "namespace". The default is pod. From string `mapstructure:"from"` } diff --git a/processor/k8sprocessor/kube/fake_informer.go b/processor/k8sprocessor/kube/fake_informer.go index c98a0797b8100..d071b1df9018f 100644 --- a/processor/k8sprocessor/kube/fake_informer.go +++ b/processor/k8sprocessor/kube/fake_informer.go @@ -139,15 +139,20 @@ func (f *NoOpInformer) GetController() cache.Controller { return f.NoOpController } -type NoOpController struct{} - -func (c *NoOpController) HasSynced() bool { - return true +type NoOpController struct { + hasStopped bool } -func (c *NoOpController) Run(stopCh <-chan struct{}) {} - +func (c *NoOpController) Run(stopCh <-chan struct{}) { + go func() { + <-stopCh + c.hasStopped = true + }() +} func (c *NoOpController) HasStopped() bool { + return c.hasStopped +} +func (c *NoOpController) HasSynced() bool { return true } From ce80894f86d7efb8f628f2b837333065b3b71948 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Fri, 11 Jun 2021 10:03:50 -0700 Subject: [PATCH 10/15] Add pod as default --- processor/k8sprocessor/kube/client.go | 6 ++++-- processor/k8sprocessor/kube/client_test.go | 24 ++++++++++++++++++++++ processor/k8sprocessor/options.go | 8 ++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/processor/k8sprocessor/kube/client.go b/processor/k8sprocessor/kube/client.go index e823e84f6631a..c4770eb032d65 100644 --- a/processor/k8sprocessor/kube/client.go +++ b/processor/k8sprocessor/kube/client.go @@ -309,7 +309,8 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Labels { - if r.From == MetadataFromPod { + // By default if the From field is not set for labels and annotations we want to extract them from pod + if r.From == MetadataFromPod || r.From == "" { if v, ok := pod.Labels[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } @@ -317,7 +318,8 @@ func (c *WatchClient) extractPodAttributes(pod *api_v1.Pod) map[string]string { } for _, r := range c.Rules.Annotations { - if r.From == MetadataFromPod { + // By default if the From field is not set for labels and annotations we want to extract them from pod + if r.From == MetadataFromPod || r.From == "" { if v, ok := pod.Annotations[r.Key]; ok { tags[r.Name] = c.extractField(v, r) } diff --git a/processor/k8sprocessor/kube/client_test.go b/processor/k8sprocessor/kube/client_test.go index ffefe3ced02cb..4de793811db6c 100644 --- a/processor/k8sprocessor/kube/client_test.go +++ b/processor/k8sprocessor/kube/client_test.go @@ -487,6 +487,30 @@ func TestExtractionRules(t *testing.T) { "l2": "v5", "a1": "av1", }, + }, { + // By default if the From field is not set for labels and annotations we want to extract them from pod + name: "labels-annotations-default-pod", + rules: ExtractionRules{ + Annotations: []FieldExtractionRule{{ + Name: "a1", + Key: "annotation1", + }, + }, + Labels: []FieldExtractionRule{{ + Name: "l1", + Key: "label1", + }, { + Name: "l2", + Key: "label2", + Regex: regexp.MustCompile(`k5=(?P[^\s]+)`), + }, + }, + }, + attributes: map[string]string{ + "l1": "lv1", + "l2": "v5", + "a1": "av1", + }, }, } for _, tc := range testCases { diff --git a/processor/k8sprocessor/options.go b/processor/k8sprocessor/options.go index fc1185dc553ad..0b53fa5823f0b 100644 --- a/processor/k8sprocessor/options.go +++ b/processor/k8sprocessor/options.go @@ -136,10 +136,14 @@ func extractFieldRules(fieldType string, fields ...FieldExtractConfig) ([]kube.F for _, a := range fields { name := a.TagName if name == "" { - if a.From == "pod" { + if a.From == kube.MetadataFromPod { name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) - } else if a.From == "namespace" { + } else if a.From == kube.MetadataFromNamespace { name = fmt.Sprintf("k8s.namespace.%s.%s", fieldType, a.Key) + // By default if the From field is not set for labels and annotations we want to extract them from pod + } else if a.From == "" { + name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) + a.From = kube.MetadataFromPod } } From 469e93766fdf63f6d39e01a1e0afb48ec1f68b7a Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jun 2021 08:30:01 -0700 Subject: [PATCH 11/15] fix for review --- processor/k8sprocessor/config_test.go | 6 ++++-- processor/k8sprocessor/options.go | 15 +++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/processor/k8sprocessor/config_test.go b/processor/k8sprocessor/config_test.go index 36a6f932713c2..a1ff28c79cd90 100644 --- a/processor/k8sprocessor/config_test.go +++ b/processor/k8sprocessor/config_test.go @@ -18,6 +18,8 @@ import ( "path" "testing" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sprocessor/kube" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" @@ -60,11 +62,11 @@ func TestLoadConfig(t *testing.T) { Metadata: []string{"k8s.pod.name", "k8s.pod.uid", "k8s.deployment.name", "k8s.cluster.name", "k8s.namespace.name", "k8s.node.name", "k8s.pod.start_time"}, Annotations: []FieldExtractConfig{ {TagName: "a1", Key: "annotation-one", From: "pod"}, - {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)", From: "pod"}, + {TagName: "a2", Key: "annotation-two", Regex: "field=(?P.+)", From: kube.MetadataFromPod}, }, Labels: []FieldExtractConfig{ {TagName: "l1", Key: "label1", From: "pod"}, - {TagName: "l2", Key: "label2", Regex: "field=(?P.+)", From: "pod"}, + {TagName: "l2", Key: "label2", Regex: "field=(?P.+)", From: kube.MetadataFromPod}, }, }, Filter: FilterConfig{ diff --git a/processor/k8sprocessor/options.go b/processor/k8sprocessor/options.go index 0b53fa5823f0b..6dad9ca331426 100644 --- a/processor/k8sprocessor/options.go +++ b/processor/k8sprocessor/options.go @@ -135,15 +135,22 @@ func extractFieldRules(fieldType string, fields ...FieldExtractConfig) ([]kube.F rules := []kube.FieldExtractionRule{} for _, a := range fields { name := a.TagName + + switch a.From { + // By default if the From field is not set for labels and annotations we want to extract them from pod + case "", kube.MetadataFromPod: + a.From = kube.MetadataFromPod + case kube.MetadataFromNamespace: + a.From = kube.MetadataFromNamespace + default: + return rules, fmt.Errorf("from must contain exactly one value like pod or namespace") + } + if name == "" { if a.From == kube.MetadataFromPod { name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) } else if a.From == kube.MetadataFromNamespace { name = fmt.Sprintf("k8s.namespace.%s.%s", fieldType, a.Key) - // By default if the From field is not set for labels and annotations we want to extract them from pod - } else if a.From == "" { - name = fmt.Sprintf("k8s.pod.%s.%s", fieldType, a.Key) - a.From = kube.MetadataFromPod } } From aa9b715c2af55f0e66e3b0f7f31bee607008bba2 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jun 2021 08:58:25 -0700 Subject: [PATCH 12/15] fix lint --- processor/k8sprocessor/config_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/processor/k8sprocessor/config_test.go b/processor/k8sprocessor/config_test.go index a1ff28c79cd90..42a61a365134e 100644 --- a/processor/k8sprocessor/config_test.go +++ b/processor/k8sprocessor/config_test.go @@ -18,8 +18,6 @@ import ( "path" "testing" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sprocessor/kube" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" @@ -28,6 +26,7 @@ import ( "go.opentelemetry.io/collector/config/configtest" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sprocessor/kube" ) func TestLoadConfig(t *testing.T) { From 9c8e775d7e570f22d9c58f51338f74da49b4fcc0 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jun 2021 13:09:49 -0700 Subject: [PATCH 13/15] reveiw cleanup --- processor/k8sprocessor/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processor/k8sprocessor/options.go b/processor/k8sprocessor/options.go index 6dad9ca331426..dbddce51f9731 100644 --- a/processor/k8sprocessor/options.go +++ b/processor/k8sprocessor/options.go @@ -143,7 +143,7 @@ func extractFieldRules(fieldType string, fields ...FieldExtractConfig) ([]kube.F case kube.MetadataFromNamespace: a.From = kube.MetadataFromNamespace default: - return rules, fmt.Errorf("from must contain exactly one value like pod or namespace") + return rules, fmt.Errorf("%s is not a valid choice for From. Must be one of: pod, namespace", a.From) } if name == "" { From c00a184ca18d1b0a49c97ef420f2d9911eba76f7 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jun 2021 17:22:09 -0700 Subject: [PATCH 14/15] Add doc --- processor/k8sprocessor/doc.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/processor/k8sprocessor/doc.go b/processor/k8sprocessor/doc.go index 3e9c2aa0f43ad..14a9c0668e05a 100644 --- a/processor/k8sprocessor/doc.go +++ b/processor/k8sprocessor/doc.go @@ -40,6 +40,33 @@ // // If Pod association rules are not configured resources are associated with metadata only by connection's IP Address. // +// +//The k8sprocessor can be used for automatic tagging of spans, metrics and logs with k8s labels and annotations from pods and namespaces. +//The config for associating the data passing through the processor (spans, metrics and logs) with specific Pod/Namespace +//annotations/labels are configured via "annotations" and "labels" keys. +//It represents a list of annotations/labels that are extracted from pods/namespaces and added to spans, metrics and logs. +//Each item is specified as a config of tag_name (representing the tag name to tag the spans with), +//key (representing the key used to extract value) and from (representing the kubernetes object used to extract the value). +//"from" has only two possible values "pod" and "namespace" and defaults to "pod" if none is specified. +// +//A few examples to use this config are as follows: +//annotations: +// - tag_name: a1 # extracts value of annotation from pods with key `annotation-one` and inserts it as a tag with key `a1` +// key: annotation-one +// from: pod +// - tag_name: a2 # extracts value of annotation from namespaces with key `annotation-two` with regexp and inserts it as a tag with key `a2` +// key: annotation-two +// regex: field=(?P.+) +// from: namespace +//labels: +// - tag_name: l1 # extracts value of label from namespaces with key `label1` and inserts it as a tag with key `l1` +// key: label1 +// from: namespace +// - tag_name: l2 # extracts value of label from pods with key `label1` with regexp and inserts it as a tag with key `l2` +// key: label2 +// regex: field=(?P.+) +// from: pod + // RBAC // // TODO: mention the required RBAC rules. From 341a2b0d13ae7e3419880df1dcaa33e14788da56 Mon Sep 17 00:00:00 2001 From: Chaitanya Phalak Date: Mon, 14 Jun 2021 17:25:40 -0700 Subject: [PATCH 15/15] cleanup --- processor/k8sprocessor/doc.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/processor/k8sprocessor/doc.go b/processor/k8sprocessor/doc.go index 14a9c0668e05a..9340f60147091 100644 --- a/processor/k8sprocessor/doc.go +++ b/processor/k8sprocessor/doc.go @@ -42,30 +42,29 @@ // // //The k8sprocessor can be used for automatic tagging of spans, metrics and logs with k8s labels and annotations from pods and namespaces. -//The config for associating the data passing through the processor (spans, metrics and logs) with specific Pod/Namespace -//annotations/labels are configured via "annotations" and "labels" keys. -//It represents a list of annotations/labels that are extracted from pods/namespaces and added to spans, metrics and logs. +//The config for associating the data passing through the processor (spans, metrics and logs) with specific Pod/Namespace annotations/labels is configured via "annotations" and "labels" keys. +//This config represents a list of annotations/labels that are extracted from pods/namespaces and added to spans, metrics and logs. //Each item is specified as a config of tag_name (representing the tag name to tag the spans with), //key (representing the key used to extract value) and from (representing the kubernetes object used to extract the value). -//"from" has only two possible values "pod" and "namespace" and defaults to "pod" if none is specified. +//The "from" field has only two possible values "pod" and "namespace" and defaults to "pod" if none is specified. // //A few examples to use this config are as follows: //annotations: // - tag_name: a1 # extracts value of annotation from pods with key `annotation-one` and inserts it as a tag with key `a1` -// key: annotation-one -// from: pod +// key: annotation-one +// from: pod // - tag_name: a2 # extracts value of annotation from namespaces with key `annotation-two` with regexp and inserts it as a tag with key `a2` -// key: annotation-two -// regex: field=(?P.+) -// from: namespace +// key: annotation-two +// regex: field=(?P.+) +// from: namespace //labels: // - tag_name: l1 # extracts value of label from namespaces with key `label1` and inserts it as a tag with key `l1` -// key: label1 -// from: namespace +// key: label1 +// from: namespace // - tag_name: l2 # extracts value of label from pods with key `label1` with regexp and inserts it as a tag with key `l2` -// key: label2 -// regex: field=(?P.+) -// from: pod +// key: label2 +// regex: field=(?P.+) +// from: pod // RBAC //