Skip to content

Commit beb21ae

Browse files
author
Abhishek Agarwal
authored
fix(BD-labels):add device attributes and location details as bd labels (#618)
* added device attributes and location details as bd labels * node attributes to be added only when env variable specifies to do so * add unit test * changes updated with configmap modification * include quotes in the configmap fields* bump chart version * removed meta-configs from default config values and changed label validation logic * ran make-manifests to generate operator yaml Signed-off-by: Abhishek Agarwal <[email protected]>
1 parent 8e36a0d commit beb21ae

18 files changed

+253
-177
lines changed

cmd/ndm_daemonset/controller/blockdevice.go

+122-3
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,17 @@ limitations under the License.
1717
package controller
1818

1919
import (
20+
"fmt"
21+
"regexp"
22+
"strings"
23+
24+
"k8s.io/apimachinery/pkg/util/validation"
25+
"k8s.io/client-go/util/jsonpath"
26+
"k8s.io/klog"
27+
2028
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
2129
bd "github.com/openebs/node-disk-manager/blockdevice"
30+
2231
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2332
)
2433

@@ -67,13 +76,116 @@ type FSInfo struct {
6776

6877
// ToDevice convert deviceInfo struct to api.BlockDevice
6978
// type which will be pushed to etcd
70-
func (di *DeviceInfo) ToDevice() apis.BlockDevice {
79+
func (di *DeviceInfo) ToDevice(controller *Controller) (apis.BlockDevice, error) {
7180
blockDevice := apis.BlockDevice{}
7281
blockDevice.Spec = di.getDeviceSpec()
7382
blockDevice.ObjectMeta = di.getObjectMeta()
7483
blockDevice.TypeMeta = di.getTypeMeta()
7584
blockDevice.Status = di.getStatus()
76-
return blockDevice
85+
err := addBdLabels(&blockDevice, controller)
86+
if err != nil {
87+
return blockDevice, fmt.Errorf("error in adding labels to the blockdevice: %v", err)
88+
}
89+
return blockDevice, nil
90+
}
91+
92+
// addBdLabels add labels to block device that may be helpful for filtering the block device
93+
// based on some generic attributes like drive-type, model, vendor etc.
94+
func addBdLabels(bd *apis.BlockDevice, ctrl *Controller) error {
95+
var JsonPathFields []string
96+
97+
// get the labels to be added from the configmap
98+
if ctrl.NDMConfig != nil {
99+
for _, metaConfig := range ctrl.NDMConfig.MetaConfigs {
100+
if metaConfig.Key == deviceLabelsKey {
101+
JsonPathFields = strings.Split(metaConfig.Type, ",")
102+
}
103+
}
104+
}
105+
106+
if len(JsonPathFields) > 0 {
107+
for _, jsonPath := range JsonPathFields {
108+
// Parse jsonpath
109+
fields, err := RelaxedJSONPathExpression(strings.TrimSpace(jsonPath))
110+
if err != nil {
111+
klog.Errorf("Error converting into a valid jsonpath expression: %+v", err)
112+
return err
113+
}
114+
115+
j := jsonpath.New(jsonPath)
116+
if err := j.Parse(fields); err != nil {
117+
klog.Errorf("Error parsing jsonpath: %s, error: %+v", fields, err)
118+
return err
119+
}
120+
121+
values, err := j.FindResults(bd)
122+
if err != nil {
123+
klog.Errorf("Error finding results for jsonpath: %s, error: %+v", fields, err)
124+
return err
125+
}
126+
127+
valueStrings := []string{}
128+
var jsonPathFieldValue string
129+
130+
if len(values) > 0 || len(values[0]) > 0 {
131+
for arrIx := range values {
132+
for valIx := range values[arrIx] {
133+
valueStrings = append(valueStrings, fmt.Sprintf("%v", values[arrIx][valIx].Interface()))
134+
}
135+
}
136+
137+
// convert the string array into a single string
138+
jsonPathFieldValue = strings.Join(valueStrings, ",")
139+
jsonPathFieldValue = strings.TrimSuffix(jsonPathFieldValue, ",")
140+
141+
// the above string formed should adhere to k8s label rules inorder for it to be
142+
// used as a label value for blockdevice object.
143+
// Check this link for more info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
144+
errs := validation.IsValidLabelValue(jsonPathFieldValue)
145+
if len(errs) > 0 {
146+
return fmt.Errorf("invalid Label found. Error: {%s}", strings.Join(errs, ","))
147+
}
148+
149+
jsonPathFields := strings.Split(jsonPath, ".")
150+
if len(jsonPathFields) > 0 && jsonPathFields[len(jsonPathFields)-1] != "" && jsonPathFieldValue != "" {
151+
bd.Labels[NDMLabelPrefix+jsonPathFields[len(jsonPathFields)-1]] = jsonPathFieldValue
152+
}
153+
}
154+
}
155+
klog.V(4).Infof("successfully added device labels")
156+
}
157+
return nil
158+
}
159+
160+
// RelaxedJSONPathExpression attempts to be flexible with JSONPath expressions, it accepts:
161+
// * metadata.name (no leading '.' or curly braces '{...}'
162+
// * {metadata.name} (no leading '.')
163+
// * .metadata.name (no curly braces '{...}')
164+
// * {.metadata.name} (complete expression)
165+
// And transforms them all into a valid jsonpath expression:
166+
// {.metadata.name}
167+
// NOTE: This code has been referenced from kubernetes kubectl github repo.
168+
// Ref: https://github.com/kubernetes/kubectl/blob/caeb9274868c57d8a320014290cc7e3d1bcb9e46/pkg/cmd/get/customcolumn.go#L47
169+
func RelaxedJSONPathExpression(pathExpression string) (string, error) {
170+
var jsonRegexp = regexp.MustCompile(`^\{\.?([^{}]+)\}$|^\.?([^{}]+)$`)
171+
172+
if len(pathExpression) == 0 {
173+
return pathExpression, nil
174+
}
175+
submatches := jsonRegexp.FindStringSubmatch(pathExpression)
176+
if submatches == nil {
177+
return "", fmt.Errorf("unexpected path string, expected a 'name1.name2' or '.name1.name2' or '{name1.name2}' or '{.name1.name2}'")
178+
}
179+
if len(submatches) != 3 {
180+
return "", fmt.Errorf("unexpected submatch list: %v", submatches)
181+
}
182+
var fieldSpec string
183+
if len(submatches[1]) != 0 {
184+
fieldSpec = submatches[1]
185+
} else {
186+
fieldSpec = submatches[2]
187+
}
188+
return fmt.Sprintf("{.%s}", fieldSpec), nil
77189
}
78190

79191
// getObjectMeta returns ObjectMeta struct which contains
@@ -85,7 +197,14 @@ func (di *DeviceInfo) getObjectMeta() metav1.ObjectMeta {
85197
Annotations: make(map[string]string),
86198
Name: di.UUID,
87199
}
88-
objectMeta.Labels[KubernetesHostNameLabel] = di.NodeAttributes[HostNameKey]
200+
//objectMeta.Labels[KubernetesHostNameLabel] = di.NodeAttributes[HostNameKey]
201+
for k, v := range di.NodeAttributes {
202+
if k == HostNameKey {
203+
objectMeta.Labels[KubernetesHostNameLabel] = v
204+
} else {
205+
objectMeta.Labels[k] = v
206+
}
207+
}
89208
objectMeta.Labels[NDMDeviceTypeKey] = NDMDefaultDeviceType
90209
objectMeta.Labels[NDMManagedKey] = TrueString
91210
// adding custom labels

cmd/ndm_daemonset/controller/blockdevicestore.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,13 @@ import (
2222
"k8s.io/apimachinery/pkg/labels"
2323
"k8s.io/apimachinery/pkg/selection"
2424

25-
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
26-
"github.com/openebs/node-disk-manager/pkg/util"
2725
"k8s.io/apimachinery/pkg/api/errors"
2826
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2927
"k8s.io/klog"
3028
"sigs.k8s.io/controller-runtime/pkg/client"
29+
30+
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
31+
"github.com/openebs/node-disk-manager/pkg/util"
3132
)
3233

3334
// CreateBlockDevice creates the BlockDevice resource in etcd
@@ -250,7 +251,11 @@ func (c *Controller) DeactivateStaleBlockDeviceResource(devices []string) {
250251
func (c *Controller) PushBlockDeviceResource(oldBlockDevice *apis.BlockDevice,
251252
deviceDetails *DeviceInfo) error {
252253
deviceDetails.NodeAttributes = c.NodeAttributes
253-
deviceAPI := deviceDetails.ToDevice()
254+
deviceAPI, err := deviceDetails.ToDevice(c)
255+
if err != nil {
256+
klog.Error("Failed to create a block device resource CR, Error: ", err)
257+
return err
258+
}
254259
if oldBlockDevice != nil {
255260
return c.UpdateBlockDevice(deviceAPI, oldBlockDevice)
256261
}

cmd/ndm_daemonset/controller/blockdevicestore_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ package controller
1919
import (
2020
"testing"
2121

22-
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
2322
"github.com/stretchr/testify/assert"
2423
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
25+
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
2526
)
2627

2728
// mockEmptyDeviceCr returns BlockDevice object with minimum attributes it is used in unit test cases.

cmd/ndm_daemonset/controller/controller.go

+47-10
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,21 @@ import (
2121
"errors"
2222
"fmt"
2323
"os"
24+
"strings"
2425
"sync"
2526
"time"
2627

27-
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
28-
"github.com/openebs/node-disk-manager/blockdevice"
29-
3028
v1 "k8s.io/api/core/v1"
3129
"k8s.io/client-go/rest"
3230
"k8s.io/klog"
3331
"sigs.k8s.io/controller-runtime/pkg/client"
3432
"sigs.k8s.io/controller-runtime/pkg/client/config"
3533
"sigs.k8s.io/controller-runtime/pkg/manager"
3634
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
35+
36+
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
37+
"github.com/openebs/node-disk-manager/blockdevice"
38+
"github.com/openebs/node-disk-manager/pkg/util"
3739
)
3840

3941
const (
@@ -49,6 +51,8 @@ const (
4951
openEBSLabelPrefix = "openebs.io/"
5052
// HostNameKey is the key for hostname
5153
HostNameKey = "hostname"
54+
// LabelTypeNode is the key present in an ENV variable(containing comma separated string value)
55+
LabelTypeNode = "node-attributes"
5256
// NodeNameKey is the node name label prefix
5357
NodeNameKey = "nodename"
5458
// KubernetesHostNameLabel is the hostname label used by k8s
@@ -70,11 +74,17 @@ const (
7074
// NDMUnknown is constant for resource unknown status.
7175
NDMUnknown = "Unknown"
7276
// NDMDeviceTypeKey specifies the block device type
73-
NDMDeviceTypeKey = "ndm.io/blockdevice-type"
77+
NDMDeviceTypeKey = NDMLabelPrefix + "blockdevice-type"
7478
// NDMManagedKey specifies blockdevice cr should be managed by ndm or not.
75-
NDMManagedKey = "ndm.io/managed"
79+
NDMManagedKey = NDMLabelPrefix + "managed"
80+
// nodeLabelsKey is the meta config key name for adding node labels
81+
nodeLabelsKey = "node-labels"
82+
// deviceLabelsKey is the meta config key name for adding device labels
83+
deviceLabelsKey = "device-labels"
84+
// NDMLabelPrefix is the label prefix for ndm labels
85+
NDMLabelPrefix = "ndm.io/"
7686
// NDMZpoolName specifies the zpool name
77-
NDMZpoolName = "ndm.io/zpool-name"
87+
NDMZpoolName = NDMLabelPrefix + "zpool-name"
7888
)
7989

8090
const (
@@ -189,16 +199,16 @@ func (c *Controller) setNodeAttributes() error {
189199
}
190200
c.NodeAttributes[NodeNameKey] = nodeName
191201

192-
// set the hostname label
193-
if err = c.setHostName(); err != nil {
202+
// set the node labels
203+
if err = c.setNodeLabels(); err != nil {
194204
return fmt.Errorf("unable to set node attributes:%v", err)
195205
}
196206
return nil
197207
}
198208

199-
// setHostName set NodeAttribute field in Controller struct
209+
// setNodeLabels set NodeAttribute field in Controller struct
200210
// from the labels in node object
201-
func (c *Controller) setHostName() error {
211+
func (c *Controller) setNodeLabels() error {
202212
nodeName := c.NodeAttributes[NodeNameKey]
203213
// get the node object and fetch the hostname label from the
204214
// node object
@@ -215,6 +225,33 @@ func (c *Controller) setHostName() error {
215225
} else {
216226
c.NodeAttributes[HostNameKey] = hostName
217227
}
228+
229+
var labelPattern []string
230+
231+
// Get the list of node label patterns to be added from the configmap
232+
if c.NDMConfig != nil {
233+
for _, metaConfig := range c.NDMConfig.MetaConfigs {
234+
if metaConfig.Key == nodeLabelsKey {
235+
labelPattern = strings.Split(metaConfig.Pattern, ",")
236+
}
237+
}
238+
}
239+
240+
if len(labelPattern) > 0 {
241+
// Add only those node labels that matches the pattern specified in the
242+
// node-labels meta config
243+
for key, value := range node.Labels {
244+
for _, pattern := range labelPattern {
245+
if util.IsMatchRegex(pattern, key) {
246+
if value != "" {
247+
c.NodeAttributes[key] = value
248+
break
249+
}
250+
}
251+
}
252+
}
253+
}
254+
218255
return nil
219256
}
220257

cmd/ndm_daemonset/controller/mockdata_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@ import (
2020
"fmt"
2121
"testing"
2222

23-
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
2423
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2524
"k8s.io/client-go/kubernetes/scheme"
2625
"sigs.k8s.io/controller-runtime/pkg/client"
2726
ndmFakeClientset "sigs.k8s.io/controller-runtime/pkg/client/fake"
27+
28+
apis "github.com/openebs/node-disk-manager/api/v1alpha1"
2829
)
2930

3031
const (

cmd/ndm_daemonset/controller/ndmconfig.go

+9
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ type NodeDiskManagerConfig struct {
3636
FilterConfigs []FilterConfig `json:"filterconfigs"` // FilterConfigs contains configs of Filters
3737
// TagConfigs contains configs for tags
3838
TagConfigs []TagConfig `json:"tagconfigs"`
39+
// MetaConfig contains configs for device labels
40+
MetaConfigs []MetaConfig `json:"metaconfigs"`
3941
}
4042

4143
// ProbeConfig contains configs of Probe
@@ -61,6 +63,13 @@ type TagConfig struct {
6163
TagName string `json:"tag"`
6264
}
6365

66+
type MetaConfig struct {
67+
Key string `json:"key"`
68+
Name string `json:"name"`
69+
Type string `json:"type"`
70+
Pattern string `json:"pattern"`
71+
}
72+
6473
// SetNDMConfig sets config for probes and filters which user provides via configmap. If
6574
// no configmap present then ndm will load default config for each probes and filters.
6675
func (c *Controller) SetNDMConfig(opts NDMOptions) {

cmd/ndm_daemonset/controller/sparsefilegenerator.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,13 @@ func (c *Controller) MarkSparseBlockDeviceStateActive(sparseFile string, sparseF
220220

221221
//If a BlockDevice CR already exits, update it. If not create a new one.
222222
klog.Info("Updating the BlockDevice CR for Sparse file: ", BlockDeviceDetails.UUID)
223-
err = c.CreateBlockDevice(BlockDeviceDetails.ToDevice())
223+
bd, err := BlockDeviceDetails.ToDevice(c)
224+
if err != nil {
225+
klog.Error("Failed to create a block device resource CR, Error: ", err)
226+
return
227+
}
228+
229+
err = c.CreateBlockDevice(bd)
224230
if err != nil {
225231
klog.Error("Failed to create a block device resource in etcd, Error: ", err)
226232
}

cmd/ndm_daemonset/probe/addhandler.go

+11-5
Original file line numberDiff line numberDiff line change
@@ -384,10 +384,14 @@ func (pe *ProbeEvent) deviceInUseByZFSLocalPV(bd blockdevice.BlockDevice, bdAPIL
384384
bd.UUID = uuid
385385

386386
deviceInfo := pe.Controller.NewDeviceInfoFromBlockDevice(&bd)
387-
bdAPI := deviceInfo.ToDevice()
387+
bdAPI, err := deviceInfo.ToDevice(pe.Controller)
388+
if err != nil {
389+
klog.Error("Failed to create a block device resource CR, Error: ", err)
390+
return true, err
391+
}
388392
bdAPI.Labels[kubernetes.BlockDeviceTagLabel] = string(blockdevice.ZFSLocalPV)
389393

390-
err := pe.Controller.CreateBlockDevice(bdAPI)
394+
err = pe.Controller.CreateBlockDevice(bdAPI)
391395
if err != nil {
392396
klog.Errorf("unable to push %s (%s) to etcd", bd.UUID, bd.DevPath)
393397
return false, err
@@ -594,11 +598,13 @@ func (pe *ProbeEvent) createOrUpdateWithPartitionUUID(bd blockdevice.BlockDevice
594598
// createOrUpdateWithAnnotation creates or updates a resource in etcd with given annotation.
595599
func (pe *ProbeEvent) createOrUpdateWithAnnotation(annotation map[string]string, bd blockdevice.BlockDevice, existingBD *apis.BlockDevice) error {
596600
deviceInfo := pe.Controller.NewDeviceInfoFromBlockDevice(&bd)
597-
bdAPI := deviceInfo.ToDevice()
598-
601+
bdAPI, err := deviceInfo.ToDevice(pe.Controller)
602+
if err != nil {
603+
klog.Error("Failed to create a block device resource CR, Error: ", err)
604+
return err
605+
}
599606
bdAPI.Annotations = annotation
600607

601-
var err error
602608
if existingBD != nil {
603609
err = pe.Controller.UpdateBlockDevice(bdAPI, existingBD)
604610
} else {

0 commit comments

Comments
 (0)