Skip to content

Commit 2701553

Browse files
sethsamuelsethsamuel
authored
Add extra_dbm template variable to Aurora autodiscovery (#36706)
1 parent 43eb713 commit 2701553

File tree

11 files changed

+151
-14
lines changed

11 files changed

+151
-14
lines changed

‎cmd/agent/dist/conf.d/postgres.d/conf_aws_aurora.yaml.default

+6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ instances:
1414
#
1515
port: "%%port%%"
1616

17+
## @param dbm - string - optional - default: false
18+
## Enable DBM monitoring. The value of the template variable matches the value of the configured `dbm_tag` on the instance,
19+
## for example, `datadoghq.com/dbm:true` to enable DBM.
20+
#
21+
dbm: "%%extra_dbm%%"
22+
1723
## This block defines the configuration for AWS RDS and Aurora instances.
1824
##
1925
## Complete this section if you have installed the Datadog AWS Integration

‎comp/core/autodiscovery/listeners/dbm_aurora.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func (l *DBMAuroraListener) discoverAuroraClusters() {
135135
log.Debugf("no aurora clusters found with provided tags %v", l.config.Tags)
136136
return
137137
}
138-
auroraCluster, err := l.awsRdsClient.GetAuroraClusterEndpoints(ctx, ids)
138+
auroraCluster, err := l.awsRdsClient.GetAuroraClusterEndpoints(ctx, ids, l.config.DbmTag)
139139
if err != nil {
140140
_ = log.Error(err)
141141
return
@@ -271,6 +271,8 @@ func (d *DBMAuroraService) HasFilter(containers.FilterType) bool {
271271
// GetExtraConfig parses the template variables with the extra_ prefix and returns the value
272272
func (d *DBMAuroraService) GetExtraConfig(key string) (string, error) {
273273
switch key {
274+
case "dbm":
275+
return strconv.FormatBool(d.instance.DbmEnabled), nil
274276
case "region":
275277
return d.region, nil
276278
case "managed_authentication_enabled":

‎comp/core/autodiscovery/listeners/dbm_aurora_test.go

+16-5
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
type mockRDSClientConfigurer func(k *aws.MockRDSClient)
2626

2727
const defaultClusterTag = "datadoghq.com/scrape:true"
28+
const defaultDbmTag = "datadoghq.com/dbm:true"
2829

2930
func TestDBMAuroraListener(t *testing.T) {
3031
testCases := []struct {
@@ -42,6 +43,7 @@ func TestDBMAuroraListener(t *testing.T) {
4243
QueryTimeout: 1,
4344
Region: "us-east-1",
4445
Tags: []string{defaultClusterTag},
46+
DbmTag: defaultDbmTag,
4547
},
4648
numDiscoveryIntervals: 0,
4749
rdsClientConfigurer: func(k *aws.MockRDSClient) {
@@ -61,13 +63,14 @@ func TestDBMAuroraListener(t *testing.T) {
6163
QueryTimeout: 1,
6264
Region: "us-east-1",
6365
Tags: []string{defaultClusterTag},
66+
DbmTag: defaultDbmTag,
6467
},
6568
numDiscoveryIntervals: 0,
6669
rdsClientConfigurer: func(k *aws.MockRDSClient) {
6770
gomock.InOrder(
6871
k.EXPECT().GetAuroraClustersFromTags(gomock.Any(), []string{defaultClusterTag}).Return([]string{"my-cluster-1"}, nil).AnyTimes(),
69-
k.EXPECT().GetAuroraClusterEndpoints(contextWithTimeout(1*time.Second), []string{"my-cluster-1"}).DoAndReturn(
70-
func(ctx context.Context, _ []string) (map[string]*aws.AuroraCluster, error) {
72+
k.EXPECT().GetAuroraClusterEndpoints(contextWithTimeout(1*time.Second), []string{"my-cluster-1"}, defaultDbmTag).DoAndReturn(
73+
func(ctx context.Context, _ []string, _ string) (map[string]*aws.AuroraCluster, error) {
7174
<-ctx.Done()
7275
return nil, ctx.Err()
7376
}).AnyTimes(),
@@ -83,6 +86,7 @@ func TestDBMAuroraListener(t *testing.T) {
8386
DiscoveryInterval: 1,
8487
Region: "us-east-1",
8588
Tags: []string{defaultClusterTag},
89+
DbmTag: defaultDbmTag,
8690
},
8791
numDiscoveryIntervals: 0,
8892
rdsClientConfigurer: func(k *aws.MockRDSClient) {
@@ -97,12 +101,13 @@ func TestDBMAuroraListener(t *testing.T) {
97101
DiscoveryInterval: 1,
98102
Region: "us-east-1",
99103
Tags: []string{defaultClusterTag},
104+
DbmTag: defaultDbmTag,
100105
},
101106
numDiscoveryIntervals: 0,
102107
rdsClientConfigurer: func(k *aws.MockRDSClient) {
103108
gomock.InOrder(
104109
k.EXPECT().GetAuroraClustersFromTags(gomock.Any(), []string{defaultClusterTag}).Return([]string{"my-cluster-1"}, nil).AnyTimes(),
105-
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}).Return(nil, errors.New("big bad error")).AnyTimes(),
110+
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}, defaultDbmTag).Return(nil, errors.New("big bad error")).AnyTimes(),
106111
)
107112
},
108113
expectedServices: []*DBMAuroraService{},
@@ -114,11 +119,12 @@ func TestDBMAuroraListener(t *testing.T) {
114119
DiscoveryInterval: 1,
115120
Region: "us-east-1",
116121
Tags: []string{defaultClusterTag},
122+
DbmTag: defaultDbmTag,
117123
},
118124
numDiscoveryIntervals: 1,
119125
rdsClientConfigurer: func(k *aws.MockRDSClient) {
120126
k.EXPECT().GetAuroraClustersFromTags(gomock.Any(), []string{defaultClusterTag}).Return([]string{"my-cluster-1"}, nil).AnyTimes()
121-
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}).Return(
127+
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}, defaultDbmTag).Return(
122128
map[string]*aws.AuroraCluster{
123129
"my-cluster-1": {
124130
Instances: []*aws.Instance{
@@ -127,6 +133,7 @@ func TestDBMAuroraListener(t *testing.T) {
127133
Port: 5432,
128134
IamEnabled: true,
129135
Engine: "aurora-postgresql",
136+
DbmEnabled: true,
130137
},
131138
},
132139
},
@@ -144,6 +151,7 @@ func TestDBMAuroraListener(t *testing.T) {
144151
Port: 5432,
145152
IamEnabled: true,
146153
Engine: "aurora-postgresql",
154+
DbmEnabled: true,
147155
},
148156
},
149157
},
@@ -155,11 +163,12 @@ func TestDBMAuroraListener(t *testing.T) {
155163
DiscoveryInterval: 1,
156164
Region: "us-east-1",
157165
Tags: []string{defaultClusterTag},
166+
DbmTag: defaultDbmTag,
158167
},
159168
numDiscoveryIntervals: 1,
160169
rdsClientConfigurer: func(k *aws.MockRDSClient) {
161170
k.EXPECT().GetAuroraClustersFromTags(gomock.Any(), []string{defaultClusterTag}).Return([]string{"my-cluster-1"}, nil).AnyTimes()
162-
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}).Return(
171+
k.EXPECT().GetAuroraClusterEndpoints(gomock.Any(), []string{"my-cluster-1"}, defaultDbmTag).Return(
163172
map[string]*aws.AuroraCluster{
164173
"my-cluster-1": {
165174
Instances: []*aws.Instance{
@@ -297,13 +306,15 @@ func TestGetExtraConfig(t *testing.T) {
297306
IamEnabled: true,
298307
Engine: "aurora-postgresql",
299308
DbName: "app",
309+
DbmEnabled: true,
300310
},
301311
},
302312
expectedExtra: map[string]string{
303313
"dbname": "app",
304314
"region": "us-east-1",
305315
"managed_authentication_enabled": "true",
306316
"dbclusteridentifier": "my-cluster-1",
317+
"dbm": "true",
307318
},
308319
},
309320
}

‎pkg/config/setup/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -1594,6 +1594,7 @@ func logsagent(config pkgconfigmodel.Setup) {
15941594
config.BindEnvAndSetDefault("database_monitoring.autodiscovery.aurora.region", "")
15951595
config.BindEnvAndSetDefault("database_monitoring.autodiscovery.aurora.query_timeout", 10)
15961596
config.BindEnvAndSetDefault("database_monitoring.autodiscovery.aurora.tags", []string{"datadoghq.com/scrape:true"})
1597+
config.BindEnvAndSetDefault("database_monitoring.autodiscovery.aurora.dbm_tag", "datadoghq.com/dbm:true")
15971598

15981599
config.BindEnvAndSetDefault("logs_config.dd_port", 10516)
15991600
config.BindEnvAndSetDefault("logs_config.dev_mode_use_proto", true)

‎pkg/config/setup/config_test.go

+6
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,7 @@ func TestDatabaseMonitoringAurora(t *testing.T) {
485485
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.discovery_interval"), 300)
486486
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.query_timeout"), 10)
487487
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.tags"), []string{"datadoghq.com/scrape:true"})
488+
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.dbm_tag"), "datadoghq.com/dbm:true")
488489
assert.Equal(t, config.GetString("database_monitoring.autodiscovery.aurora.region"), "")
489490
},
490491
},
@@ -502,19 +503,22 @@ func TestDatabaseMonitoringAurora(t *testing.T) {
502503
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.query_timeout"), 1)
503504
assert.Equal(t, config.GetString("database_monitoring.autodiscovery.aurora.region"), "us-west-2")
504505
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.tags"), []string{"datadoghq.com/scrape:true"})
506+
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.dbm_tag"), "datadoghq.com/dbm:true")
505507
},
506508
},
507509
{
508510
name: "auto discovery tag configuration set through DD env vars",
509511
setup: func(t *testing.T, _ pkgconfigmodel.Config) {
510512
t.Setenv("DD_DATABASE_MONITORING_AUTODISCOVERY_AURORA_ENABLED", "true")
511513
t.Setenv("DD_DATABASE_MONITORING_AUTODISCOVERY_AURORA_TAGS", "foo:bar other:tag")
514+
t.Setenv("DD_DATABASE_MONITORING_AUTODISCOVERY_AURORA_DBM_TAG", "usedbm")
512515
},
513516
tests: func(t *testing.T, config pkgconfigmodel.Config) {
514517
assert.True(t, config.GetBool("database_monitoring.autodiscovery.aurora.enabled"))
515518
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.discovery_interval"), 300)
516519
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.query_timeout"), 10)
517520
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.tags"), []string{"foo:bar", "other:tag"})
521+
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.dbm_tag"), "usedbm")
518522
},
519523
},
520524
{
@@ -536,12 +540,14 @@ func TestDatabaseMonitoringAurora(t *testing.T) {
536540
config.SetWithoutSource("database_monitoring.autodiscovery.aurora.discovery_interval", 10)
537541
config.SetWithoutSource("database_monitoring.autodiscovery.aurora.query_timeout", 4)
538542
config.SetWithoutSource("database_monitoring.autodiscovery.aurora.tags", []string{"foo:bar"})
543+
config.SetWithoutSource("database_monitoring.autodiscovery.aurora.dbm_tag", "usedbm")
539544
},
540545
tests: func(t *testing.T, config pkgconfigmodel.Config) {
541546
assert.True(t, config.GetBool("database_monitoring.autodiscovery.aurora.enabled"))
542547
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.discovery_interval"), 10)
543548
assert.Equal(t, config.GetInt("database_monitoring.autodiscovery.aurora.query_timeout"), 4)
544549
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.tags"), []string{"foo:bar"})
550+
assert.Equal(t, config.Get("database_monitoring.autodiscovery.aurora.dbm_tag"), "usedbm")
545551
},
546552
},
547553
}

‎pkg/databasemonitoring/aws/aurora.go

+15-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ type Instance struct {
3333
IamEnabled bool
3434
Engine string
3535
DbName string
36+
DbmEnabled bool
3637
}
3738

3839
const (
@@ -42,7 +43,7 @@ const (
4243

4344
// GetAuroraClusterEndpoints queries an AWS account for the endpoints of an Aurora cluster
4445
// requires the dbClusterIdentifier for the cluster
45-
func (c *Client) GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentifiers []string) (map[string]*AuroraCluster, error) {
46+
func (c *Client) GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentifiers []string, dbmTag string) (map[string]*AuroraCluster, error) {
4647
if len(dbClusterIdentifiers) == 0 {
4748
return nil, fmt.Errorf("at least one database cluster identifier is required")
4849
}
@@ -98,6 +99,19 @@ func (c *Client) GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentif
9899
return nil, fmt.Errorf("engine is nil for instance %s", clusterID)
99100
}
100101
}
102+
for _, tag := range db.TagList {
103+
tagString := ""
104+
if tag.Key != nil {
105+
tagString += *tag.Key
106+
}
107+
if tag.Value != nil {
108+
tagString += ":" + *tag.Value
109+
}
110+
if tagString == dbmTag {
111+
instance.DbmEnabled = true
112+
break
113+
}
114+
}
101115
if _, ok := clusters[*db.DBClusterIdentifier]; !ok {
102116
clusters[*db.DBClusterIdentifier] = &AuroraCluster{
103117
Instances: make([]*Instance, 0),

‎pkg/databasemonitoring/aws/aurora_test.go

+85-2
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,14 @@ import (
2121

2222
type mockrdsServiceConfigurer func(k *MockrdsService)
2323

24+
const defaultDbmTag = "datadoghq.com/dbm:true"
25+
2426
func TestGetAuroraClusterEndpoints(t *testing.T) {
2527
testCases := []struct {
2628
name string
2729
configureClient mockrdsServiceConfigurer
2830
clusterIDs []string
31+
dbmTag *string
2932
expectedAuroraClusterEndpoints map[string]*AuroraCluster
3033
expectedErr error
3134
}{
@@ -87,6 +90,80 @@ func TestGetAuroraClusterEndpoints(t *testing.T) {
8790
},
8891
},
8992
},
93+
{
94+
name: "single cluster id returns single endpoint from API with custom DBM tag",
95+
configureClient: func(k *MockrdsService) {
96+
k.EXPECT().DescribeDBInstances(gomock.Any(), gomock.Any()).Return(&rds.DescribeDBInstancesOutput{
97+
DBInstances: []types.DBInstance{
98+
{
99+
Endpoint: &types.Endpoint{
100+
Address: aws.String("test-endpoint"),
101+
Port: aws.Int32(5432),
102+
},
103+
DBClusterIdentifier: aws.String("test-cluster"),
104+
IAMDatabaseAuthenticationEnabled: aws.Bool(true),
105+
AvailabilityZone: aws.String("us-east-1a"),
106+
DBInstanceStatus: aws.String("available"),
107+
Engine: aws.String("aurora-postgresql"),
108+
TagList: []types.Tag{{Key: aws.String("datadoghq.com/dbm"), Value: aws.String("custom")}},
109+
},
110+
},
111+
}, nil).Times(1)
112+
},
113+
clusterIDs: []string{"test-cluster"},
114+
dbmTag: aws.String("datadoghq.com/dbm:custom"),
115+
expectedAuroraClusterEndpoints: map[string]*AuroraCluster{
116+
"test-cluster": {
117+
Instances: []*Instance{
118+
{
119+
Endpoint: "test-endpoint",
120+
Port: 5432,
121+
IamEnabled: true,
122+
Engine: "aurora-postgresql",
123+
DbName: "postgres",
124+
DbmEnabled: true,
125+
},
126+
},
127+
},
128+
},
129+
},
130+
{
131+
name: "single cluster id returns single endpoint from API with custom bare DBM tag",
132+
configureClient: func(k *MockrdsService) {
133+
k.EXPECT().DescribeDBInstances(gomock.Any(), gomock.Any()).Return(&rds.DescribeDBInstancesOutput{
134+
DBInstances: []types.DBInstance{
135+
{
136+
Endpoint: &types.Endpoint{
137+
Address: aws.String("test-endpoint"),
138+
Port: aws.Int32(5432),
139+
},
140+
DBClusterIdentifier: aws.String("test-cluster"),
141+
IAMDatabaseAuthenticationEnabled: aws.Bool(true),
142+
AvailabilityZone: aws.String("us-east-1a"),
143+
DBInstanceStatus: aws.String("available"),
144+
Engine: aws.String("aurora-postgresql"),
145+
TagList: []types.Tag{{Key: aws.String("datadoghq.com/dbm")}},
146+
},
147+
},
148+
}, nil).Times(1)
149+
},
150+
clusterIDs: []string{"test-cluster"},
151+
dbmTag: aws.String("datadoghq.com/dbm"),
152+
expectedAuroraClusterEndpoints: map[string]*AuroraCluster{
153+
"test-cluster": {
154+
Instances: []*Instance{
155+
{
156+
Endpoint: "test-endpoint",
157+
Port: 5432,
158+
IamEnabled: true,
159+
Engine: "aurora-postgresql",
160+
DbName: "postgres",
161+
DbmEnabled: true,
162+
},
163+
},
164+
},
165+
},
166+
},
90167
{
91168
name: "single cluster id returns many endpoints from API",
92169
configureClient: func(k *MockrdsService) {
@@ -265,6 +342,7 @@ func TestGetAuroraClusterEndpoints(t *testing.T) {
265342
AvailabilityZone: aws.String("us-east-1a"),
266343
DBInstanceStatus: aws.String("available"),
267344
Engine: aws.String("aurora-postgresql"),
345+
TagList: []types.Tag{{Key: aws.String("datadoghq.com/dbm"), Value: aws.String("true")}},
268346
},
269347
{
270348
Endpoint: &types.Endpoint{
@@ -306,6 +384,7 @@ func TestGetAuroraClusterEndpoints(t *testing.T) {
306384
IamEnabled: true,
307385
Engine: "aurora-postgresql",
308386
DbName: "postgres",
387+
DbmEnabled: true,
309388
},
310389
{
311390
Endpoint: "test-endpoint-2",
@@ -337,7 +416,11 @@ func TestGetAuroraClusterEndpoints(t *testing.T) {
337416
mockClient := NewMockrdsService(ctrl)
338417
tt.configureClient(mockClient)
339418
client := &Client{client: mockClient}
340-
clusters, err := client.GetAuroraClusterEndpoints(context.Background(), tt.clusterIDs)
419+
dbmTag := defaultDbmTag
420+
if tt.dbmTag != nil {
421+
dbmTag = *tt.dbmTag
422+
}
423+
clusters, err := client.GetAuroraClusterEndpoints(context.Background(), tt.clusterIDs, dbmTag)
341424
if tt.expectedErr != nil {
342425
assert.EqualError(t, err, tt.expectedErr.Error())
343426
return
@@ -510,7 +593,7 @@ func TestGetAuroraClusterEndpointsDbName(t *testing.T) {
510593
mockClient := NewMockrdsService(ctrl)
511594
tt.configureClient(mockClient)
512595
client := &Client{client: mockClient}
513-
clusters, err := client.GetAuroraClusterEndpoints(context.Background(), tt.clusterIDs)
596+
clusters, err := client.GetAuroraClusterEndpoints(context.Background(), tt.clusterIDs, defaultDbmTag)
514597
if tt.expectedErr != nil {
515598
assert.EqualError(t, err, tt.expectedErr.Error())
516599
return

‎pkg/databasemonitoring/aws/client.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020

2121
// RDSClient is the interface for describing aurora cluster endpoints
2222
type RDSClient interface {
23-
GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentifiers []string) (map[string]*AuroraCluster, error)
23+
GetAuroraClusterEndpoints(ctx context.Context, dbClusterIdentifiers []string, dbmTag string) (map[string]*AuroraCluster, error)
2424
GetAuroraClustersFromTags(ctx context.Context, tags []string) ([]string, error)
2525
}
2626

0 commit comments

Comments
 (0)