Skip to content

Commit e388dd8

Browse files
csatib02csatib02
committed
feat: awscloudwatch storage extension
Signed-off-by: Bence Csati <[email protected]>
1 parent 0796f50 commit e388dd8

File tree

10 files changed

+652
-114
lines changed

10 files changed

+652
-114
lines changed

‎receiver/awscloudwatchreceiver/README.md

+7-6
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,16 @@ This receiver uses the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/devel
2929
| `profile` | *optional* | string | The AWS profile used to authenticate, if none is specified the default is chosen from the list of profiles |
3030
| `imds_endpoint` | *optional* | string | A way of specifying a custom URL to be used by the EC2 IMDS client to validate the session. If unset, and the environment variable `AWS_EC2_METADATA_SERVICE_ENDPOINT` has a value the client will use the value of the environment variable as the endpoint for operation calls. |
3131
| `logs` | *optional* | `Logs` | Configuration for Logs ingestion of this receiver |
32+
| `storage` | *optional* | string | The ID of a storage extension to be used for state persistence. |
3233

3334
### Logs Parameters
3435

35-
| Parameter | Notes | type | Description |
36-
| ------------------------ | -------------- | ---------------------- | ------------------------------------------------------------------------------------------ |
37-
| `poll_interval` | `default=1m` | duration | The duration waiting in between requests. |
38-
| `max_events_per_request` | `default=1000` | int | The maximum number of events to process per request to Cloudwatch |
39-
| `groups` | *optional* | `See Group Parameters` | Configuration for Log Groups, by default all Log Groups and Log Streams will be collected. |
36+
| Parameter | Type | Default | Description |
37+
|--------------------------|----------|--------------------------------------------|--------------------------------------------------------------------------------------------------------|
38+
| `start_from` | String | Read all available logs from the beginning | Timestamp in `RFC3339` format (e.g., 2006-01-02T15:04:05Z07:00) indicating where to start reading logs |
39+
| `poll_interval` | Duration | 1 minute | Time to wait between log requests |
40+
| `max_events_per_request` | Integer | 1,000 | The maximum number of events to process per request to Cloudwatch |
41+
| `groups` | Optional | All Log Groups and Streams | Configuration for Log Groups, by default all Log Groups and Log Streams will be collected. |
4042

4143
### Group Parameters
4244

@@ -114,4 +116,3 @@ This receiver has a number of sample configs for reference.
114116
- Specifies the names of the log groups to collect
115117
- Does not attempt autodiscovery
116118
- Only collects from log streams matching a prefix
117-

‎receiver/awscloudwatchreceiver/config.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"net/url"
1010
"time"
1111

12+
"go.opentelemetry.io/collector/component"
1213
"go.opentelemetry.io/collector/confmap"
1314
)
1415

@@ -20,14 +21,16 @@ var (
2021

2122
// Config is the overall config structure for the awscloudwatchreceiver
2223
type Config struct {
23-
Region string `mapstructure:"region"`
24-
Profile string `mapstructure:"profile"`
25-
IMDSEndpoint string `mapstructure:"imds_endpoint"`
26-
Logs *LogsConfig `mapstructure:"logs"`
24+
Region string `mapstructure:"region"`
25+
Profile string `mapstructure:"profile"`
26+
IMDSEndpoint string `mapstructure:"imds_endpoint"`
27+
Logs *LogsConfig `mapstructure:"logs"`
28+
StorageID *component.ID `mapstructure:"storage"`
2729
}
2830

2931
// LogsConfig is the configuration for the logs portion of this receiver
3032
type LogsConfig struct {
33+
StartFrom string `mapstructure:"start_from"`
3134
PollInterval time.Duration `mapstructure:"poll_interval"`
3235
MaxEventsPerRequest int `mapstructure:"max_events_per_request"`
3336
Groups GroupConfig `mapstructure:"groups"`
@@ -101,9 +104,17 @@ func (c *Config) validateLogsConfig() error {
101104
return errNoLogsConfigured
102105
}
103106

107+
if c.Logs.StartFrom != "" {
108+
_, err := time.Parse(time.RFC3339, c.Logs.StartFrom)
109+
if err != nil {
110+
return fmt.Errorf("invalid start_from time format: %w", err)
111+
}
112+
}
113+
104114
if c.Logs.MaxEventsPerRequest <= 0 {
105115
return errInvalidEventLimit
106116
}
117+
107118
if c.Logs.PollInterval < time.Second {
108119
return errInvalidPollInterval
109120
}

‎receiver/awscloudwatchreceiver/factory.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ func NewFactory() receiver.Factory {
2424

2525
func createLogsReceiver(
2626
_ context.Context,
27-
params receiver.Settings,
27+
settings receiver.Settings,
2828
rConf component.Config,
2929
consumer consumer.Logs,
3030
) (receiver.Logs, error) {
3131
cfg := rConf.(*Config)
32-
rcvr := newLogsReceiver(cfg, params.Logger, consumer)
32+
rcvr := newLogsReceiver(cfg, settings, consumer)
3333
return rcvr, nil
3434
}
3535

‎receiver/awscloudwatchreceiver/go.mod

+38-21
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ require (
66
github.com/aws/aws-sdk-go-v2 v1.36.3
77
github.com/aws/aws-sdk-go-v2/config v1.29.14
88
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.47.3
9-
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.124.1
10-
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.124.1
9+
github.com/json-iterator/go v1.1.12
10+
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.124.0
11+
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.124.0
12+
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.124.0
1113
github.com/stretchr/testify v1.10.0
1214
go.opentelemetry.io/collector/component v1.30.1-0.20250422165940-c47951a8bf71
1315
go.opentelemetry.io/collector/component/componenttest v0.124.1-0.20250422165940-c47951a8bf71
@@ -35,33 +37,45 @@ require (
3537
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect
3638
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
3739
github.com/aws/smithy-go v1.22.2 // indirect
40+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
3841
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3942
github.com/davecgh/go-spew v1.1.1 // indirect
43+
github.com/elastic/lunes v0.1.0 // indirect
44+
github.com/expr-lang/expr v1.17.2 // indirect
4045
github.com/go-logr/logr v1.4.2 // indirect
4146
github.com/go-logr/stdr v1.2.2 // indirect
4247
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
48+
github.com/goccy/go-json v0.10.5 // indirect
4349
github.com/gogo/protobuf v1.3.2 // indirect
4450
github.com/google/uuid v1.6.0 // indirect
4551
github.com/hashicorp/go-version v1.7.0 // indirect
46-
github.com/json-iterator/go v1.1.12 // indirect
4752
github.com/knadh/koanf/maps v0.1.2 // indirect
48-
github.com/knadh/koanf/providers/confmap v1.0.0 // indirect
49-
github.com/knadh/koanf/v2 v2.2.0 // indirect
53+
github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
54+
github.com/knadh/koanf/v2 v2.1.2 // indirect
55+
github.com/leodido/go-syslog/v4 v4.2.0 // indirect
56+
github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
57+
github.com/magefile/mage v1.15.0 // indirect
5058
github.com/mitchellh/copystructure v1.2.0 // indirect
5159
github.com/mitchellh/reflectwalk v1.0.2 // indirect
5260
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5361
github.com/modern-go/reflect2 v1.0.2 // indirect
54-
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.124.1 // indirect
62+
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.124.0 // indirect
63+
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.124.0 // indirect
5564
github.com/pmezard/go-difflib v1.0.0 // indirect
5665
github.com/stretchr/objx v0.5.2 // indirect
66+
github.com/valyala/fastjson v1.6.4 // indirect
5767
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
58-
go.opentelemetry.io/collector/consumer/consumererror v0.124.1-0.20250422165940-c47951a8bf71 // indirect
59-
go.opentelemetry.io/collector/consumer/xconsumer v0.124.1-0.20250422165940-c47951a8bf71 // indirect
60-
go.opentelemetry.io/collector/featuregate v1.30.1-0.20250422165940-c47951a8bf71 // indirect
61-
go.opentelemetry.io/collector/internal/telemetry v0.124.1-0.20250422165940-c47951a8bf71 // indirect
62-
go.opentelemetry.io/collector/pdata/pprofile v0.124.1-0.20250422165940-c47951a8bf71 // indirect
63-
go.opentelemetry.io/collector/pipeline v0.124.1-0.20250422165940-c47951a8bf71 // indirect
64-
go.opentelemetry.io/collector/receiver/xreceiver v0.124.1-0.20250422165940-c47951a8bf71 // indirect
68+
go.opentelemetry.io/collector/consumer/consumererror v0.124.0 // indirect
69+
go.opentelemetry.io/collector/consumer/xconsumer v0.124.0 // indirect
70+
go.opentelemetry.io/collector/extension v1.29.1-0.20250402200755-cb5c3f4fb9dc // indirect
71+
go.opentelemetry.io/collector/extension/xextension v0.123.1-0.20250402200755-cb5c3f4fb9dc
72+
go.opentelemetry.io/collector/featuregate v1.30.0 // indirect
73+
go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect
74+
go.opentelemetry.io/collector/pdata/pprofile v0.124.0 // indirect
75+
go.opentelemetry.io/collector/pipeline v0.124.0 // indirect
76+
go.opentelemetry.io/collector/receiver/receiverhelper v0.123.1-0.20250402200755-cb5c3f4fb9dc // indirect
77+
go.opentelemetry.io/collector/receiver/xreceiver v0.124.0 // indirect
78+
go.opentelemetry.io/collector/semconv v0.123.1-0.20250402200755-cb5c3f4fb9dc // indirect
6579
go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
6680
go.opentelemetry.io/otel v1.35.0 // indirect
6781
go.opentelemetry.io/otel/log v0.11.0 // indirect
@@ -70,11 +84,12 @@ require (
7084
go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
7185
go.opentelemetry.io/otel/trace v1.35.0 // indirect
7286
go.uber.org/multierr v1.11.0 // indirect
73-
golang.org/x/net v0.39.0 // indirect
74-
golang.org/x/sys v0.32.0 // indirect
75-
golang.org/x/text v0.24.0 // indirect
76-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect
77-
google.golang.org/grpc v1.72.0 // indirect
87+
golang.org/x/net v0.38.0 // indirect
88+
golang.org/x/sys v0.31.0 // indirect
89+
golang.org/x/text v0.23.0 // indirect
90+
gonum.org/v1/gonum v0.16.0 // indirect
91+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
92+
google.golang.org/grpc v1.71.1 // indirect
7893
google.golang.org/protobuf v1.36.6 // indirect
7994
gopkg.in/yaml.v3 v3.0.1 // indirect
8095
sigs.k8s.io/yaml v1.4.0 // indirect
@@ -92,8 +107,10 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil
92107

93108
replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden
94109

95-
replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.0.0-00010101000000-000000000000 => go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.0.0-20250422165940-c47951a8bf71
110+
replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal
96111

97-
replace go.opentelemetry.io/collector/config/configmiddleware v0.0.0-00010101000000-000000000000 => go.opentelemetry.io/collector/config/configmiddleware v0.0.0-20250422165940-c47951a8bf71
112+
replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => ../../pkg/stanza
98113

99-
replace go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 => go.opentelemetry.io/collector/extension/extensionmiddleware v0.0.0-20250422165940-c47951a8bf71
114+
replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common
115+
116+
replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage => ../../extension/storage

0 commit comments

Comments
 (0)