Merge branch 'main' into go-1.23.2

Author: blakerouse

CHANGELOG-developer.next.asciidoc

@@ -205,6 +205,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Add a configuration option for TCP/UDP network type. {issue}40407[40407] {pull}40623[40623]
 - Added debug logging to parquet reader in x-pack/libbeat/reader. {pull}40651[40651]
 - Added filebeat debug histograms for s3 object size and events per processed s3 object. {pull}40775[40775]
+- Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
 - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
 
 ==== Deprecated

CHANGELOG.next.asciidoc

@@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Make HTTP library function inclusion non-conditional in CEL input. {pull}40912[40912]
 - Add support for Crowdstrike streaming API to the streaming input. {issue}40264[40264] {pull}40838[40838]
 - Add support to CEL for reading host environment variables. {issue}40762[40762] {pull}40779[40779]
+- Add CSV decoder to awss3 input. {pull}40896[40896]
 
 *Auditbeat*
 
@@ -344,6 +345,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset cluster for the vSphere module. {pull}40536[40536]
 - Add new metricset network for the vSphere module. {pull}40559[40559]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
+- Add AWS Cloudwatch capability to retrieve tags from AWS/ApiGateway resources {pull}40755[40755]
 - Add new metricset datastorecluster for vSphere module. {pull}40634[40634]
 - Add support for new metrics in datastorecluster metricset. {pull}40694[40694]
 - Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]

NOTICE.txt

@@ -27,7 +27,7 @@ require (
 	github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 // indirect
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/aws/aws-lambda-go v1.44.0
-	github.com/aws/aws-sdk-go-v2 v1.30.4
+	github.com/aws/aws-sdk-go-v2 v1.30.5
 	github.com/aws/aws-sdk-go-v2/config v1.27.29
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.29
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5
@@ -179,8 +179,11 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.24
 	github.com/apache/arrow/go/v14 v14.0.2
+	github.com/aws/aws-sdk-go v1.54.19
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13
+	github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8
+	github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5
 	github.com/aws/aws-sdk-go-v2/service/health v1.26.4
 	github.com/aws/aws-sdk-go-v2/service/kinesis v1.29.5
@@ -255,8 +258,8 @@ require (
 	github.com/apache/thrift v0.19.0 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect

go.mod

@@ -158,9 +158,11 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-lambda-go v1.44.0 h1:Xp9PANXKsSJ23IhE4ths592uWTCEewswPhSH9qpAuQQ=
 github.com/aws/aws-lambda-go v1.44.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
+github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
-github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
-github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g=
+github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
 github.com/aws/aws-sdk-go-v2/config v1.27.29 h1:+ZPKb3u9Up4KZWLGTtpTmC5T3XmRD1ZQ8XQjRCHUvJw=
@@ -171,14 +173,18 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJ
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13 h1:X8EeaOjl91c8sP14NG8EHx5ZxXLJg0tHDp+KQSghp28=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.13/go.mod h1:kEI/h2bETfm09LSd7xEEH2qcU1cd//+5HH4Le7p9JgY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8 h1:CgEyY7gfTf7lHYcCi7+w6jJ1XQBugjpadtsuN3TGxdQ=
+github.com/aws/aws-sdk-go-v2/service/apigateway v1.25.8/go.mod h1:z99ur4Ha5540t8hb5XtqV/UMOnEoEZK22lhr5ZBS0zw=
+github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8 h1:SWBNBbVbThg5Hdi3hWbVaDFjV/OyPbuqZLu4N+mj/Es=
+github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.22.8/go.mod h1:lz2IT8gzzSwao0Pa6uMSdCIPsprmgCkW83q6sHGZFDw=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5 h1:YeTVIy7cJLeahs7K0jQGDGAd1YYND/to/z8N3kqZBhY=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.5/go.mod h1:y45SdA9v+dLlweaqwAQMoFeXqdRvgwevafa2X8iTqZQ=
 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.40.5 h1:/YvqO1j75i4leoV+Z3a5s/dAlEszf2wTKBW8jc3Gd4s=

go.sum

@@ -146,6 +146,25 @@ Enforces the use of FIPS service endpoints. See <<aws-credentials-config,AWS cre
     - ec2
 ----
 
+* *apigateway_max_results*
+
+This configuration works together with AWS/APIGateway namespace. It defines the maximum number of returned results per page. The default value is 25 and the maximum value is 500.
+See https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/[email protected]#GetRestApisInput.Limit[GetRestApisInput.Limit]
+
+[source,yaml]
+----
+- module: aws
+      period: 10s
+      regions:
+        - us-east-1
+      metricsets:
+        - cloudwatch
+      metrics:
+        - namespace: "AWS/ApiGateway"
+          resource_type: "apigateway:restapis"
+          apigateway_max_results: 40
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

metricbeat/docs/modules/aws.asciidoc

@@ -123,28 +123,71 @@ The file decoding option is used to specify a codec that will be used to
 decode the file contents. This can apply to any file stream data.
 An example config is shown below:
 
+Currently supported codecs are given below:-
+
+    1. <<attrib-decoding-csv,CSV>>: This codec decodes RFC 4180 CSV data streams.
+    2. <<attrib-decoding-parquet,Parquet>>: This codec decodes parquet compressed data streams.
+
+[id="attrib-decoding-csv"]
+[float]
+==== `the CSV codec`
+The `CSV` codec is used to decode RFC 4180 CSV data streams.
+Enabling the codec without other options will use the default codec options.
+
 [source,yaml]
 ----
-  decoding.codec.parquet.enabled: true
-----
+  decoding.codec.csv.enabled: true
+----
+
+The CSV codec supports five sub attributes to control aspects of CSV decoding.
+The `comma` attribute specifies the field separator character used by the CSV
+format. If it is not specified, the comma character '`,`' is used. The `comment`
+attribute specifies the character that should be interpreted as a comment mark.
+If it is specified, lines starting with the character will be ignored. Both
+`comma` and `comment` must be single characters. The `lazy_quotes` attribute
+controls how quoting in fields is handled. If `lazy_quotes` is true, a quote may
+appear in an unquoted field and a non-doubled quote may appear in a quoted field.
+The `trim_leading_space` attribute specifies that leading white space should be
+ignored, even if the `comma` character is white space. For complete details
+of the preceding configuration attribute behaviors, see the CSV decoder
+https://pkg.go.dev/encoding/csv#Reader[documentation] The `fields_names`
+attribute can be used to specify the column names for the data. If it is
+absent, the field names are obtained from the first non-comment line of
+data. The number of fields must match the number of field names.
 
-Currently supported codecs are given below:-
+An example config is shown below:
 
-    1. <<attrib-decoding-parquet,Parquet>>: This codec decodes parquet compressed data streams.
+[source,yaml]
+----
+  decoding.codec.csv.enabled: true
+  decoding.codec.csv.comma: "\t"
+  decoding.codec.csv.comment: "#"
+----
 
 [id="attrib-decoding-parquet"]
 [float]
 ==== `the parquet codec`
 The `parquet` codec is used to decode parquet compressed data streams.
-Only enabling the codec will use the default codec options. The parquet codec supports
-two sub attributes which can make parquet decoding more efficient. The `batch_size` attribute and
-the `process_parallel` attribute. The `batch_size` attribute can be used to specify the number of
-records to read from the parquet stream at a time. By default the `batch size` is set to `1` and
-`process_parallel` is set to `false`. If the `process_parallel` attribute is set to `true` then functions
-which read multiple columns will read those columns in parallel from the parquet stream with a
-number of readers equal to the number of columns. Setting `process_parallel` to `true` will greatly
-increase the rate of processing at the cost of increased memory usage. Having a larger `batch_size`
-also helps to increase the rate of processing. An example config is shown below:
+Only enabling the codec will use the default codec options.
+
+[source,yaml]
+----
+  decoding.codec.parquet.enabled: true
+----
+
+The parquet codec supports two sub attributes which can make parquet decoding
+more efficient. The `batch_size` attribute and the `process_parallel`
+attribute. The `batch_size` attribute can be used to specify the number of
+records to read from the parquet stream at a time. By default the `batch
+size` is set to `1` and `process_parallel` is set to `false`. If the
+`process_parallel` attribute is set to `true` then functions which read
+multiple columns will read those columns in parallel from the parquet stream
+with a number of readers equal to the number of columns. Setting
+`process_parallel` to `true` will greatly increase the rate of processing at
+the cost of increased memory usage. Having a larger `batch_size` also helps
+to increase the rate of processing.
+
+An example config is shown below:
 
 [source,yaml]
 ----

x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc

@@ -7,11 +7,9 @@ package awss3
 import (
 	"fmt"
 	"io"
-
-	"github.com/elastic/beats/v7/x-pack/libbeat/reader/parquet"
 )
 
-// decoder is an interface for decoding data from an io reader.
+// decoder is an interface for decoding data from an io.Reader.
 type decoder interface {
 	// decode reads and decodes data from an io reader based on the codec type.
 	// It returns the decoded data and an error if the data cannot be decoded.
@@ -23,6 +21,13 @@ type decoder interface {
 	close() error
 }
 
+// valueDecoder is a decoder that can decode directly to a JSON serialisable value.
+type valueDecoder interface {
+	decoder
+
+	decodeValue() (any, error)
+}
+
 // newDecoder creates a new decoder based on the codec type.
 // It returns a decoder type and an error if the codec type is not supported.
 // If the reader config codec option is not set, it returns a nil decoder and nil error.
@@ -32,47 +37,9 @@ func newDecoder(config decoderConfig, r io.Reader) (decoder, error) {
 		return nil, nil
 	case config.Codec.Parquet != nil:
 		return newParquetDecoder(config, r)
+	case config.Codec.CSV != nil:
+		return newCSVDecoder(config, r)
 	default:
 		return nil, fmt.Errorf("unsupported config value: %v", config)
 	}
 }
-
-// parquetDecoder is a decoder for parquet data.
-type parquetDecoder struct {
-	reader *parquet.BufferedReader
-}
-
-// newParquetDecoder creates a new parquet decoder. It uses the libbeat parquet reader under the hood.
-// It returns an error if the parquet reader cannot be created.
-func newParquetDecoder(config decoderConfig, r io.Reader) (decoder, error) {
-	reader, err := parquet.NewBufferedReader(r, &parquet.Config{
-		ProcessParallel: config.Codec.Parquet.ProcessParallel,
-		BatchSize:       config.Codec.Parquet.BatchSize,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to create parquet decoder: %w", err)
-	}
-	return &parquetDecoder{
-		reader: reader,
-	}, nil
-}
-
-// next advances the parquet decoder to the next data item and returns true if there is more data to be decoded.
-func (pd *parquetDecoder) next() bool {
-	return pd.reader.Next()
-}
-
-// decode reads and decodes a parquet data stream. After reading the parquet data it decodes
-// the output to JSON and returns it as a byte slice. It returns an error if the data cannot be decoded.
-func (pd *parquetDecoder) decode() ([]byte, error) {
-	data, err := pd.reader.Record()
-	if err != nil {
-		return nil, err
-	}
-	return data, nil
-}
-
-// close closes the parquet decoder and releases the resources.
-func (pd *parquetDecoder) close() error {
-	return pd.reader.Close()
-}

x-pack/filebeat/input/awss3/decoding.go

@@ -4,6 +4,12 @@
 
 package awss3
 
+import (
+	"errors"
+	"fmt"
+	"unicode/utf8"
+)
+
 // decoderConfig contains the configuration options for instantiating a decoder.
 type decoderConfig struct {
 	Codec *codecConfig `config:"codec"`
@@ -12,6 +18,48 @@ type decoderConfig struct {
 // codecConfig contains the configuration options for different codecs used by a decoder.
 type codecConfig struct {
 	Parquet *parquetCodecConfig `config:"parquet"`
+	CSV     *csvCodecConfig     `config:"csv"`
+}
+
+func (c *codecConfig) Validate() error {
+	if c.Parquet != nil && c.CSV != nil {
+		return errors.New("more than one decoder configured")
+	}
+	return nil
+}
+
+// csvCodecConfig contains the configuration options for the CSV codec.
+type csvCodecConfig struct {
+	Enabled bool `config:"enabled"`
+
+	// Fields is the set of field names. If it is present
+	// it is used to specify the object names of returned
+	// values and the FieldsPerRecord field in the csv.Reader.
+	// Otherwise, names are obtained from the first
+	// line of the CSV data.
+	Fields []string `config:"fields_names"`
+
+	// The fields below have the same meaning as the
+	// fields of the same name in csv.Reader.
+	Comma            *configRune `config:"comma"`
+	Comment          configRune  `config:"comment"`
+	LazyQuotes       bool        `config:"lazy_quotes"`
+	TrimLeadingSpace bool        `config:"trim_leading_space"`
+}
+
+type configRune rune
+
+func (r *configRune) Unpack(s string) error {
+	if s == "" {
+		return nil
+	}
+	n := utf8.RuneCountInString(s)
+	if n != 1 {
+		return fmt.Errorf("single character option given more than one character: %q", s)
+	}
+	_r, _ := utf8.DecodeRuneInString(s)
+	*r = configRune(_r)
+	return nil
 }
 
 // parquetCodecConfig contains the configuration options for the parquet codec.