AlertStore: implementation using thanos objstore bucket client

CHANGELOG.md

@@ -8,7 +8,9 @@
 * [CHANGE] Store-gateway: the chunks pool controlled by `-blocks-storage.bucket-store.max-chunk-pool-bytes` is now shared across all tenants. #3830
 * [CHANGE] Ingester: return error code 400 instead of 429 when per-user/per-tenant series/metadata limits are reached. #3833
 * [CHANGE] Compactor: add `reason` label to `cortex_compactor_blocks_marked_for_deletion_total` metric. Source blocks marked for deletion by compactor are labelled as `compaction`, while blocks passing the retention period are labelled as `retention`. #3879
+* [CHANGE] Alertmanager: the `DELETE /api/v1/alerts` is now idempotent. No error is returned if the alertmanager config doesn't exist. #3888
 * [FEATURE] Experimental Ruler Storage: Add a separate set of configuration options to configure the ruler storage backend under the `-ruler-storage.` flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing `-ruler.storage` flags are left unset. #3805 #3864
+* [FEATURE] Experimental Alertmanager Storage: Add a separate set of configuration options to configure the alertmanager storage backend under the `-alertmanager-storage.` flag prefix. All blocks storage bucket clients and the config service are currently supported. Clients using this implementation will only be enabled if the existing `-alertmanager.storage` flags are left unset. #3888
 * [FEATURE] Adds support to S3 server-side encryption using KMS. The S3 server-side encryption config can be overridden on a per-tenant basis for the blocks storage and ruler. Deprecated `-<prefix>.s3.sse-encryption`, you should use the following CLI flags that have been added. #3651 #3810 #3811 #3870 #3886
   - `-<prefix>.s3.sse.type`
   - `-<prefix>.s3.sse.kms-key-id`

Makefile

@@ -188,6 +188,7 @@ lint:
 
 	# Ensure packages that no longer use a global logger don't reintroduce it
 	faillint -paths "github.com/cortexproject/cortex/pkg/util/log.{Logger}" \
+		./pkg/alertmanager/alertstore/... \
 		./pkg/ingester/... \
 		./pkg/flusher/... \
 		./pkg/querier/... \

development/tsdb-blocks-storage-s3-gossip/config/cortex.yaml

@@ -100,12 +100,15 @@ ruler_storage:
 
 alertmanager:
   enable_api: true
-  storage:
-    type: s3
-    s3:
-      bucketnames: cortex-alertmanager
-      s3forcepathstyle: true
-      s3: http://cortex:supersecret@minio.:9000
+
+alertmanager_storage:
+  backend: s3
+  s3:
+    bucket_name:       cortex-alertmanager
+    endpoint:          minio:9000
+    access_key_id:     cortex
+    secret_access_key: supersecret
+    insecure:          true
 
 storage:
   engine: blocks

development/tsdb-blocks-storage-s3/config/alertmanager.yaml

@@ -0,0 +1,16 @@
+# Example alertmanager config file to load to Cortex via the alertmanager API.
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'localhost:25'
+  smtp_from: '[email protected]'
+  smtp_auth_username: 'alertmanager'
+  smtp_auth_password: 'password'
+
+route:
+  # A default receiver.
+  receiver: send-email
+
+receivers:
+  - name: send-email
+    email_configs:
+      - to: 'someone@localhost'

development/tsdb-blocks-storage-s3/config/cortex.yaml

@@ -85,7 +85,7 @@ ruler_storage:
     endpoint:          minio:9000
     access_key_id:     cortex
     secret_access_key: supersecret
-    insecure: true
+    insecure:          true
 
 alertmanager:
   enable_api: true
@@ -98,12 +98,15 @@ alertmanager:
       store: consul
       consul:
         host: consul:8500
-  storage:
-    type: s3
-    s3:
-      bucketnames: cortex-alertmanager
-      s3forcepathstyle: true
-      s3: http://cortex:supersecret@minio.:9000
+
+alertmanager_storage:
+  backend: s3
+  s3:
+    bucket_name:       cortex-alertmanager
+    endpoint:          minio:9000
+    access_key_id:     cortex
+    secret_access_key: supersecret
+    insecure:          true
 
 storage:
   engine: blocks

docs/configuration/config-file-reference.md

@@ -362,6 +362,214 @@ ruler_storage:
 # The alertmanager_config configures the Cortex alertmanager.
 [alertmanager: <alertmanager_config>]
 
+alertmanager_storage:
+  # Backend storage to use. Supported backends are: s3, gcs, azure, swift,
+  # filesystem, configdb, local.
+  # CLI flag: -alertmanager-storage.backend
+  [backend: <string> | default = "s3"]
+
+  s3:
+    # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at
+    # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an
+    # S3-compatible service in hostname:port format.
+    # CLI flag: -alertmanager-storage.s3.endpoint
+    [endpoint: <string> | default = ""]
+
+    # S3 region. If unset, the client will issue a S3 GetBucketLocation API call
+    # to autodetect it.
+    # CLI flag: -alertmanager-storage.s3.region
+    [region: <string> | default = ""]
+
+    # S3 bucket name
+    # CLI flag: -alertmanager-storage.s3.bucket-name
+    [bucket_name: <string> | default = ""]
+
+    # S3 secret access key
+    # CLI flag: -alertmanager-storage.s3.secret-access-key
+    [secret_access_key: <string> | default = ""]
+
+    # S3 access key ID
+    # CLI flag: -alertmanager-storage.s3.access-key-id
+    [access_key_id: <string> | default = ""]
+
+    # If enabled, use http:// for the S3 endpoint instead of https://. This
+    # could be useful in local dev/test environments while using an
+    # S3-compatible backend storage, like Minio.
+    # CLI flag: -alertmanager-storage.s3.insecure
+    [insecure: <boolean> | default = false]
+
+    # The signature version to use for authenticating against S3. Supported
+    # values are: v4, v2.
+    # CLI flag: -alertmanager-storage.s3.signature-version
+    [signature_version: <string> | default = "v4"]
+
+    # The s3_sse_config configures the S3 server-side encryption.
+    # The CLI flags prefix for this block config is: alertmanager-storage
+    [sse: <s3_sse_config>]
+
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -alertmanager-storage.s3.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -alertmanager-storage.s3.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects to S3 via HTTPS and this option is enabled, the
+      # client will accept any certificate and hostname.
+      # CLI flag: -alertmanager-storage.s3.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
+      # Maximum time to wait for a TLS handshake. 0 means no limit.
+      # CLI flag: -alertmanager-storage.s3.tls-handshake-timeout
+      [tls_handshake_timeout: <duration> | default = 10s]
+
+      # The time to wait for a server's first response headers after fully
+      # writing the request headers if the request has an Expect header. 0 to
+      # send the request body immediately.
+      # CLI flag: -alertmanager-storage.s3.expect-continue-timeout
+      [expect_continue_timeout: <duration> | default = 1s]
+
+      # Maximum number of idle (keep-alive) connections across all hosts. 0
+      # means no limit.
+      # CLI flag: -alertmanager-storage.s3.max-idle-connections
+      [max_idle_connections: <int> | default = 100]
+
+      # Maximum number of idle (keep-alive) connections to keep per-host. If 0,
+      # a built-in default value is used.
+      # CLI flag: -alertmanager-storage.s3.max-idle-connections-per-host
+      [max_idle_connections_per_host: <int> | default = 100]
+
+      # Maximum number of connections per host. 0 means no limit.
+      # CLI flag: -alertmanager-storage.s3.max-connections-per-host
+      [max_connections_per_host: <int> | default = 0]
+
+  gcs:
+    # GCS bucket name
+    # CLI flag: -alertmanager-storage.gcs.bucket-name
+    [bucket_name: <string> | default = ""]
+
+    # JSON representing either a Google Developers Console
+    # client_credentials.json file or a Google Developers service account key
+    # file. If empty, fallback to Google default logic.
+    # CLI flag: -alertmanager-storage.gcs.service-account
+    [service_account: <string> | default = ""]
+
+  azure:
+    # Azure storage account name
+    # CLI flag: -alertmanager-storage.azure.account-name
+    [account_name: <string> | default = ""]
+
+    # Azure storage account key
+    # CLI flag: -alertmanager-storage.azure.account-key
+    [account_key: <string> | default = ""]
+
+    # Azure storage container name
+    # CLI flag: -alertmanager-storage.azure.container-name
+    [container_name: <string> | default = ""]
+
+    # Azure storage endpoint suffix without schema. The account name will be
+    # prefixed to this value to create the FQDN
+    # CLI flag: -alertmanager-storage.azure.endpoint-suffix
+    [endpoint_suffix: <string> | default = ""]
+
+    # Number of retries for recoverable errors
+    # CLI flag: -alertmanager-storage.azure.max-retries
+    [max_retries: <int> | default = 20]
+
+  swift:
+    # OpenStack Swift authentication API version. 0 to autodetect.
+    # CLI flag: -alertmanager-storage.swift.auth-version
+    [auth_version: <int> | default = 0]
+
+    # OpenStack Swift authentication URL
+    # CLI flag: -alertmanager-storage.swift.auth-url
+    [auth_url: <string> | default = ""]
+
+    # OpenStack Swift username.
+    # CLI flag: -alertmanager-storage.swift.username
+    [username: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -alertmanager-storage.swift.user-domain-name
+    [user_domain_name: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -alertmanager-storage.swift.user-domain-id
+    [user_domain_id: <string> | default = ""]
+
+    # OpenStack Swift user ID.
+    # CLI flag: -alertmanager-storage.swift.user-id
+    [user_id: <string> | default = ""]
+
+    # OpenStack Swift API key.
+    # CLI flag: -alertmanager-storage.swift.password
+    [password: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -alertmanager-storage.swift.domain-id
+    [domain_id: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -alertmanager-storage.swift.domain-name
+    [domain_name: <string> | default = ""]
+
+    # OpenStack Swift project ID (v2,v3 auth only).
+    # CLI flag: -alertmanager-storage.swift.project-id
+    [project_id: <string> | default = ""]
+
+    # OpenStack Swift project name (v2,v3 auth only).
+    # CLI flag: -alertmanager-storage.swift.project-name
+    [project_name: <string> | default = ""]
+
+    # ID of the OpenStack Swift project's domain (v3 auth only), only needed if
+    # it differs the from user domain.
+    # CLI flag: -alertmanager-storage.swift.project-domain-id
+    [project_domain_id: <string> | default = ""]
+
+    # Name of the OpenStack Swift project's domain (v3 auth only), only needed
+    # if it differs from the user domain.
+    # CLI flag: -alertmanager-storage.swift.project-domain-name
+    [project_domain_name: <string> | default = ""]
+
+    # OpenStack Swift Region to use (v2,v3 auth only).
+    # CLI flag: -alertmanager-storage.swift.region-name
+    [region_name: <string> | default = ""]
+
+    # Name of the OpenStack Swift container to put chunks in.
+    # CLI flag: -alertmanager-storage.swift.container-name
+    [container_name: <string> | default = ""]
+
+    # Max retries on requests error.
+    # CLI flag: -alertmanager-storage.swift.max-retries
+    [max_retries: <int> | default = 3]
+
+    # Time after which a connection attempt is aborted.
+    # CLI flag: -alertmanager-storage.swift.connect-timeout
+    [connect_timeout: <duration> | default = 10s]
+
+    # Time after which an idle request is aborted. The timeout watchdog is reset
+    # each time some data is received, so the timeout triggers after X time no
+    # data is received on a request.
+    # CLI flag: -alertmanager-storage.swift.request-timeout
+    [request_timeout: <duration> | default = 5s]
+
+  filesystem:
+    # Local filesystem storage directory.
+    # CLI flag: -alertmanager-storage.filesystem.dir
+    [dir: <string> | default = ""]
+
+  # The configstore_config configures the config database storing rules and
+  # alerts, and is used by the Cortex alertmanager.
+  # The CLI flags prefix for this block config is: alertmanager-storage
+  [configdb: <configstore_config>]
+
+  local:
+    # Path at which alertmanager configurations are stored.
+    # CLI flag: -alertmanager-storage.local.path
+    [path: <string> | default = ""]
+
 runtime_config:
   # How often to check runtime config file.
   # CLI flag: -runtime-config.reload-period
@@ -3989,6 +4197,7 @@ api:
 The `configstore_config` configures the config database storing rules and alerts, and is used by the Cortex alertmanager. The supported CLI flags `<prefix>` used to reference this config block are:
 
 - `alertmanager`
+- `alertmanager-storage`
 - `ruler`
 - `ruler-storage`
 
@@ -4887,6 +5096,7 @@ The `purger_config` configures the purger which takes care of delete requests.
 The `s3_sse_config` configures the S3 server-side encryption. The supported CLI flags `<prefix>` used to reference this config block are:
 
 - _no prefix_
+- `alertmanager-storage`
 - `alertmanager.storage`
 - `blocks-storage`
 - `ruler-storage`

docs/configuration/v1-guarantees.md

@@ -70,4 +70,5 @@ Currently experimental features are:
 - Alertmanager: Sharding of tenants across multiple instances
 - The thanosconvert tool for converting Thanos block metadata to Cortex
 - HA Tracker: cleanup of old replicas from KV Store.
-- Ruler storage: backend client configuration options using a config fields similar to the TSDB object storage clients.
+- Ruler storage: backend client configuration options using a config fields similar to the blocks storage backend clients.
+- Alertmanager storage: backend client configuration options using a config fields similar to the blocks storage backend clients.