feat(txt-registry): add option to use only new format (#4946)

* feat: add option to use only new format TXT records

* add flag and docs

* refine documentation on how to use the flag

* add section regarding manual migration

* update documentation to be same as in types.go

* fix compile issue

* add tests for new flag

* update flags documentation correctly

* add new option to helm chart

* run helm-docs

* remove unessery newline

* add entry to unreleased chart items

* Revert "run helm-docs"

This reverts commit a1d64bd.

* Revert "add new option to helm chart"

This reverts commit 299d087.

* Revert "add entry to unreleased chart items"

This reverts commit 0bcd0e3.

* fix test cases that have changed

Author: malpou

docs/flags.md

@@ -152,6 +152,7 @@
 | `--txt-wildcard-replacement=""` | When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional) |
 | `--[no-]txt-encrypt-enabled` | When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled) |
 | `--txt-encrypt-aes-key=""` | When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true) |
+| `--[no-]txt-new-format-only` | When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled) |
 | `--dynamodb-region=""` | When using the DynamoDB registry, the AWS region of the DynamoDB table (optional) |
 | `--dynamodb-table="external-dns"` | When using the DynamoDB registry, the name of the DynamoDB table (default: "external-dns") |
 | `--txt-cache-interval=0s` | The interval between cache synchronizations in duration format (default: disabled) |

docs/registry/txt.md

@@ -3,6 +3,35 @@
 The TXT registry is the default registry.
 It stores DNS record metadata in TXT records, using the same provider.
 
+## Record Format Options
+The TXT registry supports two formats for storing DNS record metadata:
+- Legacy format: Creates a TXT record without record type information
+- New format: Creates a TXT record with record type information (e.g., 'a-' prefix for A records)
+
+By default, the TXT registry creates records in both formats for backwards compatibility. You can configure it to use only the new format by using the `--txt-new-format-only` flag. This reduces the number of TXT records created, which can be helpful when working with provider-specific record limits.
+
+Note: The following record types always use only the new format regardless of this setting:
+- AAAA records
+- Encrypted TXT records (when using `--txt-encrypt-enabled`)
+
+Example:
+```sh
+# Default behavior - creates both formats
+external-dns --provider=aws --source=ingress --managed-record-types=A,TXT
+
+# Only create new format records (alongside other required flags)
+external-dns --provider=aws --source=ingress --managed-record-types=A,TXT --txt-new-format-only
+```
+The `--txt-new-format-only` flag should be used in addition to your existing external-dns configuration flags. It does not implicitly configure TXT record handling - you still need to specify `--managed-record-types=TXT` if you want external-dns to manage TXT records.
+
+### Migration to New Format Only
+When transitioning from dual-format to new-format-only records:
+- Ensure all your `external-dns` instances support the new format
+- Enable the `--txt-new-format-only` flag on your external-dns instances
+Manually clean up any existing legacy format TXT records from your DNS provider
+
+Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired.
+
 ## Prefixes and Suffixes
 
 In order to avoid having the registry TXT records collide with

main.go

@@ -389,7 +389,7 @@ func main() {
 	case "noop":
 		r, err = registry.NewNoopRegistry(p)
 	case "txt":
-		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes, cfg.ExcludeDNSRecordTypes, cfg.TXTEncryptEnabled, []byte(cfg.TXTEncryptAESKey))
+		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes, cfg.ExcludeDNSRecordTypes, cfg.TXTEncryptEnabled, []byte(cfg.TXTEncryptAESKey), cfg.TXTNewFormatOnly)
 	case "aws-sd":
 		r, err = registry.NewAWSSDRegistry(p, cfg.TXTOwnerID)
 	default:

pkg/apis/externaldns/types.go

@@ -138,6 +138,7 @@ type Config struct {
 	TXTSuffix                          string
 	TXTEncryptEnabled                  bool
 	TXTEncryptAESKey                   string `secure:"yes"`
+	TXTNewFormatOnly                   bool
 	Interval                           time.Duration
 	MinEventSyncInterval               time.Duration
 	Once                               bool
@@ -299,6 +300,7 @@ var defaultConfig = &Config{
 	MinEventSyncInterval:        5 * time.Second,
 	TXTEncryptEnabled:           false,
 	TXTEncryptAESKey:            "",
+	TXTNewFormatOnly:            false,
 	Interval:                    time.Minute,
 	Once:                        false,
 	DryRun:                      false,
@@ -591,6 +593,7 @@ func App(cfg *Config) *kingpin.Application {
 	app.Flag("txt-wildcard-replacement", "When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional)").Default(defaultConfig.TXTWildcardReplacement).StringVar(&cfg.TXTWildcardReplacement)
 	app.Flag("txt-encrypt-enabled", "When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled)").BoolVar(&cfg.TXTEncryptEnabled)
 	app.Flag("txt-encrypt-aes-key", "When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true)").Default(defaultConfig.TXTEncryptAESKey).StringVar(&cfg.TXTEncryptAESKey)
+	app.Flag("txt-new-format-only", "When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled)").BoolVar(&cfg.TXTNewFormatOnly)
 	app.Flag("dynamodb-region", "When using the DynamoDB registry, the AWS region of the DynamoDB table (optional)").Default(cfg.AWSDynamoDBRegion).StringVar(&cfg.AWSDynamoDBRegion)
 	app.Flag("dynamodb-table", "When using the DynamoDB registry, the name of the DynamoDB table (default: \"external-dns\")").Default(defaultConfig.AWSDynamoDBTable).StringVar(&cfg.AWSDynamoDBTable)
 

pkg/apis/externaldns/types_test.go

@@ -75,7 +75,7 @@ var (
 		AzureSubscriptionID:         "",
 		CloudflareProxied:           false,
 		CloudflareDNSRecordsPerPage: 100,
-		CloudflareRegionKey:		 "",
+		CloudflareRegionKey:         "",
 		CoreDNSPrefix:               "/skydns/",
 		AkamaiServiceConsumerDomain: "",
 		AkamaiClientToken:           "",
@@ -97,6 +97,7 @@ var (
 		TXTOwnerID:                  "default",
 		TXTPrefix:                   "",
 		TXTCacheInterval:            0,
+		TXTNewFormatOnly:            false,
 		Interval:                    time.Minute,
 		MinEventSyncInterval:        5 * time.Second,
 		Once:                        false,
@@ -176,7 +177,7 @@ var (
 		AzureSubscriptionID:         "arg",
 		CloudflareProxied:           true,
 		CloudflareDNSRecordsPerPage: 5000,
-		CloudflareRegionKey: 		 "us",
+		CloudflareRegionKey:         "us",
 		CoreDNSPrefix:               "/coredns/",
 		AkamaiServiceConsumerDomain: "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net",
 		AkamaiClientToken:           "o184671d5307a388180fbf7f11dbdf46",
@@ -202,6 +203,7 @@ var (
 		TXTOwnerID:                  "owner-1",
 		TXTPrefix:                   "associated-txt-record",
 		TXTCacheInterval:            12 * time.Hour,
+		TXTNewFormatOnly:            true,
 		Interval:                    10 * time.Minute,
 		MinEventSyncInterval:        50 * time.Second,
 		Once:                        true,
@@ -338,6 +340,7 @@ func TestParseFlags(t *testing.T) {
 				"--txt-owner-id=owner-1",
 				"--txt-prefix=associated-txt-record",
 				"--txt-cache-interval=12h",
+				"--txt-new-format-only",
 				"--dynamodb-table=custom-table",
 				"--interval=10m",
 				"--min-event-sync-interval=50s",
@@ -399,7 +402,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_AZURE_SUBSCRIPTION_ID":           "arg",
 				"EXTERNAL_DNS_CLOUDFLARE_PROXIED":              "1",
 				"EXTERNAL_DNS_CLOUDFLARE_DNS_RECORDS_PER_PAGE": "5000",
-				"EXTERNAL_DNS_CLOUDFLARE_REGION_KEY":			"us",
+				"EXTERNAL_DNS_CLOUDFLARE_REGION_KEY":           "us",
 				"EXTERNAL_DNS_COREDNS_PREFIX":                  "/coredns/",
 				"EXTERNAL_DNS_AKAMAI_SERVICECONSUMERDOMAIN":    "oooo-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net",
 				"EXTERNAL_DNS_AKAMAI_CLIENT_TOKEN":             "o184671d5307a388180fbf7f11dbdf46",
@@ -451,6 +454,7 @@ func TestParseFlags(t *testing.T) {
 				"EXTERNAL_DNS_TXT_OWNER_ID":                    "owner-1",
 				"EXTERNAL_DNS_TXT_PREFIX":                      "associated-txt-record",
 				"EXTERNAL_DNS_TXT_CACHE_INTERVAL":              "12h",
+				"EXTERNAL_DNS_TXT_NEW_FORMAT_ONLY":             "1",
 				"EXTERNAL_DNS_INTERVAL":                        "10m",
 				"EXTERNAL_DNS_MIN_EVENT_SYNC_INTERVAL":         "50s",
 				"EXTERNAL_DNS_ONCE":                            "1",

registry/txt.go

@@ -58,10 +58,18 @@ type TXTRegistry struct {
 	// encrypt text records
 	txtEncryptEnabled bool
 	txtEncryptAESKey  []byte
+
+	newFormatOnly bool
 }
 
-// NewTXTRegistry returns new TXTRegistry object
-func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string, cacheInterval time.Duration, txtWildcardReplacement string, managedRecordTypes, excludeRecordTypes []string, txtEncryptEnabled bool, txtEncryptAESKey []byte) (*TXTRegistry, error) {
+// NewTXTRegistry returns a new TXTRegistry object. When newFormatOnly is true, it will only
+// generate new format TXT records, otherwise it generates both old and new formats for
+// backwards compatibility.
+func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string,
+	cacheInterval time.Duration, txtWildcardReplacement string,
+	managedRecordTypes, excludeRecordTypes []string,
+	txtEncryptEnabled bool, txtEncryptAESKey []byte,
+	newFormatOnly bool) (*TXTRegistry, error) {
 	if ownerID == "" {
 		return nil, errors.New("owner id cannot be empty")
 	}
@@ -95,6 +103,7 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st
 		excludeRecordTypes:  excludeRecordTypes,
 		txtEncryptEnabled:   txtEncryptEnabled,
 		txtEncryptAESKey:    txtEncryptAESKey,
+		newFormatOnly:       newFormatOnly,
 	}, nil
 }
 
@@ -216,12 +225,14 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 	return endpoints, nil
 }
 
-// generateTXTRecord generates both "old" and "new" TXT records.
-// Once we decide to drop old format we need to drop toTXTName() and rename toNewTXTName
+// generateTXTRecord generates TXT records in either both formats (old and new) or new format only,
+// depending on the newFormatOnly configuration. The old format is maintained for backwards
+// compatibility but can be disabled to reduce the number of DNS records.
 func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpoint {
 	endpoints := make([]*endpoint.Endpoint, 0)
 
-	if !im.txtEncryptEnabled && !im.mapper.recordTypeInAffix() && r.RecordType != endpoint.RecordTypeAAAA {
+	// Create legacy format record by default unless newFormatOnly is true
+	if !im.newFormatOnly && !im.txtEncryptEnabled && !im.mapper.recordTypeInAffix() && r.RecordType != endpoint.RecordTypeAAAA {
 		// old TXT record format
 		txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey))
 		if txt != nil {
@@ -231,7 +242,8 @@ func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpo
 			endpoints = append(endpoints, txt)
 		}
 	}
-	// new TXT record format (containing record type)
+
+	// Always create new format record
 	recordType := r.RecordType
 	// AWS Alias records are encoded as type "cname"
 	if isAlias, found := r.GetProviderSpecificProperty("alias"); found && isAlias == "true" && recordType == endpoint.RecordTypeA {

registry/txt_encryption_test.go

@@ -60,7 +60,7 @@ func TestNewTXTRegistryEncryptionConfig(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
-		actual, err := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}, []string{}, test.encEnabled, test.aesKeyRaw)
+		actual, err := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}, []string{}, test.encEnabled, test.aesKeyRaw, false)
 		if test.errorExpected {
 			require.Error(t, err)
 		} else {
@@ -106,7 +106,7 @@ func TestGenerateTXTGenerateTextRecordEncryptionWihDecryption(t *testing.T) {
 		for _, k := range withEncryptionKeys {
 			t.Run(fmt.Sprintf("key '%s' with decrypted result '%s'", k, test.decrypted), func(t *testing.T) {
 				key := []byte(k)
-				r, err := NewTXTRegistry(p, "", "", "owner", time.Minute, "", []string{}, []string{}, true, key)
+				r, err := NewTXTRegistry(p, "", "", "owner", time.Minute, "", []string{}, []string{}, true, key, false)
 				assert.NoError(t, err, "Error creating TXT registry")
 				txtRecords := r.generateTXTRecord(test.record)
 				assert.Len(t, txtRecords, len(test.record.Targets))
@@ -143,7 +143,7 @@ func TestApplyRecordsWithEncryption(t *testing.T) {
 
 	key := []byte("ZPitL0NGVQBZbTD6DwXJzD8RiStSazzYXQsdUowLURY=")
 
-	r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, key)
+	r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, key, false)
 
 	_ = r.ApplyChanges(ctx, &plan.Changes{
 		Create: []*endpoint.Endpoint{
@@ -201,7 +201,7 @@ func TestApplyRecordsWithEncryptionKeyChanged(t *testing.T) {
 	}
 
 	for _, key := range withEncryptionKeys {
-		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key))
+		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key), false)
 		_ = r.ApplyChanges(ctx, &plan.Changes{
 			Create: []*endpoint.Endpoint{
 				newEndpointWithOwner("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"),
@@ -231,7 +231,7 @@ func TestApplyRecordsOnEncryptionKeyChangeWithKeyIdLabel(t *testing.T) {
 	}
 
 	for i, key := range withEncryptionKeys {
-		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key))
+		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key), false)
 		keyId := fmt.Sprintf("key-id-%d", i)
 		changes := []*endpoint.Endpoint{
 			newEndpointWithOwnerAndOwnedRecordWithKeyIDLabel("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "", keyId),