Removing Stale alert manager templates files (#4495)

CHANGELOG.md

@@ -394,6 +394,7 @@
 * [BUGFIX] Ruler: fixed counting of PromQL evaluation errors as user-errors when updating `cortex_ruler_queries_failed_total`. #4335
 * [BUGFIX] Ingester: fixed ingester stuck on start up (LEAVING ring state) when `-ingester.heartbeat-period=0` and `-ingester.unregister-on-shutdown=false`. #4366
 * [BUGFIX] Ingester: When using block storage, prevent any reads or writes while the ingester is stopping. This will prevent accessing TSDB blocks once they have been already closed. #4304
+* [BUGFIX] AlertManager: remove stale template files. #4495
 
 ## 1.10.0-rc.0 / 2021-06-28
 

pkg/alertmanager/multitenant.go

@@ -817,14 +817,25 @@ func (am *MultitenantAlertmanager) setConfig(cfg alertspb.AlertConfigDesc) error
 	var userAmConfig *amconfig.Config
 	var err error
 	var hasTemplateChanges bool
+	var userTemplateDir = filepath.Join(am.getTenantDirectory(cfg.User), templatesDir)
+	var pathsToRemove = make(map[string]struct{})
+
+	// List existing files to keep track the ones to be removed
+	if oldTemplateFiles, err := ioutil.ReadDir(userTemplateDir); err == nil {
+		for _, file := range oldTemplateFiles {
+			pathsToRemove[filepath.Join(userTemplateDir, file.Name())] = struct{}{}
+		}
+	}
 
 	for _, tmpl := range cfg.Templates {
-		templateFilepath, err := safeTemplateFilepath(filepath.Join(am.getTenantDirectory(cfg.User), templatesDir), tmpl.Filename)
+		templateFilePath, err := safeTemplateFilepath(userTemplateDir, tmpl.Filename)
 		if err != nil {
 			return err
 		}
 
-		hasChanged, err := storeTemplateFile(templateFilepath, tmpl.Body)
+		// Removing from pathsToRemove map the files that still exists in the config
+		delete(pathsToRemove, templateFilePath)
+		hasChanged, err := storeTemplateFile(templateFilePath, tmpl.Body)
 		if err != nil {
 			return err
 		}
@@ -834,6 +845,14 @@ func (am *MultitenantAlertmanager) setConfig(cfg alertspb.AlertConfigDesc) error
 		}
 	}
 
+	for pathToRemove := range pathsToRemove {
+		err := os.Remove(pathToRemove)
+		if err != nil {
+			level.Warn(am.logger).Log("msg", "failed to remove file", "file", pathToRemove, "err", err)
+		}
+		hasTemplateChanges = true
+	}
+
 	level.Debug(am.logger).Log("msg", "setting config", "user", cfg.User)
 
 	am.alertmanagersMtx.Lock()

pkg/alertmanager/multitenant_test.go

@@ -310,6 +310,23 @@ templates:
 		cortex_alertmanager_config_last_reload_successful{user="user2"} 1
 		cortex_alertmanager_config_last_reload_successful{user="user3"} 1
 	`), "cortex_alertmanager_config_last_reload_successful"))
+
+	// Removed template files should be cleaned up
+	user3Cfg.Templates = []*alertspb.TemplateDesc{
+		{
+			Filename: "first.tpl",
+			Body:     `{{ define "t1" }}Template 1 ... {{end}}`,
+		},
+	}
+
+	require.NoError(t, store.SetAlertConfig(ctx, user3Cfg))
+
+	err = am.loadAndSyncConfigs(context.Background(), reasonPeriodic)
+	require.NoError(t, err)
+
+	require.True(t, dirExists(t, user3Dir))
+	require.True(t, fileExists(t, filepath.Join(user3Dir, templatesDir, "first.tpl")))
+	require.False(t, fileExists(t, filepath.Join(user3Dir, templatesDir, "second.tpl")))
 }
 
 func TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled(t *testing.T) {