Feat/add env parse

go.mod

@@ -3,13 +3,14 @@ module github.com/flant/addon-operator
 go 1.22.8
 
 require (
+	github.com/caarlos0/env/v11 v11.2.2
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
-	github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241106140903-258b93b3334e
+	github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c
 	github.com/deckhouse/module-sdk v0.1.0
 	github.com/dominikbraun/graph v0.23.0
 	github.com/ettle/strcase v0.2.0
 	github.com/flant/kube-client v1.2.2
-	github.com/flant/shell-operator v1.5.2
+	github.com/flant/shell-operator v0.0.0-20241206091450-31a456b28f94
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-openapi/loads v0.19.5
 	github.com/go-openapi/spec v0.19.8

go.sum

@@ -50,6 +50,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
+github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg=
+github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -87,8 +89,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241106140903-258b93b3334e h1:QUQy+5Bv7/UzhfrytiG3c5gfLGhPppepVbRpbMisVIw=
-github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241106140903-258b93b3334e/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w=
+github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c h1:dK30IW9uGg0DvSy+IcdQ6zwEBRV55R7tEtaruEKYkSA=
+github.com/deckhouse/deckhouse/pkg/log v0.0.0-20241205040953-7b376bae249c/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w=
 github.com/deckhouse/module-sdk v0.1.0 h1:YgMnr0vmjVLwIdQfTE1ytR2nPndO2SfQprcW45wsT34=
 github.com/deckhouse/module-sdk v0.1.0/go.mod h1:0gxlSr0WRrGnB82J+45FBIXNP/fKl8+yEvRTzrygiAE=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@@ -138,8 +140,8 @@ github.com/flant/kube-client v1.2.2 h1:27LBs+PKJEFnkQXjPU9eIps7a7iyI13AKcSYj897D
 github.com/flant/kube-client v1.2.2/go.mod h1:eMa3aJ6V1PRWSQ/RCROkObDpY4S74uM84SJS4G/LINg=
 github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee h1:evii83J+/6QGNvyf6tjQ/p27DPY9iftxIBb37ALJRTg=
 github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee/go.mod h1:f+REaGl/+pZR97rbTcwHEka/MAipoQQ2Mc0iQUj4ak0=
-github.com/flant/shell-operator v1.5.2 h1:OPYveDCI1ZZDrtxORqarSIQIAcZEGW9ZUugCZtLmiZM=
-github.com/flant/shell-operator v1.5.2/go.mod h1:2vTLeyYLaq4Tp0NPilCHZy22rDGSS4uBsbPONj9YGRY=
+github.com/flant/shell-operator v0.0.0-20241206091450-31a456b28f94 h1:dTa+dI4RMnb9IsazX+saGxhzph4DumqOjHugTLThlzs=
+github.com/flant/shell-operator v0.0.0-20241206091450-31a456b28f94/go.mod h1:suTnsKwcUr4AhWbMz/RvfiN1L7bgovhXqtApqYka02k=
 github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=

pkg/app/app.go

@@ -170,6 +170,7 @@ func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause)
 	shapp.DefineJqFlags(cmd)
 	shapp.DefineLoggingFlags(cmd)
 
+	// TODO: working with global values not need to be in flags definition
 	shapp.DebugUnixSocket = DefaultDebugUnixSocket
 	shapp.DefineDebugFlags(kpApp, cmd)
 }

pkg/app/app_config.go

@@ -0,0 +1,148 @@
+package app
+
+import (
+	"fmt"
+
+	"github.com/caarlos0/env/v11"
+	shapp "github.com/flant/shell-operator/pkg/app"
+)
+
+type appConfig struct {
+	TmpDir                  string `env:"TMP_DIR"`
+	PrometheusMetricsPrefix string `env:"PROMETHEUS_METRICS_PREFIX"`
+}
+
+func newAppConfig() *appConfig {
+	return &appConfig{}
+}
+
+type Config struct {
+	SOp       *shapp.Config `env:"-"`
+	AppConfig *appConfig    `envPrefix:"ADDON_OPERATOR_"`
+
+	ready bool
+}
+
+func newConfig() *Config {
+	return &Config{
+		AppConfig: newAppConfig(),
+		SOp:       shapp.MustGetConfig(),
+	}
+}
+
+func (cfg *Config) Parse() error {
+	if cfg.ready {
+		return nil
+	}
+
+	opts := env.Options{
+		Prefix: "",
+	}
+
+	err := env.ParseWithOptions(cfg, opts)
+	if err != nil {
+		return fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	return nil
+}
+
+func (cfg *Config) InitConfig() {
+	cfg.OverrideDefaults()
+	cfg.SetupGlobalVars()
+
+	cfg.SetReady()
+}
+
+func (cfg *Config) OverrideDefaults() {
+	if cfg.IsReady() {
+		return
+	}
+
+	setIfNotEmpty(&shapp.TempDir, DefaultTempDir)
+	setIfNotEmpty(&shapp.PrometheusMetricsPrefix, DefaultPrometheusMetricsPrefix)
+	setIfNotEmpty(&shapp.DebugUnixSocket, DefaultDebugUnixSocket)
+}
+
+func (cfg *Config) SetupGlobalVars() {
+	if cfg.IsReady() {
+		return
+	}
+
+	cfg.SOp.SetupGlobalVars()
+
+	setIfNotEmpty(&shapp.TempDir, cfg.AppConfig.TmpDir)
+	setIfNotEmpty(&shapp.PrometheusMetricsPrefix, cfg.AppConfig.PrometheusMetricsPrefix)
+}
+
+func (cfg *Config) IsReady() bool {
+	return cfg.ready
+}
+
+func (cfg *Config) SetReady() {
+	cfg.SOp.SetReady()
+	cfg.ready = true
+}
+
+var configInstance *Config
+
+func MustGetConfig() *Config {
+	cfg, err := GetConfig()
+	if err != nil {
+		panic(err)
+	}
+
+	return cfg
+}
+
+func MustGetAndInitConfig() *Config {
+	cfg, err := GetAndInitConfig()
+	if err != nil {
+		panic(err)
+	}
+
+	return cfg
+}
+
+func GetConfig() (*Config, error) {
+	if configInstance != nil {
+		return configInstance, nil
+	}
+
+	cfg := newConfig()
+	err := cfg.Parse()
+	if err != nil {
+		return nil, err
+	}
+
+	configInstance = cfg
+
+	return configInstance, nil
+}
+
+func GetAndInitConfig() (*Config, error) {
+	cfg, err := GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.InitConfig()
+
+	return cfg, nil
+}
+
+func setIfNotEmpty[T comparable](v *T, env T) {
+	if !isZero(env) {
+		*v = env
+	}
+}
+
+func setSliceIfNotEmpty[T any](v *[]T, env []T) {
+	if len(env) != 0 {
+		*v = env
+	}
+}
+
+func isZero[T comparable](v T) bool {
+	return v == *new(T)
+}

pkg/module_manager/models/hooks/kind/batch_hook.go

@@ -14,7 +14,6 @@ import (
 	"github.com/gofrs/uuid/v5"
 
 	"github.com/flant/addon-operator/pkg/utils"
-	shapp "github.com/flant/shell-operator/pkg/app"
 	"github.com/flant/shell-operator/pkg/executor"
 	sh_hook "github.com/flant/shell-operator/pkg/hook"
 	bindingcontext "github.com/flant/shell-operator/pkg/hook/binding_context"
@@ -104,7 +103,7 @@ func (sh *BatchHook) Execute(configVersion string, bContext []bindingcontext.Bin
 	}
 	// Remove tmp files after execution
 	defer func() {
-		if shapp.DebugKeepTmpFilesVar == "yes" {
+		if sh.KeepTemporaryHookFiles {
 			return
 		}
 		for _, f := range tmpFiles {

pkg/module_manager/models/hooks/kind/shellhook.go

@@ -10,7 +10,6 @@ import (
 	"github.com/gofrs/uuid/v5"
 
 	"github.com/flant/addon-operator/pkg/utils"
-	shapp "github.com/flant/shell-operator/pkg/app"
 	"github.com/flant/shell-operator/pkg/executor"
 	sh_hook "github.com/flant/shell-operator/pkg/hook"
 	bindingcontext "github.com/flant/shell-operator/pkg/hook/binding_context"
@@ -87,16 +86,16 @@ func (sh *ShellHook) Execute(configVersion string, bContext []bindingcontext.Bin
 	versionedContextList := bindingcontext.ConvertBindingContextList(configVersion, bContext)
 	bindingContextBytes, err := versionedContextList.Json()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("versioned context list json: %w", err)
 	}
 
 	tmpFiles, err := sh.prepareTmpFilesForHookRun(bindingContextBytes, moduleSafeName, configValues, values)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("prepare tmp files for hook run: %w", err)
 	}
 	// Remove tmp files after execution
 	defer func() {
-		if shapp.DebugKeepTmpFilesVar == "yes" {
+		if sh.KeepTemporaryHookFiles {
 			return
 		}
 		for _, f := range tmpFiles {
@@ -130,7 +129,7 @@ func (sh *ShellHook) Execute(configVersion string, bContext []bindingcontext.Bin
 	usage, err := cmd.RunAndLogLines(logLabels)
 	result.Usage = usage
 	if err != nil {
-		return result, err
+		return result, fmt.Errorf("run and log lines: %w", err)
 	}
 
 	result.Patches[utils.ConfigMapPatch], err = utils.ValuesPatchFromFile(configValuesPatchPath)
@@ -155,7 +154,7 @@ func (sh *ShellHook) Execute(configVersion string, bContext []bindingcontext.Bin
 
 	result.ObjectPatcherOperations, err = objectpatch.ParseOperations(kubernetesPatchBytes)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("object patch parse operations: %w", err)
 	}
 
 	return result, nil

pkg/module_manager/models/modules/basic.go

@@ -479,7 +479,7 @@ func (bm *BasicModule) RunHooksByBinding(binding sh_op_types.BindingType, logLab
 			// This could happen when the Context is
 			// canceled, or the expected wait time exceeds the Context's Deadline.
 			// The best we can do without proper context usage is to repeat the task.
-			return err
+			return fmt.Errorf("rate limit wait: %w", err)
 		}
 
 		bc := bindingcontext.BindingContext{
@@ -507,7 +507,7 @@ func (bm *BasicModule) RunHooksByBinding(binding sh_op_types.BindingType, logLab
 			err = bm.executeHook(moduleHook, binding, []bindingcontext.BindingContext{bc}, logLabels, metricLabels)
 		}()
 		if err != nil {
-			return err
+			return fmt.Errorf("execute hook: %w", err)
 		}
 	}
 
@@ -803,7 +803,7 @@ func (bm *BasicModule) executeHook(h *hooks.ModuleHook, bindingType sh_op_types.
 	if len(hookResult.ObjectPatcherOperations) > 0 {
 		err = bm.dc.KubeObjectPatcher.ExecuteOperations(hookResult.ObjectPatcherOperations)
 		if err != nil {
-			return err
+			return fmt.Errorf("execute operations: %w", err)
 		}
 	}
 
@@ -813,14 +813,14 @@ func (bm *BasicModule) executeHook(h *hooks.ModuleHook, bindingType sh_op_types.
 		"module": bm.Name,
 	})
 	if err != nil {
-		return err
+		return fmt.Errorf("metric storage send batch: %w", err)
 	}
 
 	// Apply binding actions. (Only Go hook for now).
 	if h.GetKind() == kind.HookKindGo {
 		err = h.ApplyBindingActions(hookResult.BindingActions)
 		if err != nil {
-			return err
+			return fmt.Errorf("apply binding actions: %w", err)
 		}
 	}
 

pkg/module_manager/module_manager.go

@@ -656,7 +656,7 @@ func (mm *ModuleManager) RunModule(moduleName string, logLabels map[string]strin
 	err = bm.RunHooksByBinding(BeforeHelm, logLabels)
 	treg.End()
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("run hooks by binding, before helm: %w", err)
 	}
 
 	treg = trace.StartRegion(context.Background(), "ModuleRun-HelmPhase-helm")
@@ -669,15 +669,15 @@ func (mm *ModuleManager) RunModule(moduleName string, logLabels map[string]strin
 	}
 	helmModule, err := modules.NewHelmModule(bm, mm.defaultNamespace, mm.TempDir, deps, schemaStorage, mm.logger.Named("helm-module"))
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("new helm module: %w", err)
 	}
 	if helmModule != nil {
 		// could be nil, if it doesn't contain helm chart
 		err = helmModule.RunHelmInstall(logLabels)
 	}
 	treg.End()
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("run helm install: %w", err)
 	}
 
 	oldValues := bm.GetValues(false)
@@ -686,7 +686,7 @@ func (mm *ModuleManager) RunModule(moduleName string, logLabels map[string]strin
 	err = bm.RunHooksByBinding(AfterHelm, logLabels)
 	treg.End()
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("run hooks by binding, after helm: %w", err)
 	}
 
 	newValues := bm.GetValues(false)