From 319918ee02240bab8a255fb8a455aea30d999922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:04:24 +0100 Subject: [PATCH 01/13] test: add failing token redact test --- test/jest/acceptance/debuglog.spec.ts | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 test/jest/acceptance/debuglog.spec.ts diff --git a/test/jest/acceptance/debuglog.spec.ts b/test/jest/acceptance/debuglog.spec.ts new file mode 100644 index 0000000000..02123f5d8a --- /dev/null +++ b/test/jest/acceptance/debuglog.spec.ts @@ -0,0 +1,28 @@ +import { runSnykCLI } from '../util/runSnykCLI'; +import { createProjectFromWorkspace } from '../util/createProject'; + + +jest.setTimeout(1000 * 60); + +describe('debug log', () => { + + it('', async () => { + const project = await createProjectFromWorkspace('cocoapods-app'); + const token = "mytoken" + + + const { code, stderr } = await runSnykCLI('test -d', { + cwd: project.path(), + env: { + ...process.env, + SNYK_DISABLE_ANALYTICS: '1', + DEBUG: '*', + SNYK_TOKEN: token + }, + }); + + console.debug(stderr) + expect(stderr).not.toContain(token) + expect(code).toEqual(2); + }); +}); From 8a790c87db9c3f71850f28de3afc1ffe9b7af544 Mon Sep 17 00:00:00 2001 From: Luke Watts Date: Thu, 4 Jan 2024 15:37:26 +0100 Subject: [PATCH 02/13] feat: adopt log scrubbing --- cliv2/go.mod | 6 ++++-- cliv2/go.sum | 4 ++-- cliv2/pkg/basic_workflows/legacycli.go | 8 ++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cliv2/go.mod b/cliv2/go.mod index d754c864b9..bf7d63eaba 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -1,6 +1,8 @@ module github.com/snyk/cli/cliv2 -go 1.18 +go 1.21 + +toolchain go1.21.5 require ( github.com/elazarl/goproxy v0.0.0-20231031074852-3ec07828be7a @@ -13,7 +15,7 @@ require ( github.com/snyk/cli-extension-iac-rules v0.0.0-20230601153200-c572cfce46ce github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f - github.com/snyk/go-application-framework v0.0.0-20231222162659-c767e4a7440b + github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6 github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 github.com/snyk/snyk-iac-capture v0.6.5 github.com/snyk/snyk-ls v0.0.0-20231124091213-5a223c21e0aa diff --git a/cliv2/go.sum b/cliv2/go.sum index 54e170a132..53d73d8272 100644 --- a/cliv2/go.sum +++ b/cliv2/go.sum @@ -665,8 +665,8 @@ github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a h1:oRrk9bv github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a/go.mod h1:IwRGWjRuNkY08O7NJb7u3JuQkroEB8Qi1MlASpZVu1Q= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f h1:ghajT5PEiLP8XNFIdc7Yn4Th74RH/9Q++dDOp6Cb9eo= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM= -github.com/snyk/go-application-framework v0.0.0-20231222162659-c767e4a7440b h1:NNiXGaKELaFmejlw5BOWf8dVThl8iisU9Yhx+FSUrL4= -github.com/snyk/go-application-framework v0.0.0-20231222162659-c767e4a7440b/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= +github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6 h1:x53bcGKTb+2PMuMnFLNXxRt8EbGnKy/B0xsBOBpB4B4= +github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 h1:s9PHNkL6ueYRiAKNfd8OVxlUOqU3qY0VDbgCD1f6WQY= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/snyk/policy-engine v0.22.0 h1:od9pduGrXyfWO791X+8M1qmnvWUxaIXh0gBzGKqeseA= diff --git a/cliv2/pkg/basic_workflows/legacycli.go b/cliv2/pkg/basic_workflows/legacycli.go index 346dccbff1..ee61fb5bea 100644 --- a/cliv2/pkg/basic_workflows/legacycli.go +++ b/cliv2/pkg/basic_workflows/legacycli.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/snyk/go-application-framework/pkg/auth" "github.com/snyk/go-application-framework/pkg/configuration" + "github.com/snyk/go-application-framework/pkg/logging" pkg_utils "github.com/snyk/go-application-framework/pkg/utils" "github.com/snyk/go-application-framework/pkg/workflow" "github.com/snyk/go-httpauth/pkg/httpauth" @@ -124,6 +125,13 @@ func legacycliWorkflow( outWriter = bufio.NewWriter(&outBuffer) errWriter = bufio.NewWriter(&errBuffer) cli.SetIoStreams(in, outWriter, errWriter) + } else { + scrubDict := map[string]bool{} + pointer := config.GetString(configuration.AUTHENTICATION_TOKEN) + scrubDict[pointer] = true + + scrubbedStderr := logging.NewScrubbingIoWriter(os.Stderr, scrubDict) + cli.SetIoStreams(os.Stdin, os.Stdout, scrubbedStderr) } // init proxy object From 1bbf993cafa29cafd90e643e7aea05603a1a255d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:47:02 +0100 Subject: [PATCH 03/13] fix: debuglog test * use common func to get scrubdict * use scrub logger in Extensible CLI --- cliv2/cmd/cliv2/logheader.go | 4 +- cliv2/cmd/cliv2/main.go | 52 +++++++++++++++----------- cliv2/go.mod | 8 ++-- cliv2/go.sum | 4 +- cliv2/pkg/basic_workflows/legacycli.go | 5 +-- test/jest/acceptance/debuglog.spec.ts | 1 + 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/cliv2/cmd/cliv2/logheader.go b/cliv2/cmd/cliv2/logheader.go index 04129518a2..4d55349c6f 100644 --- a/cliv2/cmd/cliv2/logheader.go +++ b/cliv2/cmd/cliv2/logheader.go @@ -36,7 +36,7 @@ func logHeaderAuthorizationInfo( err := networkAccess.AddHeaders(apiRequest) if err != nil { - debugLogger.Print(err) + globalLogger.Print(err) } authHeader := apiRequest.Header.Get("Authorization") @@ -97,7 +97,7 @@ func writeLogHeader(config configuration.Configuration, networkAccess networking } tablePrint := func(name string, value string) { - debugLogger.Printf("%-22s %s", name+":", value) + globalLogger.Printf("%-22s %s", name+":", value) } fipsEnabled := getFipsStatus(config) diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 2452778065..a9a51e4c0d 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -1,7 +1,9 @@ package main // !!! This import needs to be the first import, please do not change this !!! -import _ "github.com/snyk/go-application-framework/pkg/networking/fips_enable" +import ( + _ "github.com/snyk/go-application-framework/pkg/networking/fips_enable" +) import ( "context" @@ -27,6 +29,7 @@ import ( "github.com/snyk/go-application-framework/pkg/auth" "github.com/snyk/go-application-framework/pkg/configuration" localworkflows "github.com/snyk/go-application-framework/pkg/local_workflows" + "github.com/snyk/go-application-framework/pkg/logging" "github.com/snyk/go-application-framework/pkg/networking" "github.com/snyk/go-application-framework/pkg/runtimeinfo" "github.com/snyk/go-application-framework/pkg/utils" @@ -44,7 +47,10 @@ var internalOS string var engine workflow.Engine var globalConfiguration configuration.Configuration var helpProvided bool -var debugLogger = zerolog.New(zerolog.ConsoleWriter{ + +var tempLogger zerolog.Logger = zerolog.New(io.Discard) +var globalLogger *zerolog.Logger = &tempLogger +var consoleWriter = zerolog.ConsoleWriter{ Out: os.Stderr, TimeFormat: time.RFC3339, NoColor: true, @@ -60,7 +66,7 @@ var debugLogger = zerolog.New(zerolog.ConsoleWriter{ t, _ := time.Parse(time.RFC3339, i.(string)) return strings.ToUpper(fmt.Sprintf("%s", t.UTC().Format(time.RFC3339))) }, -}).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() +} const ( unknownCommandMessage string = "unknown command" @@ -81,15 +87,15 @@ const ( handleErrorUnhandled HandleError = iota ) -func getDebugLevel(config configuration.Configuration) zerolog.Level { +func getDebugLevel(config configuration.Configuration, logger *zerolog.Logger) zerolog.Level { loglevel := zerolog.DebugLevel if loglevelString := config.GetString("snyk_log_level"); loglevelString != "" { var err error loglevel, err = zerolog.ParseLevel(loglevelString) if err == nil { - debugLogger.Log().Msgf("Setting log level to %s", loglevelString) + logger.Log().Msgf("Setting log level to %s", loglevelString) } else { - debugLogger.Log().Msgf("%v", err) + logger.Log().Msgf("%v", err) loglevel = zerolog.DebugLevel } } @@ -97,14 +103,16 @@ func getDebugLevel(config configuration.Configuration) zerolog.Level { } func initDebugLogger(config configuration.Configuration) *zerolog.Logger { + localLogger := zerolog.New(logging.NewScrubbingWriter(zerolog.MultiLevelWriter(consoleWriter), logging.GetScrubDictFromConfig(config))).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() debug := config.GetBool(configuration.DEBUG) if !debug { - debugLogger = debugLogger.Output(io.Discard) + debugLogger := localLogger.Output(io.Discard) + return &debugLogger } else { - loglevel := getDebugLevel(config) - debugLogger = debugLogger.Level(loglevel) + loglevel := getDebugLevel(config, &localLogger) + debugLogger := localLogger.Level(loglevel) + return &debugLogger } - return &debugLogger } func main() { @@ -132,7 +140,7 @@ func initApplicationConfiguration(config configuration.Configuration) { formattedKey := strings.ToUpper(key) _, ok := os.LookupEnv(formattedKey) if ok { - debugLogger.Printf("Found environment variable %s, disabling OAuth flow", formattedKey) + globalLogger.Printf("Found environment variable %s, disabling OAuth flow", formattedKey) config.Set(configuration.FF_OAUTH_AUTH_FLOW_ENABLED, false) break } @@ -187,21 +195,21 @@ func runMainWorkflow(config configuration.Configuration, cmd *cobra.Command, arg err := config.AddFlagSet(cmd.Flags()) if err != nil { - debugLogger.Print("Failed to add flags", err) + globalLogger.Print("Failed to add flags", err) return err } updateConfigFromParameter(config, args, rawArgs) name := getFullCommandString(cmd) - debugLogger.Print("Running ", name) + globalLogger.Print("Running ", name) engine.GetAnalytics().SetCommand(name) data, err := engine.Invoke(workflow.NewWorkflowIdentifier(name)) if err == nil { _, err = engine.InvokeWithInput(localworkflows.WORKFLOWID_OUTPUT_WORKFLOW, data) } else { - debugLogger.Print("Failed to execute the command!", err) + globalLogger.Print("Failed to execute the command!", err) } return err @@ -391,14 +399,14 @@ func MainWithErrorCode() int { globalConfiguration = configuration.New() err = globalConfiguration.AddFlagSet(rootCommand.LocalFlags()) if err != nil { - debugLogger.Print("Failed to add flags to root command", err) + fmt.Fprintln(os.Stderr, "Failed to add flags to root command", err) } debugEnabled := globalConfiguration.GetBool(configuration.DEBUG) - debugLogger := initDebugLogger(globalConfiguration) + globalLogger = initDebugLogger(globalConfiguration) initApplicationConfiguration(globalConfiguration) - engine = app.CreateAppEngineWithOptions(app.WithZeroLogger(debugLogger), app.WithConfiguration(globalConfiguration), app.WithRuntimeInfo(rInfo)) + engine = app.CreateAppEngineWithOptions(app.WithZeroLogger(globalLogger), app.WithConfiguration(globalConfiguration), app.WithRuntimeInfo(rInfo)) if noProxyAuth := globalConfiguration.GetBool(basic_workflows.PROXY_NOAUTH); noProxyAuth { globalConfiguration.Set(configuration.PROXY_AUTHENTICATION_MECHANISM, httpauth.StringFromAuthenticationMechanism(httpauth.NoAuth)) @@ -416,7 +424,7 @@ func MainWithErrorCode() int { // init engine err = engine.Init() if err != nil { - debugLogger.Print("Failed to init Workflow Engine!", err) + globalLogger.Print("Failed to init Workflow Engine!", err) return constants.SNYK_EXIT_CODE_ERROR } @@ -449,7 +457,7 @@ func MainWithErrorCode() int { cliAnalytics.SetCmdArguments(os.Args[1:]) cliAnalytics.SetOperatingSystem(internalOS) if globalConfiguration.GetBool(configuration.ANALYTICS_DISABLED) == false { - defer sendAnalytics(cliAnalytics, debugLogger) + defer sendAnalytics(cliAnalytics, globalLogger) } setTimeout(globalConfiguration, func() { @@ -462,7 +470,7 @@ func MainWithErrorCode() int { // fallback to the legacy cli or show help handleErrorResult := handleError(err) if handleErrorResult == handleErrorFallbackToLegacyCLI { - debugLogger.Printf("Using Legacy CLI to serve the command. (reason: %v)", err) + globalLogger.Printf("Using Legacy CLI to serve the command. (reason: %v)", err) err = defaultCmd(os.Args[1:]) } else if handleErrorResult == handleErrorShowHelp { err = help(nil, []string{}) @@ -475,7 +483,7 @@ func MainWithErrorCode() int { displayError(err) exitCode := cliv2.DeriveExitCode(err) - debugLogger.Printf("Exiting with %d", exitCode) + globalLogger.Printf("Exiting with %d", exitCode) return exitCode } @@ -485,7 +493,7 @@ func setTimeout(config configuration.Configuration, onTimeout func()) { if timeout == 0 { return } - debugLogger.Printf("Command timeout set for %d seconds", timeout) + globalLogger.Printf("Command timeout set for %d seconds", timeout) go func() { const gracePeriodForSubProcesses = 3 <-time.After(time.Duration(timeout+gracePeriodForSubProcesses) * time.Second) diff --git a/cliv2/go.mod b/cliv2/go.mod index bf7d63eaba..3e9b3fc404 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -1,8 +1,6 @@ module github.com/snyk/cli/cliv2 -go 1.21 - -toolchain go1.21.5 +go 1.18 require ( github.com/elazarl/goproxy v0.0.0-20231031074852-3ec07828be7a @@ -15,7 +13,7 @@ require ( github.com/snyk/cli-extension-iac-rules v0.0.0-20230601153200-c572cfce46ce github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f - github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6 + github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98 github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 github.com/snyk/snyk-iac-capture v0.6.5 github.com/snyk/snyk-ls v0.0.0-20231124091213-5a223c21e0aa @@ -175,3 +173,5 @@ require ( // version 2491eb6c1c75 contains a valid license replace github.com/mattn/go-localereader v0.0.1 => github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 + +//replace github.com/snyk/go-application-framework => ../../go-application-framework diff --git a/cliv2/go.sum b/cliv2/go.sum index 53d73d8272..1bc1daeb38 100644 --- a/cliv2/go.sum +++ b/cliv2/go.sum @@ -665,8 +665,8 @@ github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a h1:oRrk9bv github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a/go.mod h1:IwRGWjRuNkY08O7NJb7u3JuQkroEB8Qi1MlASpZVu1Q= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f h1:ghajT5PEiLP8XNFIdc7Yn4Th74RH/9Q++dDOp6Cb9eo= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM= -github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6 h1:x53bcGKTb+2PMuMnFLNXxRt8EbGnKy/B0xsBOBpB4B4= -github.com/snyk/go-application-framework v0.0.0-20240103162526-aea591663ca6/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= +github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98 h1:MsUkF4G04utkTFZlLdOwQc6tvj+O7n2AnzLiqsn6Kyg= +github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 h1:s9PHNkL6ueYRiAKNfd8OVxlUOqU3qY0VDbgCD1f6WQY= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/snyk/policy-engine v0.22.0 h1:od9pduGrXyfWO791X+8M1qmnvWUxaIXh0gBzGKqeseA= diff --git a/cliv2/pkg/basic_workflows/legacycli.go b/cliv2/pkg/basic_workflows/legacycli.go index ee61fb5bea..887299ae23 100644 --- a/cliv2/pkg/basic_workflows/legacycli.go +++ b/cliv2/pkg/basic_workflows/legacycli.go @@ -126,10 +126,7 @@ func legacycliWorkflow( errWriter = bufio.NewWriter(&errBuffer) cli.SetIoStreams(in, outWriter, errWriter) } else { - scrubDict := map[string]bool{} - pointer := config.GetString(configuration.AUTHENTICATION_TOKEN) - scrubDict[pointer] = true - + scrubDict := logging.GetScrubDictFromConfig(config) scrubbedStderr := logging.NewScrubbingIoWriter(os.Stderr, scrubDict) cli.SetIoStreams(os.Stdin, os.Stdout, scrubbedStderr) } diff --git a/test/jest/acceptance/debuglog.spec.ts b/test/jest/acceptance/debuglog.spec.ts index 02123f5d8a..97b57ede2f 100644 --- a/test/jest/acceptance/debuglog.spec.ts +++ b/test/jest/acceptance/debuglog.spec.ts @@ -17,6 +17,7 @@ describe('debug log', () => { ...process.env, SNYK_DISABLE_ANALYTICS: '1', DEBUG: '*', + SNYK_LOG_LEVEL: 'trace', SNYK_TOKEN: token }, }); From 8940b97c991bda8ba62def397ef49413fb72e461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:59:30 +0100 Subject: [PATCH 04/13] chore: run formatter --- test/jest/acceptance/debuglog.spec.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/jest/acceptance/debuglog.spec.ts b/test/jest/acceptance/debuglog.spec.ts index 97b57ede2f..4af9d39eab 100644 --- a/test/jest/acceptance/debuglog.spec.ts +++ b/test/jest/acceptance/debuglog.spec.ts @@ -1,15 +1,12 @@ import { runSnykCLI } from '../util/runSnykCLI'; import { createProjectFromWorkspace } from '../util/createProject'; - jest.setTimeout(1000 * 60); describe('debug log', () => { - it('', async () => { const project = await createProjectFromWorkspace('cocoapods-app'); - const token = "mytoken" - + const token = 'mytoken'; const { code, stderr } = await runSnykCLI('test -d', { cwd: project.path(), @@ -18,12 +15,12 @@ describe('debug log', () => { SNYK_DISABLE_ANALYTICS: '1', DEBUG: '*', SNYK_LOG_LEVEL: 'trace', - SNYK_TOKEN: token + SNYK_TOKEN: token, }, }); - console.debug(stderr) - expect(stderr).not.toContain(token) + console.debug(stderr); + expect(stderr).not.toContain(token); expect(code).toEqual(2); }); }); From 956d05c7c62c55e2f49c8a749d6fa4798344fea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:13:17 +0100 Subject: [PATCH 05/13] fix: acceptance test through gaf upgrade --- cliv2/go.mod | 6 ++++-- cliv2/go.sum | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cliv2/go.mod b/cliv2/go.mod index 3e9b3fc404..3ce864f30d 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -1,6 +1,8 @@ module github.com/snyk/cli/cliv2 -go 1.18 +go 1.21 + +toolchain go1.21.0 require ( github.com/elazarl/goproxy v0.0.0-20231031074852-3ec07828be7a @@ -13,7 +15,7 @@ require ( github.com/snyk/cli-extension-iac-rules v0.0.0-20230601153200-c572cfce46ce github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f - github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98 + github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3 github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 github.com/snyk/snyk-iac-capture v0.6.5 github.com/snyk/snyk-ls v0.0.0-20231124091213-5a223c21e0aa diff --git a/cliv2/go.sum b/cliv2/go.sum index 1bc1daeb38..59b8aa85ee 100644 --- a/cliv2/go.sum +++ b/cliv2/go.sum @@ -665,8 +665,8 @@ github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a h1:oRrk9bv github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a/go.mod h1:IwRGWjRuNkY08O7NJb7u3JuQkroEB8Qi1MlASpZVu1Q= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f h1:ghajT5PEiLP8XNFIdc7Yn4Th74RH/9Q++dDOp6Cb9eo= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM= -github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98 h1:MsUkF4G04utkTFZlLdOwQc6tvj+O7n2AnzLiqsn6Kyg= -github.com/snyk/go-application-framework v0.0.0-20240104125928-9393c2d9dc98/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= +github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3 h1:N654kYG5LFwD9FAYqjorLZrOF7buTpcYx1wzXXIPFOM= +github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 h1:s9PHNkL6ueYRiAKNfd8OVxlUOqU3qY0VDbgCD1f6WQY= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/snyk/policy-engine v0.22.0 h1:od9pduGrXyfWO791X+8M1qmnvWUxaIXh0gBzGKqeseA= From c67b7733a4f7dcada45d460369b308c78fdc3781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:14:11 +0100 Subject: [PATCH 06/13] refactor: cleaning up logger initialization --- cliv2/cmd/cliv2/main.go | 45 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index a9a51e4c0d..77816d4a4a 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -48,25 +48,8 @@ var engine workflow.Engine var globalConfiguration configuration.Configuration var helpProvided bool -var tempLogger zerolog.Logger = zerolog.New(io.Discard) -var globalLogger *zerolog.Logger = &tempLogger -var consoleWriter = zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.RFC3339, - NoColor: true, - PartsOrder: []string{ - zerolog.TimestampFieldName, - "ext", - "separator", - zerolog.CallerFieldName, - zerolog.MessageFieldName, - }, - FieldsExclude: []string{"ext", "separator"}, - FormatTimestamp: func(i interface{}) string { - t, _ := time.Parse(time.RFC3339, i.(string)) - return strings.ToUpper(fmt.Sprintf("%s", t.UTC().Format(time.RFC3339))) - }, -} +var noopLogger zerolog.Logger = zerolog.New(io.Discard) +var globalLogger *zerolog.Logger = &noopLogger const ( unknownCommandMessage string = "unknown command" @@ -103,12 +86,30 @@ func getDebugLevel(config configuration.Configuration, logger *zerolog.Logger) z } func initDebugLogger(config configuration.Configuration) *zerolog.Logger { - localLogger := zerolog.New(logging.NewScrubbingWriter(zerolog.MultiLevelWriter(consoleWriter), logging.GetScrubDictFromConfig(config))).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() debug := config.GetBool(configuration.DEBUG) if !debug { - debugLogger := localLogger.Output(io.Discard) - return &debugLogger + return &noopLogger } else { + var consoleWriter = zerolog.ConsoleWriter{ + Out: os.Stderr, + TimeFormat: time.RFC3339, + NoColor: true, + PartsOrder: []string{ + zerolog.TimestampFieldName, + "ext", + "separator", + zerolog.CallerFieldName, + zerolog.MessageFieldName, + }, + FieldsExclude: []string{"ext", "separator"}, + FormatTimestamp: func(i interface{}) string { + t, _ := time.Parse(time.RFC3339, i.(string)) + return strings.ToUpper(fmt.Sprintf("%s", t.UTC().Format(time.RFC3339))) + }, + } + + scrubLogger := logging.NewScrubbingWriter(zerolog.MultiLevelWriter(consoleWriter), logging.GetScrubDictFromConfig(config)) + localLogger := zerolog.New(scrubLogger).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() loglevel := getDebugLevel(config, &localLogger) debugLogger := localLogger.Level(loglevel) return &debugLogger From ce17669af4dd0b976242ac753f4b8693dd3645c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:21:48 +0100 Subject: [PATCH 07/13] fix: toolchain --- cliv2/go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cliv2/go.mod b/cliv2/go.mod index 3ce864f30d..19c05a4cca 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -1,8 +1,6 @@ module github.com/snyk/cli/cliv2 -go 1.21 - -toolchain go1.21.0 +go 1.18 require ( github.com/elazarl/goproxy v0.0.0-20231031074852-3ec07828be7a From 3dd1580ba7cbbca4287cc04cd93bacf0519ae605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:18:31 +0100 Subject: [PATCH 08/13] refactor: move debug logic in separate file to reduce main --- cliv2/cmd/cliv2/debug.go | 58 ++++++++++++++++++++++++++++++++++++++++ cliv2/cmd/cliv2/main.go | 47 -------------------------------- 2 files changed, 58 insertions(+), 47 deletions(-) create mode 100644 cliv2/cmd/cliv2/debug.go diff --git a/cliv2/cmd/cliv2/debug.go b/cliv2/cmd/cliv2/debug.go new file mode 100644 index 0000000000..a1df06e160 --- /dev/null +++ b/cliv2/cmd/cliv2/debug.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/snyk/go-application-framework/pkg/configuration" + "github.com/snyk/go-application-framework/pkg/logging" +) + +func getDebugLevel(config configuration.Configuration, logger *zerolog.Logger) zerolog.Level { + loglevel := zerolog.DebugLevel + if loglevelString := config.GetString("snyk_log_level"); loglevelString != "" { + var err error + loglevel, err = zerolog.ParseLevel(loglevelString) + if err == nil { + logger.Log().Msgf("Setting log level to %s", loglevelString) + } else { + logger.Log().Msgf("%v", err) + loglevel = zerolog.DebugLevel + } + } + return loglevel +} + +func initDebugLogger(config configuration.Configuration) *zerolog.Logger { + debug := config.GetBool(configuration.DEBUG) + if !debug { + return &noopLogger + } else { + var consoleWriter = zerolog.ConsoleWriter{ + Out: os.Stderr, + TimeFormat: time.RFC3339, + NoColor: true, + PartsOrder: []string{ + zerolog.TimestampFieldName, + "ext", + "separator", + zerolog.CallerFieldName, + zerolog.MessageFieldName, + }, + FieldsExclude: []string{"ext", "separator"}, + FormatTimestamp: func(i interface{}) string { + t, _ := time.Parse(time.RFC3339, i.(string)) + return strings.ToUpper(fmt.Sprintf("%s", t.UTC().Format(time.RFC3339))) + }, + } + + scrubLogger := logging.NewScrubbingWriter(zerolog.MultiLevelWriter(consoleWriter), logging.GetScrubDictFromConfig(config)) + localLogger := zerolog.New(scrubLogger).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() + loglevel := getDebugLevel(config, &localLogger) + debugLogger := localLogger.Level(loglevel) + return &debugLogger + } +} diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 77816d4a4a..0993339545 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -29,7 +29,6 @@ import ( "github.com/snyk/go-application-framework/pkg/auth" "github.com/snyk/go-application-framework/pkg/configuration" localworkflows "github.com/snyk/go-application-framework/pkg/local_workflows" - "github.com/snyk/go-application-framework/pkg/logging" "github.com/snyk/go-application-framework/pkg/networking" "github.com/snyk/go-application-framework/pkg/runtimeinfo" "github.com/snyk/go-application-framework/pkg/utils" @@ -70,52 +69,6 @@ const ( handleErrorUnhandled HandleError = iota ) -func getDebugLevel(config configuration.Configuration, logger *zerolog.Logger) zerolog.Level { - loglevel := zerolog.DebugLevel - if loglevelString := config.GetString("snyk_log_level"); loglevelString != "" { - var err error - loglevel, err = zerolog.ParseLevel(loglevelString) - if err == nil { - logger.Log().Msgf("Setting log level to %s", loglevelString) - } else { - logger.Log().Msgf("%v", err) - loglevel = zerolog.DebugLevel - } - } - return loglevel -} - -func initDebugLogger(config configuration.Configuration) *zerolog.Logger { - debug := config.GetBool(configuration.DEBUG) - if !debug { - return &noopLogger - } else { - var consoleWriter = zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.RFC3339, - NoColor: true, - PartsOrder: []string{ - zerolog.TimestampFieldName, - "ext", - "separator", - zerolog.CallerFieldName, - zerolog.MessageFieldName, - }, - FieldsExclude: []string{"ext", "separator"}, - FormatTimestamp: func(i interface{}) string { - t, _ := time.Parse(time.RFC3339, i.(string)) - return strings.ToUpper(fmt.Sprintf("%s", t.UTC().Format(time.RFC3339))) - }, - } - - scrubLogger := logging.NewScrubbingWriter(zerolog.MultiLevelWriter(consoleWriter), logging.GetScrubDictFromConfig(config)) - localLogger := zerolog.New(scrubLogger).With().Str("ext", "main").Str("separator", "-").Timestamp().Logger() - loglevel := getDebugLevel(config, &localLogger) - debugLogger := localLogger.Level(loglevel) - return &debugLogger - } -} - func main() { errorCode := MainWithErrorCode() os.Exit(errorCode) From c3dc34af93b5c9a6abee956ad9e87ec3a288c34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:22:51 +0100 Subject: [PATCH 09/13] fix: init order and redact tests --- cliv2/cmd/cliv2/main.go | 4 +++- test/jest/acceptance/debuglog.spec.ts | 26 ++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 0993339545..9e6dcbac63 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -356,10 +356,12 @@ func MainWithErrorCode() int { fmt.Fprintln(os.Stderr, "Failed to add flags to root command", err) } + // ensure to init configuration before using it + initApplicationConfiguration(globalConfiguration) + debugEnabled := globalConfiguration.GetBool(configuration.DEBUG) globalLogger = initDebugLogger(globalConfiguration) - initApplicationConfiguration(globalConfiguration) engine = app.CreateAppEngineWithOptions(app.WithZeroLogger(globalLogger), app.WithConfiguration(globalConfiguration), app.WithRuntimeInfo(rInfo)) if noProxyAuth := globalConfiguration.GetBool(basic_workflows.PROXY_NOAUTH); noProxyAuth { diff --git a/test/jest/acceptance/debuglog.spec.ts b/test/jest/acceptance/debuglog.spec.ts index 4af9d39eab..ae41a97ba6 100644 --- a/test/jest/acceptance/debuglog.spec.ts +++ b/test/jest/acceptance/debuglog.spec.ts @@ -4,11 +4,11 @@ import { createProjectFromWorkspace } from '../util/createProject'; jest.setTimeout(1000 * 60); describe('debug log', () => { - it('', async () => { + it('redacts token from env var', async () => { const project = await createProjectFromWorkspace('cocoapods-app'); const token = 'mytoken'; - const { code, stderr } = await runSnykCLI('test -d', { + const { stderr } = await runSnykCLI('test -d', { cwd: project.path(), env: { ...process.env, @@ -19,8 +19,26 @@ describe('debug log', () => { }, }); - console.debug(stderr); expect(stderr).not.toContain(token); - expect(code).toEqual(2); + }); + + it('redacts token from config file', async () => { + const project = await createProjectFromWorkspace('cocoapods-app'); + + const config = await runSnykCLI('config get api') + const expectedToken = config.stdout.trim() + + const { stderr } = await runSnykCLI('test -d', { + cwd: project.path(), + env: { + ...process.env, + SNYK_DISABLE_ANALYTICS: '1', + DEBUG: '*', + SNYK_LOG_LEVEL: 'trace', + }, + }); + + expect(expectedToken).not.toBeFalsy() + expect(stderr).not.toContain(expectedToken); }); }); From b91ffda44e81a6c8c6344dc41d8f9758929402a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:24:47 +0100 Subject: [PATCH 10/13] chore: run formatter --- test/jest/acceptance/debuglog.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/jest/acceptance/debuglog.spec.ts b/test/jest/acceptance/debuglog.spec.ts index ae41a97ba6..9824e4ea1f 100644 --- a/test/jest/acceptance/debuglog.spec.ts +++ b/test/jest/acceptance/debuglog.spec.ts @@ -25,8 +25,8 @@ describe('debug log', () => { it('redacts token from config file', async () => { const project = await createProjectFromWorkspace('cocoapods-app'); - const config = await runSnykCLI('config get api') - const expectedToken = config.stdout.trim() + const config = await runSnykCLI('config get api'); + const expectedToken = config.stdout.trim(); const { stderr } = await runSnykCLI('test -d', { cwd: project.path(), @@ -38,7 +38,7 @@ describe('debug log', () => { }, }); - expect(expectedToken).not.toBeFalsy() + expect(expectedToken).not.toBeFalsy(); expect(stderr).not.toContain(expectedToken); }); }); From 81a25c394dcf44fa6af4e534b82f8019c488fb12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:27:20 +0100 Subject: [PATCH 11/13] chore: use final gaf commit hash --- cliv2/go.mod | 2 +- cliv2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cliv2/go.mod b/cliv2/go.mod index 19c05a4cca..6189108415 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -13,7 +13,7 @@ require ( github.com/snyk/cli-extension-iac-rules v0.0.0-20230601153200-c572cfce46ce github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f - github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3 + github.com/snyk/go-application-framework v0.0.0-20240105122614-54e2b7c15259 github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 github.com/snyk/snyk-iac-capture v0.6.5 github.com/snyk/snyk-ls v0.0.0-20231124091213-5a223c21e0aa diff --git a/cliv2/go.sum b/cliv2/go.sum index 59b8aa85ee..30000dee4e 100644 --- a/cliv2/go.sum +++ b/cliv2/go.sum @@ -665,8 +665,8 @@ github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a h1:oRrk9bv github.com/snyk/cli-extension-sbom v0.0.0-20231123083311-52b1cecc1a7a/go.mod h1:IwRGWjRuNkY08O7NJb7u3JuQkroEB8Qi1MlASpZVu1Q= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f h1:ghajT5PEiLP8XNFIdc7Yn4Th74RH/9Q++dDOp6Cb9eo= github.com/snyk/container-cli v0.0.0-20230920093251-fe865879a91f/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM= -github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3 h1:N654kYG5LFwD9FAYqjorLZrOF7buTpcYx1wzXXIPFOM= -github.com/snyk/go-application-framework v0.0.0-20240105100745-c885f5887ea3/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= +github.com/snyk/go-application-framework v0.0.0-20240105122614-54e2b7c15259 h1:u6CV0KCHuqPINEs83CbVCjsxG5wMxa42T5HtMvKgm+o= +github.com/snyk/go-application-framework v0.0.0-20240105122614-54e2b7c15259/go.mod h1:Yz/qxFyfhf0xbA+z8Vzr5IM9IDG+BS+2PiGaP1yAsEw= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 h1:s9PHNkL6ueYRiAKNfd8OVxlUOqU3qY0VDbgCD1f6WQY= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/snyk/policy-engine v0.22.0 h1:od9pduGrXyfWO791X+8M1qmnvWUxaIXh0gBzGKqeseA= From 6cec0843edba5881d443467698d4b5ecb5aea0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 14:03:42 +0100 Subject: [PATCH 12/13] chore: undo unneccessary change --- cliv2/cmd/cliv2/main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 9e6dcbac63..ad4b700bb3 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -1,9 +1,7 @@ package main // !!! This import needs to be the first import, please do not change this !!! -import ( - _ "github.com/snyk/go-application-framework/pkg/networking/fips_enable" -) +import _ "github.com/snyk/go-application-framework/pkg/networking/fips_enable" import ( "context" From 323724fc27edf58a375a7fc54353948109c58288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Scha=CC=88fer?= <101886095+PeterSchafer@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:51:04 +0100 Subject: [PATCH 13/13] fix: add missing fips_enable import --- cliv2/cmd/cliv2/debug.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cliv2/cmd/cliv2/debug.go b/cliv2/cmd/cliv2/debug.go index a1df06e160..d2a94ac039 100644 --- a/cliv2/cmd/cliv2/debug.go +++ b/cliv2/cmd/cliv2/debug.go @@ -1,5 +1,8 @@ package main +// !!! This import needs to be the first import, please do not change this !!! +import _ "github.com/snyk/go-application-framework/pkg/networking/fips_enable" + import ( "fmt" "os"