Skip to content

Commit cda9ad3

Browse files
authored
VAULT-33074: add github sub-command to pipeline (#29403)
* VAULT-33074: add `github` sub-command to `pipeline` Investigating test workflow failures is common task that engineers on the sustaining rotation perform. This task often requires quite a bit of manual labor by manually inspecting all failed/cancelled workflows in the Github UI on per repo/branch/workflow basis and performing root cause analysis. As we work to improve our pipeline discoverability this PR adds a new `github` sub-command to the `pipeline` utility that allows querying for such workflows and returning either machine readable or human readable summaries in a single place. Eventually we plan to automate sending a summary of this data to an OTEL collector automatically but for now sustaining engineers can utilize it to query for workflows with lots of various criteria. A common pattern for investigating build/enos test failure workflows would be: ```shell export GITHUB_TOKEN="YOUR_TOKEN" go run -race ./tools/pipeline/... github list-workflow-runs -o hashicorp -r vault -d '2025-01-13..2025-01-23' --branch main --status failure build ``` This will list `build` workflow runs in `hashicorp/vault` repo for the `main` branch with the `status` or `conclusion` of `failure` within the date range of `2025-01-13..2025-01-23`. A sustaining engineer will likely do this for both `vault` and `vault-enterprise` repositories along with `enos-release-testing-oss` and `enos-release-testing-ent` workflows in addition to `build` in order to get a full picture of the last weeks failures. You can also use this utility to summarize workflows based on other statuses, branches, HEAD SHA's, event triggers, github actors, etc. For a full list of filter arguments you can pass `-h` to the sub-command. > [!CAUTION] > Be careful not to run this without setting strict filter arguments. > Failing to do so could result in trying to summarize way too many > workflows resulting in your API token being disabled for an hour. Signed-off-by: Ryan Cragun <[email protected]>
1 parent 6a87419 commit cda9ad3

13 files changed

+1622
-73
lines changed

.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -133,4 +133,6 @@ website/components/node_modules
133133
tools/godoctests/.bin
134134
tools/gonilnilfunctions/.bin
135135
tools/codechecker/.bin
136-
.ci-bootstrap
136+
tools/pipeline/.bin
137+
tools/pipeline/pipeline
138+
.ci-bootstrap

tools/pipeline/go.mod

+22-20
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ go 1.23.2
44

55
require (
66
github.com/Masterminds/semver v1.5.0
7-
github.com/hashicorp/hcl/v2 v2.22.0
7+
github.com/google/go-github/v68 v68.0.0
8+
github.com/hashicorp/hcl/v2 v2.23.0
89
github.com/hashicorp/releases-api v0.1.23
910
github.com/spf13/cobra v1.8.1
10-
github.com/stretchr/testify v1.9.0
11+
github.com/stretchr/testify v1.10.0
1112
github.com/veqryn/slog-context v0.7.0
1213
)
1314

@@ -17,29 +18,30 @@ require (
1718
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
1819
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
1920
github.com/docker/go-units v0.5.0 // indirect
20-
github.com/fatih/color v1.16.0 // indirect
21-
github.com/go-logr/logr v1.4.1 // indirect
21+
github.com/fatih/color v1.18.0 // indirect
22+
github.com/go-logr/logr v1.4.2 // indirect
2223
github.com/go-logr/stdr v1.2.2 // indirect
2324
github.com/go-openapi/analysis v0.23.0 // indirect
2425
github.com/go-openapi/errors v0.22.0 // indirect
2526
github.com/go-openapi/jsonpointer v0.21.0 // indirect
2627
github.com/go-openapi/jsonreference v0.21.0 // indirect
2728
github.com/go-openapi/loads v0.22.0 // indirect
28-
github.com/go-openapi/runtime v0.26.0 // indirect
29+
github.com/go-openapi/runtime v0.28.0 // indirect
2930
github.com/go-openapi/spec v0.21.0 // indirect
3031
github.com/go-openapi/strfmt v0.23.0 // indirect
3132
github.com/go-openapi/swag v0.23.0 // indirect
3233
github.com/go-openapi/validate v0.24.0 // indirect
3334
github.com/google/go-cmp v0.6.0 // indirect
35+
github.com/google/go-querystring v1.1.0 // indirect
3436
github.com/google/uuid v1.6.0 // indirect
3537
github.com/hashicorp/go-hclog v1.6.3 // indirect
3638
github.com/hashicorp/go-version v1.7.0 // indirect
37-
github.com/imdario/mergo v0.3.15 // indirect
39+
github.com/imdario/mergo v0.3.16 // indirect
3840
github.com/inconshreveable/mousetrap v1.1.0 // indirect
3941
github.com/jessevdk/go-flags v1.6.1 // indirect
4042
github.com/josharian/intern v1.0.0 // indirect
41-
github.com/mailru/easyjson v0.7.7 // indirect
42-
github.com/mattn/go-colorable v0.1.13 // indirect
43+
github.com/mailru/easyjson v0.9.0 // indirect
44+
github.com/mattn/go-colorable v0.1.14 // indirect
4345
github.com/mattn/go-isatty v0.0.20 // indirect
4446
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
4547
github.com/mitchellh/hashstructure v1.1.0 // indirect
@@ -49,17 +51,17 @@ require (
4951
github.com/opentracing/opentracing-go v1.2.0 // indirect
5052
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
5153
github.com/spf13/pflag v1.0.5 // indirect
52-
github.com/zclconf/go-cty v1.15.0 // indirect
53-
go.mongodb.org/mongo-driver v1.14.0 // indirect
54-
go.opentelemetry.io/otel v1.20.0 // indirect
55-
go.opentelemetry.io/otel/metric v1.20.0 // indirect
56-
go.opentelemetry.io/otel/trace v1.20.0 // indirect
57-
golang.org/x/mod v0.21.0 // indirect
58-
golang.org/x/net v0.30.0 // indirect
59-
golang.org/x/sync v0.8.0 // indirect
60-
golang.org/x/sys v0.26.0 // indirect
61-
golang.org/x/text v0.19.0 // indirect
62-
golang.org/x/tools v0.26.0 // indirect
63-
gopkg.in/yaml.v2 v2.4.0 // indirect
54+
github.com/zclconf/go-cty v1.16.2 // indirect
55+
go.mongodb.org/mongo-driver v1.17.2 // indirect
56+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
57+
go.opentelemetry.io/otel v1.34.0 // indirect
58+
go.opentelemetry.io/otel/metric v1.34.0 // indirect
59+
go.opentelemetry.io/otel/trace v1.34.0 // indirect
60+
golang.org/x/mod v0.22.0 // indirect
61+
golang.org/x/net v0.34.0 // indirect
62+
golang.org/x/sync v0.10.0 // indirect
63+
golang.org/x/sys v0.29.0 // indirect
64+
golang.org/x/text v0.21.0 // indirect
65+
golang.org/x/tools v0.29.0 // indirect
6466
gopkg.in/yaml.v3 v3.0.1 // indirect
6567
)

tools/pipeline/go.sum

+52-48
Large diffs are not rendered by default.

tools/pipeline/internal/cmd/github.go

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright (c) HashiCorp, Inc.
2+
// SPDX-License-Identifier: BUSL-1.1
3+
4+
package cmd
5+
6+
import "github.com/spf13/cobra"
7+
8+
type githubCommandFlags struct {
9+
Format string `json:"format,omitempty"`
10+
}
11+
12+
var githubCmdFlags = &githubCommandFlags{}
13+
14+
func newGithubCmd() *cobra.Command {
15+
github := &cobra.Command{
16+
Use: "github",
17+
Short: "Github commands",
18+
Long: "Github commands",
19+
}
20+
21+
github.PersistentFlags().StringVarP(&githubCmdFlags.Format, "format", "f", "table", "The output format. Can be 'json' or 'table'")
22+
23+
github.AddCommand(newGithubListRunCmd())
24+
25+
return github
26+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright (c) HashiCorp, Inc.
2+
// SPDX-License-Identifier: BUSL-1.1
3+
4+
package cmd
5+
6+
import (
7+
"context"
8+
"encoding/json"
9+
"errors"
10+
"fmt"
11+
"os"
12+
"time"
13+
14+
ghclient "github.com/google/go-github/v68/github"
15+
"github.com/spf13/cobra"
16+
17+
"github.com/hashicorp/vault/tools/pipeline/internal/pkg/github"
18+
)
19+
20+
var listGithubWorkflowRuns = &github.ListWorkflowRunsReq{}
21+
22+
func newGithubListRunCmd() *cobra.Command {
23+
listRuns := &cobra.Command{
24+
Use: "list-workflow-runs [WORKFLOW_NAME]",
25+
Short: "List workflow runs",
26+
Long: "List Github Actions workflow runs for a given workflow. Be sure to use filter arguments to reduce the search, otherwise you'll likely hit your API limit.",
27+
RunE: runListGithubWorkflowsCmd,
28+
Args: func(cmd *cobra.Command, args []string) error {
29+
switch len(args) {
30+
case 1:
31+
listGithubWorkflowRuns.WorkflowName = args[0]
32+
return nil
33+
case 0:
34+
return errors.New("no workflow name argument has been provided")
35+
default:
36+
return fmt.Errorf("expected a single workflow name as an argument, received (%d): %v", len(args), args)
37+
}
38+
},
39+
}
40+
41+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Actor, "actor", "a", "", "Filter using a specific Github actor")
42+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Branch, "branch", "b", "", "Filter using a specific Github branch")
43+
listRuns.PersistentFlags().Int64VarP(&listGithubWorkflowRuns.CheckSuiteID, "check-suite-id", "c", 0, "Filter using a specific Github check suite")
44+
listRuns.PersistentFlags().BoolVar(&listGithubWorkflowRuns.Compact, "compact", true, "When given a status filter, only fetch data for workflows, jobs, checks, and annotations that match our status and/or conclusion. Disabling compact mode with a large query range might result in Github throttling the requests.")
45+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.DateQuery, "date-query", "d", fmt.Sprintf("%s..*", time.Now().Add(-168*time.Hour).Format(time.DateOnly)), "Filter using a date range query. It supports the Github ISO8601-ish date range query format. Default is newer than one week ago")
46+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Event, "event", "e", "", "Filter using a workflow triggered by an event type. E.g. push, pull_request, issue")
47+
listRuns.PersistentFlags().BoolVarP(&listGithubWorkflowRuns.IncludePRs, "include-prs", "p", false, "Include workflow runs triggered via pull requests")
48+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Owner, "owner", "o", "hashicorp", "The Github organization")
49+
listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Repo, "repo", "r", "vault", "The Github repository. Private repositories require auth via a GITHUB_TOKEN env var")
50+
listRuns.PersistentFlags().StringVar(&listGithubWorkflowRuns.Sha, "sha", "", "Filter based on the HEAD SHA associated with the workflow run")
51+
listRuns.PersistentFlags().StringVar(&listGithubWorkflowRuns.Status, "status", "", "Filter by a given run status. For example: completed, cancelled, failure, skipped, success, in_progress")
52+
53+
return listRuns
54+
}
55+
56+
func runListGithubWorkflowsCmd(cmd *cobra.Command, args []string) error {
57+
cmd.SilenceUsage = true // Don't spam the usage on failure
58+
59+
client := ghclient.NewClient(nil)
60+
if token, set := os.LookupEnv("GITHUB_TOKEN"); set {
61+
client = client.WithAuthToken(token)
62+
} else {
63+
fmt.Println("\x1b[1;33;49mWARNING\x1b[0m: GITHUB_TOKEN has not been set. While not required for public repositories you're likely to get throttled without it")
64+
}
65+
66+
res, err := listGithubWorkflowRuns.Run(context.TODO(), client)
67+
if err != nil {
68+
return fmt.Errorf("listing github workflow failures: %w", err)
69+
}
70+
71+
switch githubCmdFlags.Format {
72+
case "json":
73+
b, err := json.Marshal(res)
74+
if err != nil {
75+
return fmt.Errorf("marshaling response to JSON: %w", err)
76+
}
77+
fmt.Println(string(b))
78+
default:
79+
for _, run := range res.Runs {
80+
summary, err := run.Summary()
81+
if err != nil {
82+
return fmt.Errorf("generating workflow run response summary: %w", err)
83+
}
84+
fmt.Println(summary)
85+
}
86+
}
87+
88+
return err
89+
}

tools/pipeline/internal/cmd/root.go

+1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func newRootCmd() *cobra.Command {
2828
rootCmd.PersistentFlags().StringVar(&rootCfg.logLevel, "log", "warn", "Set the log level. One of 'debug', 'info', 'warn', 'error'")
2929

3030
rootCmd.AddCommand(newGenerateCmd())
31+
rootCmd.AddCommand(newGithubCmd())
3132
rootCmd.AddCommand(newReleasesCmd())
3233

3334
rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {

tools/pipeline/internal/pkg/generate/enos_dynamic_config_test.go

+13-4
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@ import (
1010
"slices"
1111
"testing"
1212

13-
"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"
1413
"github.com/stretchr/testify/require"
14+
15+
"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"
1516
)
1617

1718
var testAPIVersions = []string{
@@ -230,21 +231,29 @@ func Test_EnosDynamicConfigReq_Run(t *testing.T) {
230231
AWSRegion: []string{"us-east-1", "us-west-2"},
231232
DistroVersionAmzn: []string{"2023"},
232233
DistroVersionLeap: []string{"15.6"},
233-
DistroVersionRhel: []string{"8.10, 9.4"},
234+
DistroVersionRhel: []string{"8.10", "9.4"},
234235
DistroVersionSles: []string{"15.6"},
235236
DistroVersionUbuntu: []string{"20.04", "24.04"},
236237
UpgradeInitialVersion: versions,
237238
},
238239
},
239240
}
240241
},
241-
hcl: []byte(`
242+
hcl: []byte(`# Copyright (c) HashiCorp, Inc.
243+
# SPDX-License-Identifier: BUSL-1.1
244+
245+
# Code generated by pipeline generate enos-dynamic-config DO NOT EDIT.
246+
247+
# This file is overwritten in CI as it contains branch specific and sometimes ever-changing values.
248+
# It's checked in here so that enos samples and scenarios can be performed, just be aware that this
249+
# might change out from under you.
250+
242251
globals {
243252
sample_attributes = {
244253
aws_region = ["us-east-1", "us-west-2"]
245254
distro_version_amzn = ["2023"]
246255
distro_version_leap = ["15.6"]
247-
distro_version_rhel = ["8.10, 9.4"]
256+
distro_version_rhel = ["8.10", "9.4"]
248257
distro_version_sles = ["15.6"]
249258
distro_version_ubuntu = ["20.04", "24.04"]
250259
upgrade_initial_version = ["1.16.6", "1.16.7", "1.16.8", "1.16.9", "1.16.10", "1.17.3", "1.17.4", "1.17.6", "1.18.0-rc1"]

0 commit comments

Comments
 (0)