history: make sure build record is finalized before exporting #3152
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -248,6 +248,28 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q | |
| return out, nil | ||
| } | ||
|
|
||
| func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error { | ||
| eg, ctx := errgroup.WithContext(ctx) | ||
| for _, node := range nodes { | ||
| node := node | ||
| eg.Go(func() error { | ||
| if node.Driver == nil { | ||
| return nil | ||
| } | ||
| c, err := node.Driver.Client(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| _, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{ | ||
| Ref: ref, | ||
| Finalize: true, | ||
| }) | ||
| return err | ||
| }) | ||
| } | ||
|
Comment on lines
+253
to
+269
There was a problem hiding this comment. @tonistiigi For multi-node case I think this is safe even if the ref doesn't exist on unrelated node: https://github.com/moby/buildkit/blob/b46daef219fd318bd0f9c8f630472af771b2bb4a/solver/llbsolver/history.go#L676-L686 |
||
| return eg.Wait() | ||
| } | ||
|
|
||
| func formatDuration(d time.Duration) string { | ||
| if d < time.Minute { | ||
| return fmt.Sprintf("%.1fs", d.Seconds()) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| package tests | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/moby/buildkit/util/testutil/integration" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| var historyTests = []func(t *testing.T, sb integration.Sandbox){ | ||
| testHistoryExport, | ||
| testHistoryExportFinalize, | ||
| testHistoryInspect, | ||
| testHistoryLs, | ||
| testHistoryRm, | ||
| } | ||
|
|
||
| func testHistoryExport(t *testing.T, sb integration.Sandbox) { | ||
| ref := buildTestProject(t, sb) | ||
| require.NotEmpty(t, ref.Ref) | ||
|
|
||
| outFile := path.Join(t.TempDir(), "export.dockerbuild") | ||
| cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile)) | ||
| out, err := cmd.Output() | ||
| require.NoError(t, err, string(out)) | ||
| require.FileExists(t, outFile) | ||
| } | ||
|
|
||
| func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) { | ||
| ref := buildTestProject(t, sb) | ||
| require.NotEmpty(t, ref.Ref) | ||
|
|
||
| outFile := path.Join(t.TempDir(), "export.dockerbuild") | ||
| cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile)) | ||
| out, err := cmd.Output() | ||
| require.NoError(t, err, string(out)) | ||
| require.FileExists(t, outFile) | ||
| } | ||
|
|
||
| func testHistoryInspect(t *testing.T, sb integration.Sandbox) { | ||
| ref := buildTestProject(t, sb) | ||
| require.NotEmpty(t, ref.Ref) | ||
|
|
||
| cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json")) | ||
| out, err := cmd.Output() | ||
| require.NoError(t, err, string(out)) | ||
|
|
||
| type recT struct { | ||
| Name string | ||
| Ref string | ||
| Context string | ||
| Dockerfile string | ||
| StartedAt *time.Time | ||
| CompletedAt *time.Time | ||
| Duration time.Duration | ||
| Status string | ||
| NumCompletedSteps int32 | ||
| NumTotalSteps int32 | ||
| NumCachedSteps int32 | ||
| } | ||
| var rec recT | ||
| err = json.Unmarshal(out, &rec) | ||
| require.NoError(t, err) | ||
| require.Equal(t, ref.Ref, rec.Ref) | ||
| require.NotEmpty(t, rec.Name) | ||
| } | ||
|
|
||
| func testHistoryLs(t *testing.T, sb integration.Sandbox) { | ||
| ref := buildTestProject(t, sb) | ||
| require.NotEmpty(t, ref.Ref) | ||
|
|
||
| cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json")) | ||
| out, err := cmd.Output() | ||
| require.NoError(t, err, string(out)) | ||
|
|
||
| type recT struct { | ||
| Ref string `json:"ref"` | ||
| Name string `json:"name"` | ||
| Status string `json:"status"` | ||
| CreatedAt *time.Time `json:"created_at"` | ||
| CompletedAt *time.Time `json:"completed_at"` | ||
| TotalSteps int32 `json:"total_steps"` | ||
| CompletedSteps int32 `json:"completed_steps"` | ||
| CachedSteps int32 `json:"cached_steps"` | ||
| } | ||
| var rec recT | ||
| err = json.Unmarshal(out, &rec) | ||
| require.NoError(t, err) | ||
| require.Equal(t, ref.String(), rec.Ref) | ||
| require.NotEmpty(t, rec.Name) | ||
| } | ||
|
|
||
| func testHistoryRm(t *testing.T, sb integration.Sandbox) { | ||
| ref := buildTestProject(t, sb) | ||
| require.NotEmpty(t, ref.Ref) | ||
|
|
||
| cmd := buildxCmd(sb, withArgs("history", "rm", ref.Ref)) | ||
| out, err := cmd.Output() | ||
| require.NoError(t, err, string(out)) | ||
| } | ||
|
|
||
| type buildRef struct { | ||
| Builder string | ||
| Node string | ||
| Ref string | ||
| } | ||
|
|
||
| func (b buildRef) String() string { | ||
| return b.Builder + "/" + b.Node + "/" + b.Ref | ||
| } | ||
|
|
||
| func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef { | ||
| dir := createTestProject(t) | ||
| out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir)) | ||
| require.NoError(t, err, string(out)) | ||
|
|
||
| dt, err := os.ReadFile(filepath.Join(dir, "md.json")) | ||
| require.NoError(t, err) | ||
|
|
||
| type mdT struct { | ||
| BuildRef string `json:"buildx.build.ref"` | ||
| } | ||
| var md mdT | ||
| err = json.Unmarshal(dt, &md) | ||
| require.NoError(t, err) | ||
|
|
||
| refParts := strings.Split(md.BuildRef, "/") | ||
| require.Len(t, refParts, 3) | ||
|
|
||
| return buildRef{ | ||
| Builder: refParts[0], | ||
| Node: refParts[1], | ||
| Ref: refParts[2], | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.