diff --git a/README.md b/README.md index aa68ef47..2b18cae5 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,12 @@ For all other platforms, you can download the binary directly from [the latest r Run a Docker build from a HCL, JSON, or Compose file using Depot's remote builder infrastructure. This command accepts all the command line flags as Docker's `docker buildx bake` command, you can run `depot bake --help` for the full list. -The `bake` command needs to know which [project](https://depot.dev/docs/core-concepts#projects) id to route the build to. For passing the project id you have three options available to you: +The `bake` command needs to know which [project](https://depot.dev/docs/core-concepts#projects) id to route the build to. For passing the project id you have four options available to you: 1. Run `depot init` at the root of your repository and commit the resulting `depot.json` file 2. Use the `--project` flag in your `depot bake` command -3. Set the `DEPOT_PROJECT_ID` environment variable which will be automatically detected +3. Set the `DEPOT_PROJECT_ID` environment variable which will be automatically detected. +4. Use [`x-depot` ](http://depot.dev/docs/cli/reference#compose-support) extension field in your `docker-compose.yml` file. By default, `depot bake` will leave the built image in the remote builder cache. If you would like to download the image to your local Docker daemon (for instance, to `docker run` the result), you can use the `--load` flag. @@ -109,6 +110,43 @@ If you want to build a specific target in the bake file, you can specify it in t depot bake -f docker-bake.hcl original ``` +#### compose support + +Depot supports using bake to build [Docker Compose](https://depot.dev/blog/depot-with-docker-compose) files. + +To use `depot bake` with a Docker Compose file, you can specify the file with the `-f` flag: + +```shell +depot bake -f docker-compose.yml +``` + +Compose files have special extensions prefixed with `x-` to give additional information to the build process. + +In this example, the `x-bake` extension is used to specify the tags for each service +and the `x-depot` extension is used to specify different project IDs for each. + +```yaml +services: + mydb: + build: + dockerfile: ./Dockerfile.db + x-bake: + tags: + - ghcr.io/myorg/mydb:latest + - ghcr.io/myorg/mydb:v1.0.0 + x-depot: + project-id: 1234567890 + myapp: + build: + dockerfile: ./Dockerfile.app + x-bake: + tags: + - ghcr.io/myorg/myapp:latest + - ghcr.io/myorg/myapp:v1.0.0 + x-depot: + project-id: 9876543210 +``` + #### Flags for `bake` | Name | Description | diff --git a/pkg/buildx/bake/bake.go b/pkg/buildx/bake/bake.go index 2b54cd9e..c20f64a9 100644 --- a/pkg/buildx/bake/bake.go +++ b/pkg/buildx/bake/bake.go @@ -628,6 +628,8 @@ type Target struct { // linked is a private field to mark a target used as a linked one linked bool + + ProjectID string `json:"project_id,omitempty" hcl:"project_id,optional" cty:"project_id"` } var _ hclparser.WithEvalContexts = &Target{} @@ -730,6 +732,9 @@ func (t *Target) Merge(t2 *Target) { if t2.NoCacheFilter != nil { // merge t.NoCacheFilter = append(t.NoCacheFilter, t2.NoCacheFilter...) } + if t2.ProjectID != "" { + t.ProjectID = t2.ProjectID + } t.Inherits = append(t.Inherits, t2.Inherits...) } @@ -927,16 +932,50 @@ func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func( return value.AsString(), nil } -func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { - m2 := make(map[string]build.Options, len(m)) - for k, v := range m { - bo, err := toBuildOpt(v, inp) +type DepotBakeOptions struct { + ProjectTargetOptions map[string]map[string]build.Options +} + +// input is only used for remote bake. +func NewDepotBakeOptions(defaultProjectID string, targets map[string]*Target, input *Input) (*DepotBakeOptions, error) { + opts := &DepotBakeOptions{ + ProjectTargetOptions: map[string]map[string]build.Options{}, + } + + for targetName, target := range targets { + projectID := target.ProjectID + if projectID == "" { + projectID = defaultProjectID + } + if projectID == "" { + return nil, errors.Errorf("Project ID is missing for target %s, please specify with --project, DEPOT_PROJECT_ID, or run `depot init`", targetName) + } + buildOpt, err := toBuildOpt(target, input) if err != nil { return nil, err } - m2[k] = *bo + + if _, ok := opts.ProjectTargetOptions[projectID]; !ok { + opts.ProjectTargetOptions[projectID] = map[string]build.Options{} + } + opts.ProjectTargetOptions[projectID][targetName] = *buildOpt + } + + return opts, nil +} + +// ProjectOpts returns the targeted build options for a specific project ID. +func (o *DepotBakeOptions) ProjectOpts(id string) map[string]build.Options { + return o.ProjectTargetOptions[id] +} + +// ProjectIDs returns the x-depot project IDs. +func (o *DepotBakeOptions) ProjectIDs() []string { + projectIDs := make([]string, 0, len(o.ProjectTargetOptions)) + for projectID := range o.ProjectTargetOptions { + projectIDs = append(projectIDs, projectID) } - return m2, nil + return projectIDs } func updateContext(t *build.Inputs, inp *Input) { diff --git a/pkg/buildx/bake/compose.go b/pkg/buildx/bake/compose.go index 8c4e5fe4..801809d9 100644 --- a/pkg/buildx/bake/compose.go +++ b/pkg/buildx/bake/compose.go @@ -232,6 +232,10 @@ type xbake struct { // docs/manuals/bake/compose-file.md#extension-field-with-x-bake } +type XDepot struct { + ProjectID string `yaml:"project-id,omitempty"` +} + type stringMap map[string]string type stringArray []string @@ -253,6 +257,18 @@ func (sa *stringArray) UnmarshalYAML(unmarshal func(interface{}) error) error { // composeExtTarget converts Compose build extension x-bake to bake Target // https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension func (t *Target) composeExtTarget(exts map[string]interface{}) error { + var xdepot XDepot + buf, ok := exts["x-depot"] + if ok && buf != nil { + yb, _ := yaml.Marshal(buf) + if err := yaml.Unmarshal(yb, &xdepot); err != nil { + return err + } + if xdepot.ProjectID != "" { + t.ProjectID = xdepot.ProjectID + } + } + var xb xbake ext, ok := exts["x-bake"] diff --git a/pkg/buildx/commands/bake.go b/pkg/buildx/commands/bake.go index 842f041e..dd5c4700 100644 --- a/pkg/buildx/commands/bake.go +++ b/pkg/buildx/commands/bake.go @@ -42,7 +42,7 @@ type BakeOptions struct { DepotOptions } -func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (err error) { +func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator, printer *progresshelper.SharedPrinter) (err error) { ctx := appcontext.Context() ctx, end, err := tracing.TraceCurrentCommand(ctx, "bake") @@ -53,29 +53,16 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er end(err) }() - ctx2, cancel := context.WithCancel(context.TODO()) - - printer, err := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress) - if err != nil { - cancel() - return err - } - defer func() { // There is extra logic far below that will also do a printer.Wait() // if there are no errors. We want to control when the buildx printer // finishes writing so that we can write our own information such as // linting without it being interleaved. if printer != nil && err != nil { - err1 := printer.Wait() - if err == nil && !errors.Is(err1, context.Canceled) { - err = err1 - } + _ = printer.Wait() } }() - defer cancel() - if os.Getenv("DEPOT_NO_SUMMARY_LINK") == "" { progress.Write(printer, "[depot] build: "+in.buildURL, func() error { return err }) } @@ -95,11 +82,21 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er return err } - buildOpts, requestedTargets, err := validator.Validate(ctx, nodes, printer) + validatedOpts, _, err := validator.Validate(ctx, nodes, printer) if err != nil { return err } + buildOpts := validatedOpts.ProjectOpts(in.project) + if buildOpts == nil { + return fmt.Errorf("project %s build options not found", in.project) + } + + requestedTargets := make([]string, 0, len(buildOpts)) + for target := range buildOpts { + requestedTargets = append(requestedTargets, target) + } + var ( pullOpts map[string]load.PullOptions // Only used for failures to pull images. @@ -165,13 +162,14 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er } dt[buildRes.Name] = metadata } - if err := writeMetadataFile(in.metadataFile, in.project, in.buildID, requestedTargets, dt); err != nil { + err = writeMetadataFile(in.metadataFile, in.project, in.buildID, requestedTargets, dt) + if err != nil { return err } } if in.sbomDir != "" { - err := sbom.Save(ctx, in.sbomDir, resp) + err = sbom.Save(ctx, in.sbomDir, resp) if err != nil { return err } @@ -197,7 +195,7 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er }(i, requestedTargets) } - err := eg.Wait() + err = eg.Wait() if err != nil && !errors.Is(err, context.Canceled) { // For now, we will fallback by rebuilding with load. if in.exportLoad { @@ -213,7 +211,7 @@ func RunBake(dockerCli command.Cli, in BakeOptions, validator BakeValidator) (er _ = printer.Wait() if in.save { - printSaveUsage(in.project, in.buildID, in.progress, requestedTargets) + printSaveHelp(in.project, in.buildID, in.progress, requestedTargets) } linter.Print(os.Stderr, in.progress) return nil @@ -267,7 +265,7 @@ func BakeCmd(dockerCli command.Cli) *cobra.Command { var ( validator BakeValidator - validatedOpts map[string]buildx.Options + validatedOpts *bake.DepotBakeOptions ) if isRemoteTarget(args) { validator = NewRemoteBakeValidator(options, args) @@ -280,49 +278,73 @@ func BakeCmd(dockerCli command.Cli) *cobra.Command { } } - req := helpers.NewBakeRequest( - options.project, - validatedOpts, - helpers.UsingDepotFeatures{ - Push: options.exportPush, - Load: options.exportLoad, - Save: options.save, - Lint: options.lint, - }, - ) - build, err := helpers.BeginBuild(context.Background(), req, token) + projectIDs := validatedOpts.ProjectIDs() + + printer, err := progresshelper.NewSharedPrinter(options.progress) if err != nil { return err } - var buildErr error - defer func() { - build.Finish(buildErr) - PrintBuildURL(build.BuildURL, options.progress) - }() - - options.builderOptions = []builder.Option{builder.WithDepotOptions(buildPlatform, build)} - buildProject := build.BuildProject() - if buildProject != "" { - options.project = buildProject + for range projectIDs { + printer.Add() } - if options.save { - options.additionalCredentials = build.AdditionalCredentials() - options.additionalTags = build.AdditionalTags() - } - options.buildID = build.ID - options.buildURL = build.BuildURL - options.token = build.Token - options.build = &build - if options.allowNoOutput { - _ = os.Setenv("BUILDX_NO_DEFAULT_LOAD", "1") + eg, ctx := errgroup.WithContext(context.Background()) + for _, projectID := range projectIDs { + options.project = projectID + bakeOpts := validatedOpts.ProjectOpts(projectID) + + req := helpers.NewBakeRequest( + options.project, + bakeOpts, + helpers.UsingDepotFeatures{ + Push: options.exportPush, + Load: options.exportLoad, + Save: options.save, + Lint: options.lint, + }, + ) + build, err := helpers.BeginBuild(context.Background(), req, token) + if err != nil { + return err + } + var buildErr error + defer func() { + build.Finish(buildErr) + PrintBuildURL(build.BuildURL, options.progress) + }() + + options.builderOptions = []builder.Option{builder.WithDepotOptions(buildPlatform, build)} + + buildProject := build.BuildProject() + if buildProject != "" { + options.project = buildProject + } + if options.save { + options.additionalCredentials = build.AdditionalCredentials() + options.additionalTags = build.AdditionalTags() + } + options.buildID = build.ID + options.buildURL = build.BuildURL + options.token = build.Token + options.build = &build + + if options.allowNoOutput { + _ = os.Setenv("BUILDX_NO_DEFAULT_LOAD", "1") + } + + func(c command.Cli, o BakeOptions, v BakeValidator, p *progresshelper.SharedPrinter) { + eg.Go(func() error { + buildErr = retryRetryableErrors(ctx, func() error { + return RunBake(c, o, v, p) + }) + + return rewriteFriendlyErrors(buildErr) + }) + }(dockerCli, options, validator, printer) } - buildErr = retryRetryableErrors(context.Background(), func() error { - return RunBake(dockerCli, options, validator) - }) - return rewriteFriendlyErrors(buildErr) + return eg.Wait() }, } @@ -379,7 +401,7 @@ var ( // BakeValidator returns either local or remote build options for targets as well as the targets themselves. type BakeValidator interface { - Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (opts map[string]buildx.Options, targets []string, err error) + Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (opts *bake.DepotBakeOptions, targets []string, err error) } type LocalBakeValidator struct { @@ -387,7 +409,7 @@ type LocalBakeValidator struct { bakeTargets bakeTargets once sync.Once - buildOpts map[string]buildx.Options + buildOpts *bake.DepotBakeOptions targets []string err error } @@ -399,7 +421,7 @@ func NewLocalBakeValidator(options BakeOptions, args []string) *LocalBakeValidat } } -func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ progress.Writer) (map[string]buildx.Options, []string, error) { +func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ progress.Writer) (*bake.DepotBakeOptions, []string, error) { // Using a sync.Once because I _think_ the bake file may not always be read // more than one time such as passed over stdin. t.once.Do(func() { @@ -448,7 +470,7 @@ func (t *LocalBakeValidator) Validate(ctx context.Context, _ []builder.Node, _ p } } - t.buildOpts, t.err = bake.TargetsToBuildOpt(targets, nil) + t.buildOpts, t.err = bake.NewDepotBakeOptions(t.options.project, targets, nil) }) return t.buildOpts, t.targets, t.err @@ -466,7 +488,7 @@ func NewRemoteBakeValidator(options BakeOptions, args []string) *RemoteBakeValid } } -func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (map[string]buildx.Options, []string, error) { +func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node, pw progress.Writer) (*bake.DepotBakeOptions, []string, error) { files, inp, err := bake.ReadRemoteFiles(ctx, builder.ToBuildxNodes(nodes), t.bakeTargets.FileURL, t.options.files, pw) if err != nil { return nil, nil, err @@ -499,7 +521,7 @@ func (t *RemoteBakeValidator) Validate(ctx context.Context, nodes []builder.Node requestedTargets = append(requestedTargets, target) } - opts, err := bake.TargetsToBuildOpt(targets, inp) + opts, err := bake.NewDepotBakeOptions(t.options.project, targets, inp) return opts, requestedTargets, err } @@ -534,8 +556,8 @@ func parseBakeTargets(targets []string) (bkt bakeTargets) { return bkt } -// printSaveUsage prints instructions to pull or push the saved targets. -func printSaveUsage(project, buildID, progressMode string, requestedTargets []string) { +// printSaveHelp prints instructions to pull or push the saved targets. +func printSaveHelp(project, buildID, progressMode string, requestedTargets []string) { if progressMode != progress.PrinterModeQuiet { fmt.Fprintln(os.Stderr) saved := "target" diff --git a/pkg/buildx/commands/build.go b/pkg/buildx/commands/build.go index a402b5b2..3e09dc59 100644 --- a/pkg/buildx/commands/build.go +++ b/pkg/buildx/commands/build.go @@ -350,7 +350,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, nodes []builder.No printWarnings(os.Stderr, printer.Warnings(), progressMode) if depotOpts.save { - printSaveUsage(depotOpts.project, depotOpts.buildID, progressMode, nil) + printSaveHelp(depotOpts.project, depotOpts.buildID, progressMode, nil) } linter.Print(os.Stderr, progressMode) diff --git a/pkg/buildx/commands/lint.go b/pkg/buildx/commands/lint.go index 209a36f4..6a12b991 100644 --- a/pkg/buildx/commands/lint.go +++ b/pkg/buildx/commands/lint.go @@ -84,13 +84,13 @@ type Linter struct { FailureMode LintFailure Clients []*client.Client BuildxNodes []builder.Node - printer *progress.Printer + printer progress.Writer mu sync.Mutex issues map[string][]client.VertexWarning } -func NewLinter(printer *progress.Printer, failureMode LintFailure, clients []*client.Client, nodes []builder.Node) *Linter { +func NewLinter(printer progress.Writer, failureMode LintFailure, clients []*client.Client, nodes []builder.Node) *Linter { return &Linter{ FailureMode: failureMode, Clients: clients, diff --git a/pkg/progresshelper/shared.go b/pkg/progresshelper/shared.go new file mode 100644 index 00000000..e9415f18 --- /dev/null +++ b/pkg/progresshelper/shared.go @@ -0,0 +1,75 @@ +package progresshelper + +import ( + "context" + "os" + "sync" + "sync/atomic" + + "github.com/docker/buildx/util/progress" + "github.com/moby/buildkit/client" + "github.com/opencontainers/go-digest" +) + +var _ progress.Writer = (*SharedPrinter)(nil) + +// SharedPrinter is a reference counted progress.Writer that can be used +// to share progress updates between several concurrent builds. +// Originally used for bake files with multiple projects. +// +// the `Wait()` method will wait until all writers have +// run Wait(). +type SharedPrinter struct { + wg sync.WaitGroup + printer *progress.Printer + cancel context.CancelFunc + + numPrinters atomic.Int32 +} + +func NewSharedPrinter(mode string) (*SharedPrinter, error) { + ctx, cancel := context.WithCancel(context.Background()) + printer, err := progress.NewPrinter(ctx, os.Stderr, os.Stderr, mode) + if err != nil { + cancel() + return nil, err + } + + return &SharedPrinter{ + printer: printer, + cancel: cancel, + }, nil +} + +// Add increments the reference count of the writer. +// Each call to Add() should be matched with a call to Wait(). +func (w *SharedPrinter) Add() { + w.wg.Add(1) + w.numPrinters.Add(1) +} + +func (w *SharedPrinter) Wait() error { + w.wg.Done() + w.wg.Wait() + + w.cancel() + + lastPrinter := w.numPrinters.Add(-1) == 0 + + // The docker progress writer will only return an + // error if it is a context cancellation error. + // + // Only the last printer will be the one to stop the docker printer as + // the docker printer closes channels. + if lastPrinter { + _ = w.printer.Wait() + } + + return nil +} + +func (w *SharedPrinter) Write(status *client.SolveStatus) { w.printer.Write(status) } +func (w *SharedPrinter) ClearLogSource(v interface{}) { w.printer.ClearLogSource(v) } +func (w *SharedPrinter) ValidateLogSource(d digest.Digest, v interface{}) bool { + return w.printer.ValidateLogSource(d, v) +}