Add support for multiple exporters (#235)

Buildkit 0.13 introduced support for [multiple
exporters](https://docs.docker.com/build/exporters/#multiple-exporters).
We currently return an error in these situations.

Instead, inspect the builder's version when loading the node and relax
this error if we see it's running at least 0.13.

Fixes #21.

CHANGELOG.md

@@ -1,10 +1,16 @@
 ## Unreleased
 
+### Added
+
+- Multiple exports are now allowed if the build daemon is detected to have
+  version 0.13 of Buildkit or newer. (https://github.com/pulumi/pulumi-docker-build/issues/21)
+
 ### Changed
 
 - Upgraded buildx from 0.16.0 to 0.18.0.
 
 ### Fixed
+
 - Custom `# syntax=` directives no longer cause validation errors. (https://github.com/pulumi/pulumi-docker-build/issues/300)
 
 ## 0.0.7 (2024-10-16)

go.mod

@@ -32,6 +32,9 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/theupdateframework/notary v0.7.0
 	github.com/tonistiigi/fsutil v0.0.0-20241121093142-31cf1f437184
+	go.opentelemetry.io/otel/metric v1.29.0
+	go.opentelemetry.io/otel/sdk v1.29.0
+	go.opentelemetry.io/otel/trace v1.29.0
 	go.uber.org/mock v0.5.0
 	golang.org/x/crypto v0.27.0
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
@@ -444,10 +447,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 // indirect
-	go.opentelemetry.io/otel/metric v1.29.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.29.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect
-	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.pennock.tech/tabular v1.1.3 // indirect
 	go.uber.org/atomic v1.11.0 // indirect

provider/internal/cli.go

@@ -141,6 +141,10 @@ func (c *cli) Err() *streams.Out {
 	return streams.NewOut(&c.err)
 }
 
+func (c *cli) SupportsMultipleExports() bool {
+	return c.host.supportsMultipleExports
+}
+
 // rc returns a registry client with matching auth.
 func (c *cli) rc() *regclient.RegClient {
 	hosts := []config.Host{}

provider/internal/cli_test.go

@@ -15,6 +15,7 @@
 package internal
 
 import (
+	"context"
 	"io"
 	"testing"
 
@@ -27,7 +28,7 @@ import (
 func TestExec(t *testing.T) {
 	t.Parallel()
 
-	h, err := newHost(nil)
+	h, err := newHost(context.Background(), nil)
 	require.NoError(t, err)
 	cli, err := wrap(h)
 	require.NoError(t, err)
@@ -44,7 +45,7 @@ func TestWrappedAuth(t *testing.T) {
 	t.Parallel()
 	ecr := "https://1234.dkr.ecr.us-west-2.amazonaws.com"
 
-	realhost, err := newHost(nil)
+	realhost, err := newHost(context.Background(), nil)
 	require.NoError(t, err)
 
 	h := &host{
@@ -102,7 +103,7 @@ func TestWrappedAuth(t *testing.T) {
 	assert.Len(t, h.auths, 2) // In-memory host auth is unchanged.
 
 	// Assert that our on-disk host's auth is untouched.
-	realhostRefreshed, err := newHost(nil)
+	realhostRefreshed, err := newHost(context.Background(), nil)
 	require.NoError(t, err)
 	assert.Equal(t, realhost.auths, realhostRefreshed.auths)
 }

provider/internal/client.go

@@ -58,6 +58,8 @@ type Client interface {
 	ManifestCreate(ctx context.Context, push bool, target string, refs ...string) error
 	ManifestInspect(ctx context.Context, target string) (string, error)
 	ManifestDelete(ctx context.Context, target string) error
+
+	SupportsMultipleExports() bool
 }
 
 // Build encapsulates all of the user-provider build parameters and options.
@@ -88,8 +90,6 @@ func newDockerCLI(config *Config) (*command.DockerCli, error) {
 		return nil, err
 	}
 
-	// TODO: Log some version information for debugging.
-
 	return cli, nil
 }
 

provider/internal/client_test.go

@@ -55,7 +55,7 @@ func TestCustomHost(t *testing.T) {
 	t.Run("env", func(t *testing.T) {
 		t.Setenv("DOCKER_HOST", socket)
 
-		h, err := newHost(nil)
+		h, err := newHost(context.Background(), nil)
 		require.NoError(t, err)
 		cli, err := wrap(h)
 		require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestCustomHost(t *testing.T) {
 
 	t.Run("config", func(t *testing.T) {
 		t.Parallel()
-		h, err := newHost(&Config{Host: socket})
+		h, err := newHost(context.Background(), &Config{Host: socket})
 		require.NoError(t, err)
 		cli, err := wrap(h)
 		require.NoError(t, err)
@@ -290,7 +290,7 @@ func TestBuild(t *testing.T) {
 			ctx := context.Background()
 			cli := testcli(t, true, tt.auths...)
 
-			build, err := tt.args.toBuild(ctx, false)
+			build, err := tt.args.toBuild(ctx, true, false)
 			require.NoError(t, err)
 
 			_, err = cli.Build(ctx, build)
@@ -383,7 +383,7 @@ func TestBuildError(t *testing.T) {
 	ctx := context.Background()
 	cli := testcli(t, true)
 
-	build, err := args.toBuild(ctx, false)
+	build, err := args.toBuild(ctx, true, false)
 	require.NoError(t, err)
 
 	_, err = cli.Build(ctx, build)
@@ -418,7 +418,7 @@ func TestBuildExecError(t *testing.T) {
 	ctx := context.Background()
 	cli := testcli(t, true)
 
-	build, err := args.toBuild(ctx, false)
+	build, err := args.toBuild(ctx, true, false)
 	require.NoError(t, err)
 
 	_, err = cli.Build(ctx, build)
@@ -438,7 +438,7 @@ func TestBuildExecError(t *testing.T) {
 // testcli returns a new standalone CLI instance. Set ping to true if a live
 // daemon is required -- the test will be skipped if the daemon is not available.
 func testcli(t *testing.T, ping bool, auths ...Registry) *cli {
-	h, err := newHost(nil)
+	h, err := newHost(context.Background(), nil)
 	require.NoError(t, err)
 
 	cli, err := wrap(h, auths...)

provider/internal/embed/image-examples.md

@@ -1214,7 +1214,7 @@ image = docker_build.Image("image",
     context={
         "location": "app",
         "named": {
-            "golang_latest": {
+            "golang:latest": {
                 "location": "docker-image://golang@sha256:b8e62cf593cdaff36efd90aa3a37de268e6781a2e68c6610940c48f7cdf36984",
             },
         },

provider/internal/host.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/store/storeutil"
 	"github.com/docker/cli/cli/command"
@@ -35,9 +36,12 @@ type host struct {
 	config   *Config
 	builders map[string]*cachedBuilder
 	auths    map[string]cfgtypes.AuthConfig
+
+	// True if the buildkit daemon is at least v0.13.
+	supportsMultipleExports bool
 }
 
-func newHost(config *Config) (*host, error) {
+func newHost(ctx context.Context, config *Config) (*host, error) {
 	docker, err := newDockerCLI(config)
 	if err != nil {
 		return nil, err
@@ -47,11 +51,13 @@ func newHost(config *Config) (*host, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	h := &host{
-		cli:      docker,
-		config:   config,
-		builders: map[string]*cachedBuilder{},
-		auths:    auths,
+		cli:                     docker,
+		config:                  config,
+		builders:                map[string]*cachedBuilder{},
+		auths:                   auths,
+		supportsMultipleExports: false, // Determined when we boot the builder.
 	}
 	return h, err
 }
@@ -151,10 +157,22 @@ func (h *host) builderFor(build Build) (*cachedBuilder, error) {
 	// Attempt to load nodes in order to determine the builder's driver. Ignore
 	// errors for "exec" builds because it's possible to request builders with
 	// drivers that are unknown to us.
-	nodes, err := b.LoadNodes(context.Background())
+	nodes, err := b.LoadNodes(context.Background(), builder.WithData())
 	if err != nil && !build.ShouldExec() {
 		return nil, fmt.Errorf("loading nodes: %w", err)
 	}
+	// Attempt to determine our builder's buildkit version.
+	for idx := range nodes {
+		if nodes[idx].Version == "" {
+			continue
+		}
+		v, err := semver.ParseTolerant(nodes[idx].Version)
+		if err != nil {
+			return nil, fmt.Errorf("parsing buildkit version %q: %w", nodes[idx].Version, err)
+		}
+		h.supportsMultipleExports = v.GE(semver.MustParse("0.13.0"))
+		break
+	}
 
 	cached := &cachedBuilder{name: b.Name, driver: b.Driver, nodes: nodes}
 	h.builders[opts.Builder] = cached

provider/internal/image.go

@@ -362,7 +362,12 @@ func (i *Image) Check(
 	// :(
 	preview := news.ContainsUnknowns()
 
-	if _, berr := args.validate(preview); berr != nil {
+	cfg := infer.GetConfig[Config](ctx)
+	supportsMultipleExports := true
+	if cfg.host != nil {
+		supportsMultipleExports = cfg.host.supportsMultipleExports
+	}
+	if _, berr := args.validate(supportsMultipleExports, preview); berr != nil {
 		errs := berr.(interface{ Unwrap() []error }).Unwrap()
 		for _, e := range errs {
 			if cf, ok := e.(checkFailure); ok {
@@ -488,9 +493,10 @@ func (b *build) ShouldExec() bool {
 
 func (ia ImageArgs) toBuild(
 	ctx context.Context,
+	supportsMultipleExports bool,
 	preview bool,
 ) (Build, error) {
-	opts, err := ia.validate(preview)
+	opts, err := ia.validate(supportsMultipleExports, preview)
 	if err != nil {
 		return nil, err
 	}
@@ -517,27 +523,29 @@ func (ia ImageArgs) toBuild(
 
 // validate confirms the ImageArgs are valid and returns BuildOptions
 // appropriate for passing to builders.
-func (ia *ImageArgs) validate(preview bool) (controllerapi.BuildOptions, error) {
+func (ia *ImageArgs) validate(supportsMultipleExports, preview bool) (controllerapi.BuildOptions, error) {
 	var multierr error
 
-	if len(ia.Exports) > 1 {
-		multierr = errors.Join(multierr,
-			newCheckFailure(errors.New("multiple exports are currently unsupported"), "exports"),
-		)
-	}
-	if ia.Push && ia.Load {
-		multierr = errors.Join(
-			multierr,
-			newCheckFailure(
-				errors.New("push and load may not be set together at the moment"),
-				"push",
-			),
-		)
-	}
-	if len(ia.Exports) > 0 && (ia.Push || ia.Load) {
-		multierr = errors.Join(multierr,
-			newCheckFailure(errors.New("exports can't be provided with push or load"), "exports"),
-		)
+	if !supportsMultipleExports {
+		if len(ia.Exports) > 1 {
+			multierr = errors.Join(multierr,
+				newCheckFailure(errors.New("multiple exports require a v0.13 buildkit daemon or newer"), "exports"),
+			)
+		}
+		if ia.Push && ia.Load {
+			multierr = errors.Join(
+				multierr,
+				newCheckFailure(
+					errors.New("simultaneous push and load requires a v0.13 buildkit daemon or newer"),
+					"push",
+				),
+			)
+		}
+		if len(ia.Exports) > 0 && (ia.Push || ia.Load) {
+			multierr = errors.Join(multierr,
+				newCheckFailure(errors.New("multiple exports require a v0.13 buildkit daemon or newer"), "exports"),
+			)
+		}
 	}
 
 	dockerfile, context, err := ia.Context.validate(preview, ia.Dockerfile)
@@ -694,7 +702,7 @@ func (i *Image) Create(
 		return id, state, errors.New("buildkit is not supported on this host")
 	}
 
-	build, err := input.toBuild(ctx, preview)
+	build, err := input.toBuild(ctx, cli.SupportsMultipleExports(), preview)
 	if err != nil {
 		return id, state, fmt.Errorf("preparing: %w", err)
 	}