Merge pull request #1101 from tonistiigi/target-exists

exporter: allow oci exporters visibility to response metadata

client/client_test.go

@@ -100,6 +100,7 @@ func TestClientIntegration(t *testing.T) {
 		testExportBusyboxLocal,
 		testBridgeNetworking,
 		testCacheMountNoCache,
+		testExporterTargetExists,
 	}, mirrors)
 
 	integration.Run(t, []integration.Test{
@@ -740,7 +741,7 @@ func testFrontendImageNaming(t *testing.T, sb integration.Sandbox) {
 					case ExporterDocker:
 						outW, err := os.Create(out)
 						require.NoError(t, err)
-						so.Exports[0].Output = outW
+						so.Exports[0].Output = fixedWriteCloser(outW)
 					case ExporterImage:
 						imageName = registry + "/" + imageName
 						so.Exports[0].Attrs["push"] = "true"
@@ -1304,7 +1305,7 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
 				{
 					Type:   exp,
 					Attrs:  attrs,
-					Output: outW,
+					Output: fixedWriteCloser(outW),
 				},
 			},
 		}, nil)
@@ -1388,7 +1389,7 @@ func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) {
 			{
 				Type:   ExporterOCI,
 				Attrs:  map[string]string{},
-				Output: nopWriteCloser{ioutil.Discard},
+				Output: fixedWriteCloser(nopWriteCloser{ioutil.Discard}),
 			},
 		},
 	}, "", frontend, nil)
@@ -1400,6 +1401,36 @@ func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) {
 	checkAllReleasable(t, c, sb, true)
 }
 
+func testExporterTargetExists(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Image("busybox:latest")
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	var mdDgst string
+	res, err := c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:  ExporterOCI,
+				Attrs: map[string]string{},
+				Output: func(m map[string]string) (io.WriteCloser, error) {
+					mdDgst = m["containerimage.digest"]
+					return nil, nil
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+	dgst := res.ExporterResponse["containerimage.digest"]
+
+	require.True(t, strings.HasPrefix(dgst, "sha256:"))
+	require.Equal(t, dgst, mdDgst)
+}
+
 func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(context.TODO(), sb.Address())
@@ -2060,7 +2091,7 @@ func testDuplicateWhiteouts(t *testing.T, sb integration.Sandbox) {
 		Exports: []ExportEntry{
 			{
 				Type:   ExporterOCI,
-				Output: outW,
+				Output: fixedWriteCloser(outW),
 			},
 		},
 	}, nil)
@@ -2130,7 +2161,7 @@ func testWhiteoutParentDir(t *testing.T, sb integration.Sandbox) {
 		Exports: []ExportEntry{
 			{
 				Type:   ExporterOCI,
-				Output: outW,
+				Output: fixedWriteCloser(outW),
 			},
 		},
 	}, nil)
@@ -2465,7 +2496,7 @@ func testInvalidExporter(t *testing.T, sb integration.Sandbox) {
 			{
 				Type:   ExporterLocal,
 				Attrs:  attrs,
-				Output: f,
+				Output: fixedWriteCloser(f),
 			},
 		},
 	}, nil)
@@ -2611,3 +2642,9 @@ func (*netModeDefault) UpdateConfigFile(in string) string {
 
 var hostNetwork integration.ConfigUpdater = &netModeHost{}
 var defaultNetwork integration.ConfigUpdater = &netModeDefault{}
+
+func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
+	return func(map[string]string) (io.WriteCloser, error) {
+		return wc, nil
+	}
+}

client/solve.go

@@ -46,8 +46,8 @@ type SolveOpt struct {
 type ExportEntry struct {
 	Type      string
 	Attrs     map[string]string
-	Output    io.WriteCloser // for ExporterOCI and ExporterDocker
-	OutputDir string         // for ExporterLocal
+	Output    func(map[string]string) (io.WriteCloser, error) // for ExporterOCI and ExporterDocker
+	OutputDir string                                          // for ExporterLocal
 }
 
 type CacheOptionsEntry struct {

cmd/buildctl/build/output.go

@@ -88,7 +88,12 @@ func ParseLegacyExporter(legacyExporter string, legacyExporterOpts []string) ([]
 }
 
 // resolveExporterDest returns at most either one of io.WriteCloser (single file) or a string (directory path).
-func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error) {
+func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.WriteCloser, error), string, error) {
+	wrapWriter := func(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
+		return func(m map[string]string) (io.WriteCloser, error) {
+			return wc, nil
+		}
+	}
 	switch exporter {
 	case client.ExporterLocal:
 		if dest == "" {
@@ -105,13 +110,13 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error)
 				return nil, "", errors.Errorf("destination file is a directory")
 			}
 			w, err := os.Create(dest)
-			return w, "", err
+			return wrapWriter(w), "", err
 		}
 		// if no output file is specified, use stdout
 		if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
 			return nil, "", errors.Errorf("output file is required for %s exporter. refusing to write to console", exporter)
 		}
-		return os.Stdout, "", nil
+		return wrapWriter(os.Stdout), "", nil
 	default: // e.g. client.ExporterImage
 		if dest != "" {
 			return nil, "", errors.Errorf("output %s is not supported by %s exporter", dest, exporter)

examples/build-using-dockerfile/main.go

@@ -166,7 +166,9 @@ func newSolveOpt(clicontext *cli.Context, w io.WriteCloser) (*client.SolveOpt, e
 				Attrs: map[string]string{
 					"name": clicontext.String("tag"),
 				},
-				Output: w,
+				Output: func(_ map[string]string) (io.WriteCloser, error) {
+					return w, nil
+				},
 			},
 		},
 		LocalDirs:     localDirs,

exporter/oci/export.go

@@ -19,6 +19,8 @@ import (
 	"github.com/moby/buildkit/util/progress"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 type ExporterVariant string
@@ -135,6 +137,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
 	desc.Annotations[ocispec.AnnotationCreated] = time.Now().UTC().Format(time.RFC3339)
 
 	resp := make(map[string]string)
+	resp["containerimage.digest"] = desc.Digest.String()
 
 	if n, ok := src.Metadata["image.name"]; e.name == "*" && ok {
 		e.name = string(n)
@@ -154,16 +157,23 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
 		return nil, err
 	}
 
-	w, err := filesync.CopyFileWriter(ctx, e.caller)
+	w, err := filesync.CopyFileWriter(ctx, resp, e.caller)
 	if err != nil {
 		return nil, err
 	}
 	report := oneOffProgress(ctx, "sending tarball")
 	if err := exp.Export(ctx, e.opt.ImageWriter.ContentStore(), *desc, w); err != nil {
 		w.Close()
+		if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists {
+			return resp, report(nil)
+		}
 		return nil, report(err)
 	}
-	return resp, report(w.Close())
+	err = w.Close()
+	if st, ok := status.FromError(errors.Cause(err)); ok && st.Code() == codes.AlreadyExists {
+		return resp, report(nil)
+	}
+	return resp, report(err)
 }
 
 func oneOffProgress(ctx context.Context, id string) func(err error) error {

exporter/tar/export.go

@@ -147,7 +147,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
 		fs = d.FS
 	}
 
-	w, err := filesync.CopyFileWriter(ctx, e.caller)
+	w, err := filesync.CopyFileWriter(ctx, nil, e.caller)
 	if err != nil {
 		return nil, err
 	}

frontend/dockerfile/dockerfile_test.go

@@ -385,7 +385,7 @@ FROM stage-$TARGETOS
 		Exports: []client.ExportEntry{
 			{
 				Type:   client.ExporterTar,
-				Output: &nopWriteCloser{buf},
+				Output: fixedWriteCloser(&nopWriteCloser{buf}),
 			},
 		},
 		LocalDirs: map[string]string{
@@ -408,7 +408,7 @@ FROM stage-$TARGETOS
 		Exports: []client.ExportEntry{
 			{
 				Type:   client.ExporterTar,
-				Output: &nopWriteCloser{buf},
+				Output: fixedWriteCloser(&nopWriteCloser{buf}),
 			},
 		},
 		FrontendAttrs: map[string]string{
@@ -1129,7 +1129,7 @@ COPY arch-$TARGETARCH whoami
 		Exports: []client.ExportEntry{
 			{
 				Type:   client.ExporterOCI,
-				Output: outW,
+				Output: fixedWriteCloser(outW),
 			},
 		},
 	}, nil)
@@ -4244,3 +4244,9 @@ func (*secModeInsecure) UpdateConfigFile(in string) string {
 
 var securitySandbox integration.ConfigUpdater = &secModeSandbox{}
 var securityInsecure integration.ConfigUpdater = &secModeInsecure{}
+
+func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
+	return func(map[string]string) (io.WriteCloser, error) {
+		return wc, nil
+	}
+}

session/filesync/diffcopy.go

@@ -40,6 +40,12 @@ type streamWriterCloser struct {
 
 func (wc *streamWriterCloser) Write(dt []byte) (int, error) {
 	if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil {
+		// SendMsg return EOF on remote errors
+		if errors.Cause(err) == io.EOF {
+			if err := errors.WithStack(wc.ClientStream.RecvMsg(struct{}{})); err != nil {
+				return 0, err
+			}
+		}
 		return 0, errors.WithStack(err)
 	}
 	return len(dt), nil

session/filesync/filesync.go

@@ -18,11 +18,12 @@ import (
 )
 
 const (
-	keyOverrideExcludes = "override-excludes"
-	keyIncludePatterns  = "include-patterns"
-	keyExcludePatterns  = "exclude-patterns"
-	keyFollowPaths      = "followpaths"
-	keyDirName          = "dir-name"
+	keyOverrideExcludes   = "override-excludes"
+	keyIncludePatterns    = "include-patterns"
+	keyExcludePatterns    = "exclude-patterns"
+	keyFollowPaths        = "followpaths"
+	keyDirName            = "dir-name"
+	keyExporterMetaPrefix = "exporter-md-"
 )
 
 type fsSyncProvider struct {
@@ -238,16 +239,16 @@ func NewFSSyncTargetDir(outdir string) session.Attachable {
 }
 
 // NewFSSyncTarget allows writing into an io.WriteCloser
-func NewFSSyncTarget(w io.WriteCloser) session.Attachable {
+func NewFSSyncTarget(f func(map[string]string) (io.WriteCloser, error)) session.Attachable {
 	p := &fsSyncTarget{
-		outfile: w,
+		f: f,
 	}
 	return p
 }
 
 type fsSyncTarget struct {
-	outdir  string
-	outfile io.WriteCloser
+	outdir string
+	f      func(map[string]string) (io.WriteCloser, error)
 }
 
 func (sp *fsSyncTarget) Register(server *grpc.Server) {
@@ -258,11 +259,26 @@ func (sp *fsSyncTarget) DiffCopy(stream FileSend_DiffCopyServer) error {
 	if sp.outdir != "" {
 		return syncTargetDiffCopy(stream, sp.outdir)
 	}
-	if sp.outfile == nil {
+
+	if sp.f == nil {
 		return errors.New("empty outfile and outdir")
 	}
-	defer sp.outfile.Close()
-	return writeTargetFile(stream, sp.outfile)
+	opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object
+	md := map[string]string{}
+	for k, v := range opts {
+		if strings.HasPrefix(k, keyExporterMetaPrefix) {
+			md[strings.TrimPrefix(k, keyExporterMetaPrefix)] = strings.Join(v, ",")
+		}
+	}
+	wc, err := sp.f(md)
+	if err != nil {
+		return err
+	}
+	if wc == nil {
+		return status.Errorf(codes.AlreadyExists, "target already exists")
+	}
+	defer wc.Close()
+	return writeTargetFile(stream, wc)
 }
 
 func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress func(int, bool)) error {
@@ -281,14 +297,21 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, c session.Caller, progress
 	return sendDiffCopy(cc, fs, progress)
 }
 
-func CopyFileWriter(ctx context.Context, c session.Caller) (io.WriteCloser, error) {
+func CopyFileWriter(ctx context.Context, md map[string]string, c session.Caller) (io.WriteCloser, error) {
 	method := session.MethodURL(_FileSend_serviceDesc.ServiceName, "diffcopy")
 	if !c.Supports(method) {
 		return nil, errors.Errorf("method %s not supported by the client", method)
 	}
 
 	client := NewFileSendClient(c.Conn())
 
+	opts := make(map[string][]string, len(md))
+	for k, v := range md {
+		opts[keyExporterMetaPrefix+k] = []string{v}
+	}
+
+	ctx = metadata.NewOutgoingContext(ctx, opts)
+
 	cc, err := client.DiffCopy(ctx)
 	if err != nil {
 		return nil, errors.WithStack(err)

vendor/modules.txt

@@ -175,8 +175,8 @@ github.com/gogo/protobuf/protoc-gen-gogo/descriptor
 # github.com/golang/protobuf v1.2.0
 github.com/golang/protobuf/ptypes/timestamp
 github.com/golang/protobuf/proto
-github.com/golang/protobuf/ptypes/any
 github.com/golang/protobuf/ptypes
+github.com/golang/protobuf/ptypes/any
 github.com/golang/protobuf/ptypes/duration
 # github.com/google/go-cmp v0.2.0
 github.com/google/go-cmp/cmp
@@ -296,10 +296,10 @@ google.golang.org/genproto/googleapis/rpc/status
 google.golang.org/grpc
 google.golang.org/grpc/credentials
 google.golang.org/grpc/metadata
+google.golang.org/grpc/codes
+google.golang.org/grpc/status
 google.golang.org/grpc/health
 google.golang.org/grpc/health/grpc_health_v1
-google.golang.org/grpc/status
-google.golang.org/grpc/codes
 google.golang.org/grpc/balancer
 google.golang.org/grpc/balancer/roundrobin
 google.golang.org/grpc/connectivity