Merge pull request #4874 from tonistiigi/v0.13.2-picks

[v0.13] cherry-picks for v0.13.2

client/client_test.go

@@ -165,6 +165,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testFileOpInputSwap,
 	testRelativeMountpoint,
 	testLocalSourceDiffer,
+	testLocalSourceWithHardlinksFilter,
 	testOCILayoutSource,
 	testOCILayoutPlatformSource,
 	testBuildExportZstd,
@@ -1964,6 +1965,57 @@ func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffT
 	}
 }
 
+// moby/buildkit#4831
+func testLocalSourceWithHardlinksFilter(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("bar", []byte("bar"), 0600),
+		fstest.Link("bar", "foo1"),
+		fstest.Link("bar", "foo2"),
+	)
+
+	st := llb.Local("mylocal", llb.FollowPaths([]string{"foo*"}))
+
+	def, err := st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir := t.TempDir()
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalMounts: map[string]fsutil.FS{
+			"mylocal": dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	_, err = os.ReadFile(filepath.Join(destDir, "bar"))
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
+
+	dt, err := os.ReadFile(filepath.Join(destDir, "foo1"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("bar"), dt)
+
+	st1, err := os.Stat(filepath.Join(destDir, "foo1"))
+	require.NoError(t, err)
+
+	st2, err := os.Stat(filepath.Join(destDir, "foo2"))
+	require.NoError(t, err)
+
+	require.True(t, os.SameFile(st1, st2))
+}
+
 func testOCILayoutSource(t *testing.T, sb integration.Sandbox) {
 	workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureOCILayout)
 	requiresLinux(t)

cmd/buildkitd/config/config.go

@@ -40,10 +40,11 @@ type LogConfig struct {
 }
 
 type GRPCConfig struct {
-	Address      []string `toml:"address"`
-	DebugAddress string   `toml:"debugAddress"`
-	UID          *int     `toml:"uid"`
-	GID          *int     `toml:"gid"`
+	Address            []string `toml:"address"`
+	DebugAddress       string   `toml:"debugAddress"`
+	UID                *int     `toml:"uid"`
+	GID                *int     `toml:"gid"`
+	SecurityDescriptor string   `toml:"securityDescriptor"`
 
 	TLS TLSConfig `toml:"tls"`
 	// MaxRecvMsgSize int    `toml:"max_recv_message_size"`

cmd/buildkitd/config/gcpolicy.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
+	"github.com/moby/buildkit/util/bklog"
 	"github.com/pkg/errors"
 )
 
@@ -104,3 +105,25 @@ func stripQuotes(s string) string {
 	}
 	return s
 }
+
+func DetectDefaultGCCap() DiskSpace {
+	return DiskSpace{Percentage: DiskSpacePercentage}
+}
+
+func (d DiskSpace) AsBytes(root string) int64 {
+	if d.Bytes != 0 {
+		return d.Bytes
+	}
+	if d.Percentage == 0 {
+		return 0
+	}
+
+	diskSize, err := getDiskSize(root)
+	if err != nil {
+		bklog.L.Warnf("failed to get disk size: %v", err)
+		return defaultCap
+	}
+	avail := diskSize * d.Percentage / 100
+	rounded := (avail/(1<<30) + 1) * 1e9 // round up
+	return rounded
+}

cmd/buildkitd/config/gcpolicy_unix.go

@@ -7,23 +7,13 @@ import (
 	"syscall"
 )
 
-func DetectDefaultGCCap() DiskSpace {
-	return DiskSpace{Percentage: 10}
-}
-
-func (d DiskSpace) AsBytes(root string) int64 {
-	if d.Bytes != 0 {
-		return d.Bytes
-	}
-	if d.Percentage == 0 {
-		return 0
-	}
+var DiskSpacePercentage int64 = 10
 
+func getDiskSize(root string) (int64, error) {
 	var st syscall.Statfs_t
 	if err := syscall.Statfs(root, &st); err != nil {
-		return defaultCap
+		return 0, err
 	}
 	diskSize := int64(st.Bsize) * int64(st.Blocks)
-	avail := diskSize * d.Percentage / 100
-	return (avail/(1<<30) + 1) * 1e9 // round up
+	return diskSize, nil
 }

cmd/buildkitd/config/gcpolicy_windows.go

@@ -3,10 +3,29 @@
 
 package config
 
-func DetectDefaultGCCap() DiskSpace {
-	return DiskSpace{Bytes: defaultCap}
-}
+import (
+	"golang.org/x/sys/windows"
+)
+
+// set as double that for Linux since
+// Windows images are generally larger.
+var DiskSpacePercentage int64 = 20
+
+func getDiskSize(root string) (int64, error) {
+	rootUTF16, err := windows.UTF16FromString(root)
+	if err != nil {
+		return 0, err
+	}
+	var freeAvailableBytes uint64
+	var totalBytes uint64
+	var totalFreeBytes uint64
 
-func (d DiskSpace) AsBytes(root string) int64 {
-	return d.Bytes
+	if err := windows.GetDiskFreeSpaceEx(
+		&rootUTF16[0],
+		&freeAvailableBytes,
+		&totalBytes,
+		&totalFreeBytes); err != nil {
+		return 0, err
+	}
+	return int64(totalBytes), nil
 }

cmd/buildkitd/main.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -397,9 +398,18 @@ func newGRPCListeners(cfg config.GRPCConfig) ([]net.Listener, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	sd := cfg.SecurityDescriptor
+	if sd == "" {
+		sd, err = groupToSecurityDescriptor("")
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	listeners := make([]net.Listener, 0, len(addrs))
 	for _, addr := range addrs {
-		l, err := getListener(addr, *cfg.UID, *cfg.GID, tlsConfig)
+		l, err := getListener(addr, *cfg.UID, *cfg.GID, sd, tlsConfig)
 		if err != nil {
 			for _, l := range listeners {
 				l.Close()
@@ -569,11 +579,19 @@ func applyMainFlags(c *cli.Context, cfg *config.Config) error {
 	}
 
 	if group := c.String("group"); group != "" {
-		gid, err := groupToGid(group)
-		if err != nil {
-			return err
+		if runtime.GOOS == "windows" {
+			secDescriptor, err := groupToSecurityDescriptor(group)
+			if err != nil {
+				return err
+			}
+			cfg.GRPC.SecurityDescriptor = secDescriptor
+		} else {
+			gid, err := groupToGid(group)
+			if err != nil {
+				return err
+			}
+			cfg.GRPC.GID = &gid
 		}
-		cfg.GRPC.GID = &gid
 	}
 
 	if tlscert := c.String("tlscert"); tlscert != "" {
@@ -628,7 +646,7 @@ func groupToGid(group string) (int, error) {
 	return id, nil
 }
 
-func getListener(addr string, uid, gid int, tlsConfig *tls.Config) (net.Listener, error) {
+func getListener(addr string, uid, gid int, secDescriptor string, tlsConfig *tls.Config) (net.Listener, error) {
 	addrSlice := strings.SplitN(addr, "://", 2)
 	if len(addrSlice) < 2 {
 		return nil, errors.Errorf("address %s does not contain proto, you meant unix://%s ?",
@@ -641,6 +659,9 @@ func getListener(addr string, uid, gid int, tlsConfig *tls.Config) (net.Listener
 		if tlsConfig != nil {
 			bklog.L.Warnf("TLS is disabled for %s", addr)
 		}
+		if proto == "npipe" {
+			return getLocalListener(listenAddr, secDescriptor)
+		}
 		return sys.GetLocalListener(listenAddr, uid, gid)
 	case "fd":
 		return listenFD(listenAddr, tlsConfig)
@@ -928,7 +949,7 @@ func parseBoolOrAuto(s string) (*bool, error) {
 func runTraceController(p string, exp sdktrace.SpanExporter) error {
 	server := grpc.NewServer()
 	tracev1.RegisterTraceServiceServer(server, &traceCollector{exporter: exp})
-	l, err := getLocalListener(p)
+	l, err := getLocalListener(p, "")
 	if err != nil {
 		return errors.Wrap(err, "creating trace controller listener")
 	}

cmd/buildkitd/main_unix.go

@@ -48,7 +48,7 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	return nil, errors.New("not supported yet")
 }
 
-func getLocalListener(listenerPath string) (net.Listener, error) {
+func getLocalListener(listenerPath, _ string) (net.Listener, error) {
 	uid := os.Getuid()
 	l, err := sys.GetLocalListener(listenerPath, uid, uid)
 	if err != nil {
@@ -60,3 +60,7 @@ func getLocalListener(listenerPath string) (net.Listener, error) {
 	}
 	return l, nil
 }
+
+func groupToSecurityDescriptor(_ string) (string, error) {
+	return "", nil
+}

cmd/buildkitd/main_windows.go

@@ -5,7 +5,9 @@ package main
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net"
+	"strings"
 
 	"github.com/Microsoft/go-winio"
 	_ "github.com/moby/buildkit/solver/llbsolver/ops"
@@ -19,14 +21,18 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
 	return nil, errors.New("listening server on fd not supported on windows")
 }
 
-func getLocalListener(listenerPath string) (net.Listener, error) {
-	pc := &winio.PipeConfig{
+func getLocalListener(listenerPath, secDescriptor string) (net.Listener, error) {
+	if secDescriptor == "" {
 		// Allow generic read and generic write access to authenticated users
 		// and system users. On Linux, this pipe seems to be given rw access to
 		// user, group and others (666).
 		// TODO(gabriel-samfira): should we restrict access to this pipe to just
 		// authenticated users? Or Administrators group?
-		SecurityDescriptor: "D:P(A;;GRGW;;;AU)(A;;GRGW;;;SY)",
+		secDescriptor = "D:P(A;;GRGW;;;AU)(A;;GRGW;;;SY)"
+	}
+
+	pc := &winio.PipeConfig{
+		SecurityDescriptor: secDescriptor,
 	}
 
 	listener, err := winio.ListenPipe(listenerPath, pc)
@@ -35,3 +41,17 @@ func getLocalListener(listenerPath string) (net.Listener, error) {
 	}
 	return listener, nil
 }
+
+func groupToSecurityDescriptor(group string) (string, error) {
+	sddl := "D:P(A;;GA;;;BA)(A;;GA;;;SY)"
+	if group != "" {
+		for _, g := range strings.Split(group, ",") {
+			sid, err := winio.LookupSidByName(g)
+			if err != nil {
+				return "", errors.Wrapf(err, "failed to lookup sid for group %s", g)
+			}
+			sddl += fmt.Sprintf("(A;;GRGW;;;%s)", sid)
+		}
+	}
+	return sddl, nil
+}

frontend/dockerfile/dockerfile2llb/convert.go

@@ -1340,11 +1340,10 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		copyOpts := []llb.ConstraintsOpt{
 			llb.Platform(*d.platform),
 		}
-		copy(copyOpts, fileOpt)
+		copyOpts = append(copyOpts, fileOpt...)
 		copyOpts = append(copyOpts, llb.ProgressGroup(pgID, pgName, true))
 
-		var mergeOpts []llb.ConstraintsOpt
-		copy(mergeOpts, fileOpt)
+		mergeOpts := append([]llb.ConstraintsOpt{}, fileOpt...)
 		d.cmdIndex--
 		mergeOpts = append(mergeOpts, llb.ProgressGroup(pgID, pgName, false), llb.WithCustomName(prefixCommand(d, "LINK "+name, d.prefixPlatform, &platform, env)))