ci: exclude .git dir in ci build (dagger#7234)

* ci: exclude .git dir in ci build Previously, we were using this to generate a work-tree hash, so that new builds would have a different version tag, which helped to solve some weird caching issues with nested execs. However! A problem with this was that we needed to upload *the entire* git directory every time we built - which is an expensive operation, my .git directory for dagger/dagger is 147MB. Instead of relying on git for this, we can use the ID of the input directory! Because of our use of blob sources, a different ID represents a different directory. Additionally, we prefix these new builds with a special "dev-" string to specifically indicate that these are dev builds, with no specific version passed in - these aren't commit hashes, and cannot be treated as such. Signed-off-by: Justin Chadwell <[email protected]> * ci: remove dependency on go sdk The only thing this was being used for here was the connect job - we can just use `--help` to get this info to sanity check it. Signed-off-by: Justin Chadwell <[email protected]> * ci: run linting over ci itself Signed-off-by: Justin Chadwell <[email protected]> --------- Signed-off-by: Justin Chadwell <[email protected]>
vikram-dagger · May 3, 2024 · ea5be7f · ea5be7f
1 parent 1289c74
commit ea5be7f
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 85 deletions.
diff --git a/ci/engine.go b/ci/engine.go
@@ -124,19 +124,19 @@ func (e *Engine) Lint(
  // +optional
  all bool,
 ) error {
- eg, ctx := errgroup.WithContext(ctx)
+ src := e.Dagger.Source.AsModule().GeneratedContextDirectory()
 
- pkgs := []string{""}
- // pkgs := []string{"", "ci"}
+ eg, ctx := errgroup.WithContext(ctx)
 
+ pkgs := []string{"", "ci"}
  cmd := []string{"golangci-lint", "run", "-v", "--timeout", "5m"}
  if all {
  cmd = append(cmd, "--max-issues-per-linter=0", "--max-same-issues=0")
  }
  for _, pkg := range pkgs {
  golangci := dag.Container().
  From(consts.GolangLintImage).
- WithMountedDirectory("/app", util.GoDirectory(e.Dagger.Source)).
+ WithMountedDirectory("/app", util.GoDirectory(src)).
  WithWorkdir(path.Join("/app", pkg)).
  WithExec(cmd)
  eg.Go(func() error {
@@ -145,8 +145,8 @@ func (e *Engine) Lint(
  })
 
  eg.Go(func() error {
- return util.DiffDirectoryF(ctx, util.GoDirectory(e.Dagger.Source), func(ctx context.Context) (*dagger.Directory, error) {
- return util.GoBase(e.Dagger.Source).
+ return util.DiffDirectoryF(ctx, util.GoDirectory(src), func(ctx context.Context) (*dagger.Directory, error) {
+ return util.GoBase(src).
  WithExec([]string{"go", "mod", "tidy"}).
  Directory("."), nil
  }, "go.mod", "go.sum")

diff --git a/ci/go.mod b/ci/go.mod
@@ -2,17 +2,11 @@ module github.com/dagger/dagger/ci
 
 go 1.21.7
 
-replace github.com/dagger/dagger => ../
-
-require (
- dagger.io/dagger v0.11.1
- github.com/dagger/dagger v0.11.1
-)
-
 require (
  github.com/99designs/gqlgen v0.17.44
  github.com/Khan/genqlient v0.7.0
  github.com/containerd/containerd v1.7.15-0.20240329193453-0dcf21c1528a
+ github.com/dagger/dagger v0.11.1
  github.com/magefile/mage v1.15.0
  github.com/moby/buildkit v0.13.0-rc3.0.20240403135707-dc23e43dc15c
  github.com/opencontainers/image-spec v1.1.0
@@ -28,31 +22,28 @@ require (
 )
 
 require (
+ github.com/Microsoft/hcsshim v0.11.4 // indirect
  github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+ github.com/containerd/log v0.1.0 // indirect
  github.com/go-logr/logr v1.4.1 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
- github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
- go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
- go.opentelemetry.io/otel/metric v1.24.0 // indirect
- go.opentelemetry.io/proto/otlp v1.1.0 // indirect
- golang.org/x/net v0.24.0 // indirect
- golang.org/x/text v0.14.0 // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
-)
-
-require (
- github.com/Microsoft/hcsshim v0.11.4 // indirect
- github.com/adrg/xdg v0.4.0 // indirect
- github.com/containerd/log v0.1.0 // indirect
  github.com/golang/protobuf v1.5.4 // indirect
  github.com/google/uuid v1.6.0 // indirect
- github.com/mitchellh/go-homedir v1.1.0 // indirect
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/sirupsen/logrus v1.9.3 // indirect
  github.com/sosodev/duration v1.2.0 // indirect
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+ go.opentelemetry.io/otel/metric v1.24.0 // indirect
+ go.opentelemetry.io/proto/otlp v1.1.0 // indirect
+ golang.org/x/net v0.24.0 // indirect
  golang.org/x/sys v0.19.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
  google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect
  google.golang.org/grpc v1.62.1
  google.golang.org/protobuf v1.33.0 // indirect
 )
+
+replace github.com/dagger/dagger => ../
diff --git a/ci/go.sum b/ci/go.sum
@@ -1,13 +1,9 @@
-dagger.io/dagger v0.11.1 h1:LXKVvGPIq4Uii67jJScmaiOOFaG91F+4GOxCMc3I/GE=
-dagger.io/dagger v0.11.1/go.mod h1:ABrEbaXuGQtqOlc0WlHWHQt/azY0jEs/O/X8xkX8xxM=
 github.com/99designs/gqlgen v0.17.44 h1:OS2wLk/67Y+vXM75XHbwRnNYJcbuJd4OBL76RX3NQQA=
 github.com/99designs/gqlgen v0.17.44/go.mod h1:UTCu3xpK2mLI5qcMNw+HKDiEL77it/1XtAjisC4sLwM=
 github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
 github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
-github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
-github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -34,8 +30,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
 github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/moby/buildkit v0.13.0-rc3.0.20240403135707-dc23e43dc15c h1:79uQuWhJQjc9Xl1ZwogIkS/BoKNF6GWps5sKTSOXXrs=
 github.com/moby/buildkit v0.13.0-rc3.0.20240403135707-dc23e43dc15c/go.mod h1:k97+3l2jpDDC8JxSHywmN1lNFGgEYXk7WaK6LFrg9p0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -84,7 +78,6 @@ golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

diff --git a/ci/mage/engine.go b/ci/mage/engine.go
@@ -8,7 +8,6 @@ import (
  "path/filepath"
  "strings"
 
- "dagger.io/dagger"
  "github.com/containerd/containerd/platforms"
  "github.com/magefile/mage/mg"
  "golang.org/x/mod/semver"
@@ -27,11 +26,7 @@ var (
 
 // Connect tests a connection to a Dagger Engine
 func (t Engine) Connect(ctx context.Context) error {
- c, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
- if err != nil {
- return err
- }
- return c.Close()
+ return util.DaggerCall(ctx, "--help")
 }
 
 // Lint lints the engine
@@ -145,7 +140,7 @@ func (t Engine) Dev(ctx context.Context) error {
  }
  tarPath := "./bin/engine.tar"
  args = append(args, "container", "export", "--path="+tarPath)
- args = append(args, "--forced-compression="+string(dagger.Gzip)) // use gzip to avoid incompatibility w/ older docker versions
+ args = append(args, "--forced-compression=Gzip") // use gzip to avoid incompatibility w/ older docker versions
  err := util.DaggerCall(ctx, args...)
  if err != nil {
  return err

diff --git a/ci/main.go b/ci/main.go
@@ -7,7 +7,6 @@ import (
 
  "github.com/dagger/dagger/ci/internal/dagger"
  "github.com/dagger/dagger/ci/util"
- "golang.org/x/mod/semver"
 )
 
 // A dev environment for the Dagger Engine
@@ -29,18 +28,9 @@ func New(
  // +optional
  hostDockerConfig *Secret,
 ) (*Dagger, error) {
- var versionInfo *VersionInfo
- switch {
- case version == "":
- var err error
- versionInfo, err = newVersionFromGit(ctx, source.Directory(".git"))
- if err != nil {
- return nil, err
- }
- case semver.IsValid(version):
- versionInfo = &VersionInfo{Tag: version}
- default:
- versionInfo = &VersionInfo{Commit: version}
+ versionInfo, err := newVersion(ctx, source, version)
+ if err != nil {
+ return nil, err
  }
 
  return &Dagger{

diff --git a/ci/sdk.go b/ci/sdk.go
@@ -10,20 +10,20 @@ import (
 // A dev environment for the official Dagger SDKs
 type SDK struct {
  // Develop the Dagger Go SDK
- Go  *GoSDK
+ Go *GoSDK
  // Develop the Dagger Python SDK
- Python  *PythonSDK
+ Python *PythonSDK
  // Develop the Dagger Typescript SDK
  Typescript *TypescriptSDK
 
  // Develop the Dagger Elixir SDK (experimental)
  Elixir *ElixirSDK
  // Develop the Dagger Rust SDK (experimental)
- Rust  *RustSDK
+ Rust *RustSDK
  // Develop the Dagger Java SDK (experimental)
- Java  *JavaSDK
+ Java *JavaSDK
  // Develop the Dagger PHP SDK (experimental)
- PHP  *PHPSDK
+ PHP *PHPSDK
 }
 
 func (ci *Dagger) installer(ctx context.Context, name string) (func(*Container) *Container, error) {

diff --git a/ci/version.go b/ci/version.go
@@ -2,40 +2,42 @@ package main
 
 import (
  "context"
+ "crypto/sha1" //nolint:gosec
+ "encoding/hex"
  "fmt"
- "strings"
+ "regexp"
 
- "github.com/dagger/dagger/ci/consts"
  "github.com/dagger/dagger/ci/internal/dagger"
+ "golang.org/x/mod/semver"
 )
 
 type VersionInfo struct {
- Tag  string
- Commit  string
- TreeHash string
+ Tag string
+ Commit string
+ Dev  string
 }
 
-func newVersionFromGit(ctx context.Context, dir *dagger.Directory) (*VersionInfo, error) {
- base := dag.Container().
- From(consts.AlpineImage).
- WithExec([]string{"apk", "add", "git"}).
- WithMountedDirectory("/app/.git", dir).
- WithWorkdir("/app")
-
- info := &VersionInfo{}
-
- // use git write-tree to get a content hash of the current state of the repo
- var err error
- info.TreeHash, err = base.
- WithExec([]string{"git", "add", "."}).
- WithExec([]string{"git", "write-tree"}).
- Stdout(ctx)
- if err != nil {
- return nil, fmt.Errorf("get tree hash: %w", err)
+var commitRegexp = regexp.MustCompile("^[0-9a-f]{40}$")
+
+func newVersion(ctx context.Context, dir *dagger.Directory, version string) (*VersionInfo, error) {
+ switch {
+ case version == "":
+ id, err := dir.ID(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ h := sha1.New() //nolint:gosec
+ h.Write([]byte(id))
+ dgst := hex.EncodeToString(h.Sum(nil))
+ return &VersionInfo{Dev: dgst}, nil
+ case semver.IsValid(version):
+ return &VersionInfo{Tag: version}, nil
+ case commitRegexp.MatchString(version):
+ return &VersionInfo{Commit: version}, nil
+ default:
+ return nil, fmt.Errorf("could not parse version info %q", version)
  }
- info.TreeHash = strings.TrimSpace(info.TreeHash)
-
- return info, nil
 }
 
 func (info *VersionInfo) String() string {
@@ -45,5 +47,8 @@ func (info *VersionInfo) String() string {
  if info.Commit != "" {
  return info.Commit
  }
- return info.TreeHash
+ if info.Dev != "" {
+ return "dev-" + info.Dev
+ }
+ return "dev-unknown"
 }
diff --git a/dagger.json b/dagger.json
@@ -8,6 +8,7 @@
  "name": "default",
  "patterns": [
  "!bin",
+ "!.git",
  "!**/node_modules",
  "!**/.venv",
  "!**/__pycache__"