Skip to content

Commit

Permalink
Merge pull request #47 from wunderio/feature/reuse-tags
Browse files Browse the repository at this point in the history 
Feature/reuse tags
  • Loading branch information
@Jancis
Jancis authored Aug 10, 2023
2 parents 0034360 + 790dde3 commit 0150e83
Show file tree
Hide file tree
Showing 7 changed files with 320 additions and 304 deletions.
130 changes: 93 additions & 37 deletions cmd/ciImageBuild.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"encoding/json"
"fmt"
"log"
"os"
Expand All @@ -10,6 +11,9 @@ import (

"github.com/spf13/cobra"
"github.com/wunderio/silta-cli/internal/common"

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/v1/remote"
)

// buildCmd represents the build command
Expand Down Expand Up @@ -93,65 +97,117 @@ var ciImageBuildCmd = &cobra.Command{
branchName = strings.ToLower(branchName)
reg, _ := regexp.Compile("[^[:alnum:]]")
branchName = reg.ReplaceAllString(branchName, "-")
extraImageTag = fmt.Sprintf("--tag '%s:%s'", imageUrl, branchName)
extraImageTag = fmt.Sprintf("branch--%s", branchName)
}

// TODO: reuseExisting

// Reuse existing image if it exists
if !debug {
if reuseExisting {
if imageRepoHost == "gcr.io" || strings.HasSuffix(imageRepoHost, ".gcr.io") || strings.HasSuffix(imageRepoHost, ".pkg.dev") {
_, useGCloud := os.LookupEnv("SILTA_USE_GCLOUD")
if useGCloud {
command := fmt.Sprintf("gcloud container images list-tags '%s' --filter='tags:%s' --format=json | grep -q '\"%s\"';", imageUrl, imageTag, imageTag)
err := exec.Command("bash", "-c", command).Run()

if err == nil {
fmt.Printf("Image %s:%s already exists, existing image will be used.", imageUrl, imageTag)
return
}
_, useGCloud := os.LookupEnv("SILTA_USE_GCLOUD")

} else {
if useGCloud && (imageRepoHost == "gcr.io" || strings.HasSuffix(imageRepoHost, ".gcr.io") || strings.HasSuffix(imageRepoHost, ".pkg.dev")) {

gcpToken := common.GetGCPOAuth2Token()
repositoryJWT := common.GetGCPJWT(gcpToken, imageRepoHost, common.Image, imageRepoProject, imageIdentifier)
tags := common.GCPListTags(repositoryJWT, namespace+"-"+imageIdentifier, imageRepoHost, imageRepoProject)
if common.HasString(tags, imageTag) {
fmt.Printf("Image %s:%s already exists, existing image will be used.", imageUrl, imageTag)
return
// Get image tags via gcloud
command := fmt.Sprintf("gcloud container images list-tags '%s' --filter='tags:%s' --format=json", imageUrl, imageTag)
output, err := exec.Command("bash", "-c", command).CombinedOutput()
if err != nil {
log.Fatal("Error (gcloud list-tags): ", err)
}
// Unmarshal or Decode the JSON to the interface.
type TagList struct {
Digest string `json:"digest"`
Tags []string `json:"tags"`
}
var taglist []TagList
err = json.Unmarshal([]byte(output), &taglist)
if err != nil {
log.Fatal("Error (json unmarshal): ", err)
}
var tagExists bool = false
var extraTagExists bool = false
for _, tag := range taglist {
for _, t := range tag.Tags {
if t == imageTag {
tagExists = true
}
if len(extraImageTag) > 0 && t == extraImageTag {
extraTagExists = true
}
}
}
} else if strings.HasSuffix(imageRepoHost, ".amazonaws.com") {
command := fmt.Sprintf("aws ecr describe-images --repository-name='%s' --image-ids='imageTag=%s' 2>&1 > /dev/null", imageUrl, imageTag)
err := exec.Command("bash", "-c", command).Run()
if err == nil {
fmt.Printf("Image %s:%s already exists, existing image will be used.", imageUrl, imageTag)
// If tag exists in taglist, return and don't rebuild.
if tagExists {
fmt.Printf("Image %s:%s already exists, existing image will be used.\n", imageUrl, imageTag)
// Add extra tag if it does not exist yet
if len(extraImageTag) > 0 && !extraTagExists {
fmt.Printf("Image %s:%s already exists, but extra tag %s:%s does not exist yet, it will be added.\n", imageUrl, imageTag, imageUrl, extraImageTag)
command := fmt.Sprintf("gcloud container images add-tag '%s:%s' '%s:%s'", imageUrl, imageTag, imageUrl, extraImageTag)
err = exec.Command("bash", "-c", command).Run()
if err != nil {
log.Fatal("Error (gcloud add-tag): ", err)
}
}
return
}
} else if strings.HasSuffix(imageRepoHost, ".azurecr.io") {

imageUrl := fmt.Sprintf("%s/%s/%s-%s", imageRepoHost, imageRepoProject, namespace, imageIdentifier)

command := fmt.Sprintf("docker manifest inspect '%s/%s/%s-%s:%s' > /dev/null 2>&1", imageRepoHost, imageRepoProject, namespace, imageIdentifier, imageTag)
err := exec.Command("bash", "-c", command).Run()
if err == nil {
fmt.Printf("Image %s:%s already exists, existing image will be used.", imageUrl, imageTag)
} else {
// Generic docker registry, e.g. docker.io
// Supports ACR, AR, GCR and ECR

// Reuse docker cli credentials
authenticator := remote.WithAuthFromKeychain(authn.DefaultKeychain)

imageTag_digest := common.GetImageTagDigest(authenticator, imageUrl, imageTag)

if imageTag_digest != "" {
fmt.Printf("Image %s:%s already exists, existing image will be used.\n", imageUrl, imageTag)

// Add extra tag (branch name) if it does not exist yet
if len(extraImageTag) > 0 {

extraImageTag_digest := common.GetImageTagDigest(authenticator, imageUrl, extraImageTag)

imageTag_digest := common.GetImageTagDigest(authenticator, imageUrl, imageTag)
if extraImageTag_digest == "" || extraImageTag_digest != imageTag_digest {
// Have to pull images, manifest creation is unreliable due to digest differences
// https://github.com/docker/hub-feedback/issues/1925
fmt.Printf("Image tag %s:%s already exists, but extra tag %s:%s does not exist yet, it will be added.\n", imageUrl, imageTag, imageUrl, extraImageTag)
// Pull image, tag it and push it
err := exec.Command("bash", "-c", fmt.Sprintf("docker pull '%s:%s'", imageUrl, imageTag)).Run()
if err != nil {
log.Fatal("Error (docker pull): ", err)
}
err = exec.Command("bash", "-c", fmt.Sprintf("docker tag '%s:%s' '%s:%s'", imageUrl, imageTag, imageUrl, extraImageTag)).Run()
if err != nil {
log.Fatal("Error (docker tag): ", err)
}
err = exec.Command("bash", "-c", fmt.Sprintf("docker push '%s:%s'", imageUrl, extraImageTag)).Run()
if err != nil {
log.Fatal("Error (docker push): ", err)
}
}
}
return
}
}
}
}

// Run docker build
command := fmt.Sprintf("docker build --tag '%s:%s' %s -f '%s' %s", imageUrl, imageTag, extraImageTag, dockerfile, buildPath)
extraImageTagString := ""
if len(extraImageTag) > 0 {
extraImageTagString = fmt.Sprintf("--tag '%s:%s'", imageUrl, extraImageTag)
}
command := fmt.Sprintf("docker build --tag '%s:%s' %s -f '%s' %s", imageUrl, imageTag, extraImageTagString, dockerfile, buildPath)
pipedExec(command, debug)

// Create AWS/ECR repository (ECR requires a dedicated repository per project)
if strings.HasSuffix(imageRepoHost, ".amazonaws.com") {
command = fmt.Sprintf("aws ecr describe-repositories --repository-name '%s'", imageUrl)

command = fmt.Sprintf("aws ecr describe-repositories --repository-name '%s/%s-%s'", imageRepoProject, namespace, imageIdentifier)
err := exec.Command("bash", "-c", command).Run()
if err != nil {
command = fmt.Sprintf("aws ecr create-repository --repository-name '%s'", imageUrl)

command = fmt.Sprintf("aws ecr create-repository --repository-name '%s/%s-%s'", imageRepoProject, namespace, imageIdentifier)
err = exec.Command("bash", "-c", command).Run()
if err != nil {
log.Fatal("Error (aws ecr create-repository): ", err)
Expand All @@ -165,7 +221,7 @@ var ciImageBuildCmd = &cobra.Command{

// Push extra tags
if len(branchName) > 0 {
command = fmt.Sprintf("docker push '%s:%s'", imageUrl, branchName)
command = fmt.Sprintf("docker push '%s:%s'", imageUrl, extraImageTag)
pipedExec(command, debug)
}
},
Expand Down
2 changes: 2 additions & 0 deletions cmd/ciImageLogin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ Available flags and environment variables:
} else if awsSecretAccessKey != "" {
// ECR login
command = fmt.Sprintf("aws ecr get-login --no-include-email | bash")
// TODO: use aws cli v2
// command = fmt.Sprintf("echo %q | docker login --username AWS --password-stdin %s", awsSecretAccessKey, imageRepoHost)

} else if aksSPPass != "" {
// ACR Login
Expand Down
1 change: 0 additions & 1 deletion cmd/ciToolsCheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ var ciToolsCheckCmd = &cobra.Command{
bins := make(map[string]string)
bins["helm"] = "version"
bins["kubectl"] = "version"
bins["oauth2l"] = ""

for bin, cmd := range bins {
_, err := exec.LookPath(bin)
Expand Down
105 changes: 56 additions & 49 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,141 +4,148 @@ go 1.19

require (
github.com/Luzifer/go-openssl/v4 v4.1.0
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.7.0
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/google/go-containerregistry v0.15.2
github.com/mittwald/go-helm-client v0.11.3
k8s.io/apimachinery v0.24.3
k8s.io/client-go v0.24.3
k8s.io/apimachinery v0.26.2
k8s.io/client-go v0.26.2
)

require (
cloud.google.com/go v0.99.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/BurntSushi/toml v1.0.0 // indirect
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.2 // indirect
github.com/Masterminds/squirrel v1.5.3 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/containerd/containerd v1.6.6 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/containerd/containerd v1.7.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.17+incompatible // indirect
github.com/docker/cli v23.0.5+incompatible // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.17+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/docker v23.0.5+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/emicklei/go-restful/v3 v3.8.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-gorp/gorp/v3 v3.0.2 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.6 // indirect
github.com/lib/pq v1.10.7 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rubenv/sql-migrate v1.1.1 // indirect
github.com/russross/blackfriday v1.5.2 // indirect
github.com/russross/blackfriday v1.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.1 // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
helm.sh/helm/v3 v3.9.4 // indirect
k8s.io/api v0.24.3 // indirect
k8s.io/api v0.26.2 // indirect
k8s.io/apiextensions-apiserver v0.24.2 // indirect
k8s.io/apiserver v0.24.2 // indirect
k8s.io/apiserver v0.26.2 // indirect
k8s.io/cli-runtime v0.24.3 // indirect
k8s.io/component-base v0.24.2 // indirect
k8s.io/klog/v2 v2.60.1 // indirect
k8s.io/kube-openapi v0.0.0-20220627174259-011e075b9cb8 // indirect
k8s.io/component-base v0.26.2 // indirect
k8s.io/klog/v2 v2.90.1 // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/kubectl v0.24.2 // indirect
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
oras.land/oras-go v1.2.0 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/kustomize/api v0.11.4 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace oras.land/oras-go => oras.land/oras-go v1.2.3
Loading

0 comments on commit 0150e83

Please sign in to comment.