Skip to content

Commit

Permalink
Merge pull request #79 from ivanilves/smarter-push
Browse files Browse the repository at this point in the history 
Push in a more intelligent manner
  • Loading branch information
@vonrabbe
vonrabbe authored Oct 19, 2017
2 parents 4c5b065 + c24ae50 commit d613f2c
Show file tree
Hide file tree
Showing 6 changed files with 283 additions and 95 deletions.
16 changes: 6 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,16 @@ You could use `lstags`, if you ...
* ... poll registry for the new images pushed (to take some action afterwards, run CI for example).
* ... compare local images with registry ones (e.g. know, if image tagged "latest" was re-pushed).

## How do I use it myself?
I run `lstags` inside a Cron Job on my Kubernetes worker nodes to poll my own Docker registry for a new [stable] images.
... pull Ubuntu 14.04 & 16.04, all the Alpine images and Debian "stretch" to have latest software to play with:
```
lstags --pull registry.ivanilves.local/tools/sicario~/v1\\.[0-9]+$/
lstags --pull ubuntu~/^1[46]\\.04$/ alpine debian~/stretch/
```
**NB!** In case you use private registry with authentication, make sure your Docker client knows how to authenticate against it!
`lstags` will reuse credentials saved by Docker client in its `config.json` file, one usually found at `~/.docker/config.json`

... and following cronjob runs on my CI server to ensure I always have latest Ubuntu 14.04 and 16.04 images to play with:
... pull and re-push CoreOS-related images from `quay.io` to your own registry (in case these hipsters will break everything):
```
lstags --pull ubuntu~/^1[46]\\.04$/
lstags --push-prefix=/quay --push-registry=registry.company.io quay.io/coreos/hyperkube quay.io/coreos/flannel
```
My CI server is connected over crappy Internet link and pulling images in advance makes `docker run` much faster. :wink:
**NB!** In case you use private registry with authentication, make sure your Docker client knows how to authenticate against it!
`lstags` will reuse credentials saved by Docker client in its `config.json` file, one usually found at `~/.docker/config.json`

## Image state
`lstags` distinguishes four states of Docker image:
Expand All @@ -55,7 +52,6 @@ There is also special `UNKNOWN` state, which means `lstags` failed to detect ima
You can either:
* rely on `lstags` discovering credentials "automagically" :tophat:
* load credentials from any Docker JSON config file specified
* pass username and password explicitly, via the command line

## Install: Binaries
https://github.com/ivanilves/lstags/releases
Expand Down
147 changes: 107 additions & 40 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"os"
"strings"

"github.com/jessevdk/go-flags"

Expand All @@ -21,8 +22,10 @@ import (
type Options struct {
DockerJSON string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"`
Pull bool `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
Push bool `short:"P" long:"push" description:"Push Docker images matched by filter to some registry (See 'push-registry')" env:"PUSH"`
PushRegistry string `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"`
PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
PushUpdate bool `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"`
ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
Expand Down Expand Up @@ -61,8 +64,12 @@ func parseFlags() (*Options, error) {
return nil, errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'")
}

if o.PushRegistry != "" {
o.Pull = true
if o.PushRegistry != "localhost:5000" && o.PushRegistry != "" {
o.Push = true
}

if o.Pull && o.Push {
return nil, errors.New("You either '--pull' or '--push', not both")
}

remote.TraceRequests = o.TraceRequests
Expand Down Expand Up @@ -92,17 +99,15 @@ func main() {
suicide(err, true)
}

const format = "%-12s %-45s %-15s %-25s %s\n"
fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")

repoCount := len(o.Positional.Repositories)
pullCount := 0
pushCount := 0

tcc := make(chan tag.Collection, repoCount)

pullCount := 0
pushCount := 0

for _, repoWithFilter := range o.Positional.Repositories {
go func(repoWithFilter string, concurrentRequests int, tcc chan tag.Collection) {
go func(repoWithFilter string, tcc chan tag.Collection) {
repository, filter, err := util.SeparateFilterAndRepo(repoWithFilter)
if err != nil {
suicide(err, true)
Expand All @@ -113,14 +118,16 @@ func main() {
repoPath := docker.GetRepoPath(repository, registry)
repoName := docker.GetRepoName(repository, registry)

fmt.Printf("ANALYZE %s\n", repoName)

username, password, _ := dockerConfig.GetCredentials(registry)

tr, err := auth.NewToken(registry, repoPath, username, password)
if err != nil {
suicide(err, true)
}

remoteTags, err := remote.FetchTags(registry, repoPath, tr.AuthHeader(), concurrentRequests)
remoteTags, err := remote.FetchTags(registry, repoPath, tr.AuthHeader(), o.ConcurrentRequests, filter)
if err != nil {
suicide(err, true)
}
Expand All @@ -133,6 +140,7 @@ func main() {
sortedKeys, names, joinedTags := tag.Join(remoteTags, localTags)

tags := make([]*tag.Tag, 0)
pullTags := make([]*tag.Tag, 0)
for _, key := range sortedKeys {
name := names[key]

Expand All @@ -143,34 +151,91 @@ func main() {
}

if tg.NeedsPull() {
pullTags = append(pullTags, tg)
pullCount++
}
pushCount++

tags = append(tags, tg)
}

var pushPrefix string
pushTags := make([]*tag.Tag, 0)
if o.Push {
tags = make([]*tag.Tag, 0)

pushPrefix = o.PushPrefix
if pushPrefix == "" {
pushPrefix = util.GeneratePathFromHostname(registry)
}

var pushRepoPath string
pushRepoPath = pushPrefix + "/" + repoPath
pushRepoPath = pushRepoPath[1:] // Leading "/" in prefix should be removed!

username, password, _ := dockerConfig.GetCredentials(o.PushRegistry)

tr, err := auth.NewToken(o.PushRegistry, pushRepoPath, username, password)
if err != nil {
suicide(err, true)
}

alreadyPushedTags, err := remote.FetchTags(o.PushRegistry, pushRepoPath, tr.AuthHeader(), o.ConcurrentRequests, filter)
if err != nil {
if !strings.Contains(err.Error(), "404 Not Found") {
suicide(err, true)
}

alreadyPushedTags = make(map[string]*tag.Tag)
}

sortedKeys, names, joinedTags := tag.Join(remoteTags, alreadyPushedTags)
for _, key := range sortedKeys {
name := names[key]

tg := joinedTags[name]

if !util.DoesMatch(tg.GetName(), filter) {
continue
}

if tg.NeedsPush(o.PushUpdate) {
pushTags = append(pushTags, tg)
pushCount++
}

tags = append(tags, tg)
}
}

tcc <- tag.Collection{
Registry: registry,
RepoName: repoName,
RepoPath: repoPath,
Tags: tags,
Registry: registry,
RepoName: repoName,
RepoPath: repoPath,
Tags: tags,
PullTags: pullTags,
PushTags: pushTags,
PushPrefix: pushPrefix,
}
}(repoWithFilter, o.ConcurrentRequests, tcc)
}(repoWithFilter, tcc)
}

tagCollections := make([]tag.Collection, repoCount)
repoNumber := 0
tagCollections := make([]tag.Collection, repoCount-1)

r := 0
for tc := range tcc {
tagCollections = append(tagCollections, tc)
fmt.Printf("FETCHED %s\n", tc.RepoName)

repoNumber++
tagCollections = append(tagCollections, tc)
r++

if repoNumber >= repoCount {
if r >= repoCount {
close(tcc)
}
}

const format = "%-12s %-45s %-15s %-25s %s\n"
fmt.Printf("-\n")
fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")
for _, tc := range tagCollections {
for _, tg := range tc.Tags {
fmt.Printf(
Expand All @@ -183,25 +248,23 @@ func main() {
)
}
}
fmt.Printf("-\n")

if o.Pull {
done := make(chan bool, pullCount)

for _, tc := range tagCollections {
go func(tc tag.Collection, done chan bool) {
for _, tg := range tc.Tags {
if tg.NeedsPull() {
ref := tc.RepoName + ":" + tg.GetName()

fmt.Printf("PULLING %s\n", ref)
err := dc.Pull(ref)
if err != nil {
suicide(err, false)
}
for _, tg := range tc.PullTags {
ref := tc.RepoName + ":" + tg.GetName()

done <- true
fmt.Printf("PULLING %s\n", ref)
err := dc.Pull(ref)
if err != nil {
suicide(err, false)
}

done <- true
}
}(tc, done)
}
Expand All @@ -218,21 +281,25 @@ func main() {
}
}

if o.Pull && o.PushRegistry != "" {
if o.Push {
done := make(chan bool, pushCount)

for _, tc := range tagCollections {
go func(tc tag.Collection, pushRegistry, pushPrefix string, done chan bool) {
for _, tg := range tc.Tags {
if pushPrefix == "" {
pushPrefix = util.GeneratePathFromHostname(tc.Registry)
}
go func(tc tag.Collection, done chan bool) {
for _, tg := range tc.PushTags {
var err error

srcRef := tc.RepoName + ":" + tg.GetName()
dstRef := pushRegistry + pushPrefix + "/" + tc.RepoPath + ":" + tg.GetName()
dstRef := o.PushRegistry + tc.PushPrefix + "/" + tc.RepoPath + ":" + tg.GetName()

fmt.Printf("[PULL/PUSH] PULLING %s\n", srcRef)
err = dc.Pull(srcRef)
if err != nil {
suicide(err, false)
}

fmt.Printf("PUSHING %s => %s\n", srcRef, dstRef)
err := dc.Tag(srcRef, dstRef)
fmt.Printf("[PULL/PUSH] PUSHING %s => %s\n", srcRef, dstRef)
err = dc.Tag(srcRef, dstRef)
if err != nil {
suicide(err, true)
}
Expand All @@ -243,7 +310,7 @@ func main() {

done <- true
}
}(tc, o.PushRegistry, o.PushPrefix, done)
}(tc, done)
}

p := 0
Expand Down
4 changes: 2 additions & 2 deletions main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func TestDockerHubWithPublicRepo(t *testing.T) {
t.Fatalf("Failed to get DockerHub public repo token: %s", err.Error())
}

tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128)
tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128, ".*")
if err != nil {
t.Fatalf("Failed to list DockerHub public repo (%s) tags: %s", repo, err.Error())
}
Expand Down Expand Up @@ -60,7 +60,7 @@ func TestDockerHubWithPrivateRepo(t *testing.T) {
t.Fatalf("Failed to get DockerHub private repo token: %s", err.Error())
}

tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128)
tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128, ".*")
if err != nil {
t.Fatalf("Failed to list DockerHub private repo (%s) tags: %s", repo, err.Error())
}
Expand Down
Loading

0 comments on commit d613f2c

Please sign in to comment.