initial commit

.gitignore

@@ -0,0 +1,2 @@
+config.yml
+.vscode/

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.15 AS gobuilder
+WORKDIR /build
+COPY go.mod go.sum /build/
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build
+
+FROM alpine:3.12
+RUN apk update && \
+    apk add ca-certificates && \
+    rm -rf /var/cache/apk/*
+COPY --from=gobuilder /build/gitlab-registry-cleanup /bin/gitlab-registry-cleanup
+ENTRYPOINT ["/bin/gitlab-registry-cleanup"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 UKFast.net Limited
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,73 @@
+# gitlab-registry-cleanup
+
+A small utility for cleaning up Gitlab container registry image tags, inspired by the built in registry cleanup policies built into Gitlab.
+
+
+## Why?
+
+When attempting to implement [Cleanup Policies](https://docs.gitlab.com/ee/user/packages/container_registry/#cleanup-policy), we found configuration options to be limited, with only one policy per repository currently possible. This application implements the same functionality, however allows many policies to be specified.
+
+## Usage
+
+```
+A tool for cleaning up gitlab registries
+
+Usage:
+  gitlab-registry-cleanup [command]
+
+Available Commands:
+  execute     Executes cleanup
+  help        Help about any command
+
+Flags:
+      --config string   config file (default "config.yml")
+      --debug           specifies logging level should be set to debug
+  -h, --help            help for gitlab-registry-cleanup
+```
+
+## Commands
+
+**execute**
+
+Executes cleanup process
+
+#### Flags
+
+* `--dry-run`: Specifies execution should be ran in dry run mode. Tag deletions will not occur
+
+
+## Config
+
+Application config is specified with a yaml configuration file, with an example below:
+
+```yaml
+access_token: myaccesstoken
+url: https://gitlab.privateinstance.com
+repositories:
+- project: 123
+  image: myproject/somerepository
+  filter:
+    include: .*
+    exclude: v.+
+    keep: 5
+    age: 30
+```
+
+* `access_token`: Private access token with `api` read/write scope
+* `url`: Gitlab instance URL
+* `repositories` __array__
+  * `project`: Project ID to target
+  * `image`: Path of repository/image
+  * `filter`: __object__
+    * `include`: (Optional) Regex specifying image tags to include
+    * `exclude`: (Optional) Regex specifying image tags to exclude
+    * `keep`: (Optional) Specifies amount of tags to keep
+    * `age`: (Optional) Specifies amount of days to keep tags
+
+## Docker
+
+We recommend using Docker for executing this utility. Example usage can be found below:
+
+```
+docker run -v "${PWD}/config.yml:/config.yml" --rm -it ukfast/gitlab-registry-cleanup execute --config /config.yml
+```

cmd/execute.go

@@ -0,0 +1,140 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/ukfast/gitlab-registry-cleanup/pkg/config"
+	"github.com/ukfast/gitlab-registry-cleanup/pkg/filter"
+	"github.com/xanzy/go-gitlab"
+)
+
+func ExecuteCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "execute",
+		Short: "Executes cleanup",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return executeCleanup(cmd, args)
+		},
+	}
+
+	cmd.Flags().Bool("dry-run", false, "specifies command should be ran in dry-run mode")
+
+	return cmd
+}
+
+func executeCleanup(cmd *cobra.Command, args []string) error {
+	cfg := &config.Config{}
+	err := viper.Unmarshal(cfg)
+	if err != nil {
+		return fmt.Errorf("Failed to unmarshal config: %w", err)
+	}
+
+	client, err := gitlab.NewClient(viper.GetString("access_token"), gitlab.WithBaseURL(viper.GetString("url")))
+	if err != nil {
+		return fmt.Errorf("Failed initialising Gitlab client: %s", err)
+	}
+
+	for _, repositoryCfg := range cfg.Repositories {
+		log.Infof("Processing repository %s", repositoryCfg.Image)
+		repositories, err := getAllRepositories(client, repositoryCfg)
+		if err != nil {
+			return fmt.Errorf("Error retrieving all Gitlab registry repositories for project %d: %s", repositoryCfg.Project, err)
+		}
+
+		for _, repository := range repositories {
+			if repositoryCfg.Image == repository.Path {
+				err := processRepository(cmd, client, repository, repositoryCfg)
+				if err != nil {
+					return fmt.Errorf("Failed processing repository %s: %s", repositoryCfg.Image, err)
+				}
+			}
+		}
+		log.Infof("Finished processing repository %s", repositoryCfg.Image)
+	}
+
+	return nil
+}
+
+func getAllRepositories(client *gitlab.Client, repositoryCfg config.RepositoryConfig) ([]*gitlab.RegistryRepository, error) {
+	var allRepositories []*gitlab.RegistryRepository
+	page := 1
+	for {
+		repositories, resp, err := client.ContainerRegistry.ListRegistryRepositories(repositoryCfg.Project, &gitlab.ListRegistryRepositoriesOptions{Page: page})
+		if err != nil {
+			return nil, err
+		}
+
+		allRepositories = append(allRepositories, repositories...)
+		if resp.CurrentPage >= resp.TotalPages {
+			break
+		}
+
+		page++
+	}
+
+	return allRepositories, nil
+}
+
+func getAllTags(client *gitlab.Client, repository *gitlab.RegistryRepository, repositoryCfg config.RepositoryConfig) ([]*gitlab.RegistryRepositoryTag, error) {
+	var allTags []*gitlab.RegistryRepositoryTag
+	page := 1
+	for {
+		tags, resp, err := client.ContainerRegistry.ListRegistryRepositoryTags(repositoryCfg.Project, repository.ID, &gitlab.ListRegistryRepositoryTagsOptions{Page: page})
+		if err != nil {
+			return nil, err
+		}
+
+		allTags = append(allTags, tags...)
+		if resp.CurrentPage >= resp.TotalPages {
+			break
+		}
+
+		page++
+	}
+
+	return allTags, nil
+}
+
+func processRepository(cmd *cobra.Command, client *gitlab.Client, repository *gitlab.RegistryRepository, repositoryCfg config.RepositoryConfig) error {
+	tags, err := getAllTags(client, repository, repositoryCfg)
+	if err != nil {
+		return fmt.Errorf("Failed retrieving tags: %w", err)
+	}
+
+	f := filter.NewFilterPipeline(tags, repositoryCfg.Filter)
+
+	filteredTags, err := f.Execute(
+		filter.ExcludeLatestFilter,
+		filter.IncludeFilter,
+		filter.OrderedFilter,
+		filter.KeepFilter,
+		filter.AgeFilter,
+		filter.ExcludeFilter,
+	)
+
+	if err != nil {
+		return fmt.Errorf("Failed to execute filter: %w", err)
+	}
+
+	log.Infof("Found %d tags to remove", len(filteredTags))
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	for _, filteredTag := range filteredTags {
+		logLine := fmt.Sprintf("Removing tag %s", filteredTag.Name)
+		if dryRun {
+			log.Infof("[DRY RUN]: %s", logLine)
+		} else {
+			log.Info(logLine)
+			_, err := client.ContainerRegistry.DeleteRegistryRepositoryTag(repositoryCfg.Project, repository.ID, filteredTag.Name)
+			if err != nil {
+				return fmt.Errorf("Failed to remove tag %s: %w", filteredTag.Name, err)
+			}
+		}
+	}
+
+	log.Infof("Finished removing %d tags", len(filteredTags))
+
+	return nil
+}

cmd/root.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var rootCmd = &cobra.Command{
+	Use:           "gitlab-registry-cleanup",
+	Short:         "A tool for cleaning up gitlab registries",
+	SilenceErrors: true,
+	SilenceUsage:  true,
+}
+
+// Execute executes the root command.
+func Execute() error {
+	return rootCmd.Execute()
+}
+
+func init() {
+	cobra.OnInitialize(
+		initConfig,
+		initLogging,
+	)
+
+	rootCmd.PersistentFlags().String("config", "config.yml", "config file")
+	rootCmd.PersistentFlags().Bool("debug", false, "specifies logging level should be set to debug")
+	viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))
+
+	rootCmd.AddCommand(ExecuteCmd())
+}
+
+func initConfig() {
+	config, _ := rootCmd.Flags().GetString("config")
+	if config != "" {
+		viper.SetConfigFile(config)
+	} else {
+		viper.AddConfigPath(".")
+		viper.SetConfigName("config")
+	}
+
+	viper.AutomaticEnv()
+
+	err := viper.ReadInConfig()
+	if err != nil {
+		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+			log.Fatalf("Failed to load/parse config file: %s", err)
+		}
+		return
+	}
+	log.Infof("Using config file: %s", viper.ConfigFileUsed())
+}
+
+func initLogging() {
+	if viper.GetBool("debug") {
+		log.SetLevel(log.DebugLevel)
+	}
+}

config.example.yml

@@ -0,0 +1,12 @@
+---
+
+access_token: myaccesstoken
+url: https://gitlab.privateinstance.com
+repositories:
+- project: 123
+  image: myproject/somerepository
+  filter:
+    include: .*
+    exclude: v.+
+    keep: 5
+    age: 30

go.mod

@@ -0,0 +1,12 @@
+module github.com/ukfast/gitlab-registry-cleanup
+
+go 1.15
+
+require (
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/cobra v1.1.1
+	github.com/spf13/viper v1.7.0
+	github.com/stretchr/testify v1.4.0
+	github.com/xanzy/go-gitlab v0.39.0
+	gopkg.in/yaml.v2 v2.2.8
+)