From 8adbb2c6a99680a0c45ee478868c26ed553496a4 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 4 Oct 2020 19:56:33 +0200 Subject: [PATCH] Refactor to start working on #48 --- README.md | 2 + cmd/main.go | 11 +- internal/app/ftpgrab.go | 302 ++------------------ internal/{model => config}/cli.go | 2 +- internal/config/config.go | 21 +- internal/config/config_test.go | 121 ++++---- internal/{model => config}/db.go | 2 +- internal/{model => config}/download.go | 2 +- internal/config/file_finder.go | 51 ---- internal/config/file_finder_test.go | 155 ---------- internal/{model => config}/meta.go | 2 +- internal/{model => config}/notif.go | 2 +- internal/{model => config}/notif_mail.go | 2 +- internal/{model => config}/notif_slack.go | 2 +- internal/{model => config}/notif_webhook.go | 2 +- internal/{model => config}/server.go | 2 +- internal/{model => config}/server_ftp.go | 2 +- internal/{model => config}/server_sftp.go | 2 +- internal/db/client.go | 10 +- internal/grabber/file.go | 69 +++++ internal/grabber/grabber.go | 238 +++++++++++++++ internal/journal/client.go | 23 +- internal/journal/entry.go | 40 +++ internal/{model => journal}/journal.go | 19 +- internal/logging/logger.go | 4 +- internal/notif/client.go | 28 +- internal/notif/mail/client.go | 21 +- internal/notif/notifier/notifier.go | 2 +- internal/notif/slack/slack.go | 12 +- internal/notif/webhook/client.go | 20 +- internal/server/client.go | 4 +- internal/server/ftp/client.go | 38 +-- internal/server/sftp/client.go | 15 +- 33 files changed, 571 insertions(+), 657 deletions(-) rename internal/{model => config}/cli.go (97%) rename internal/{model => config}/db.go (96%) rename internal/{model => config}/download.go (98%) delete mode 100644 internal/config/file_finder.go delete mode 100644 internal/config/file_finder_test.go rename internal/{model => config}/meta.go (93%) rename internal/{model => config}/notif.go (96%) rename internal/{model => config}/notif_mail.go (98%) rename internal/{model => config}/notif_slack.go (96%) rename internal/{model => config}/notif_webhook.go (98%) rename internal/{model => config}/server.go (97%) rename internal/{model => config}/server_ftp.go (99%) rename internal/{model => config}/server_sftp.go (99%) create mode 100644 internal/grabber/file.go create mode 100644 internal/grabber/grabber.go create mode 100644 internal/journal/entry.go rename internal/{model => journal}/journal.go (67%) diff --git a/README.md b/README.md index 377fdfe0..37905d23 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ to your NAS, server or computer. ![](.res/screenshot.png) +💡 Want to be notified of new releases? Check out 🔔 [Diun (Docker Image Update Notifier)](https://github.com/crazy-max/diun) project! + ## Documentation Documentation can be found on https://crazy-max.github.io/ftpgrab/ diff --git a/cmd/main.go b/cmd/main.go index 708f97ab..f9d39c14 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,15 +13,14 @@ import ( "github.com/crazy-max/ftpgrab/v7/internal/app" "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/logging" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/rs/zerolog/log" ) var ( ftpgrab *app.FtpGrab - cli model.Cli + cli config.Cli version = "dev" - meta = model.Meta{ + meta = config.Meta{ ID: "ftpgrab", Name: "FTPGrab", Desc: "Grab your files periodically from a remote FTP or SFTP server easily", @@ -61,7 +60,7 @@ func main() { } // Init - logging.Configure(&cli, location) + logging.Configure(cli, location) log.Info().Str("version", version).Msgf("Starting %s", meta.Name) // Handle os signals @@ -75,14 +74,14 @@ func main() { }() // Load configuration - cfg, err := config.Load(cli.Cfgfile, cli.Schedule) + cfg, err := config.Load(cli, meta) if err != nil { log.Fatal().Err(err).Msg("Cannot load configuration") } log.Debug().Msg(cfg.String()) // Init - if ftpgrab, err = app.New(meta, cfg, location); err != nil { + if ftpgrab, err = app.New(cfg, location); err != nil { log.Fatal().Err(err).Msgf("Cannot initialize %s", meta.Name) } diff --git a/internal/app/ftpgrab.go b/internal/app/ftpgrab.go index 94a122a6..675e2abd 100644 --- a/internal/app/ftpgrab.go +++ b/internal/app/ftpgrab.go @@ -1,23 +1,13 @@ package app import ( - "fmt" - "os" - "path" - "runtime" "sync/atomic" "time" "github.com/crazy-max/ftpgrab/v7/internal/config" - "github.com/crazy-max/ftpgrab/v7/internal/db" - "github.com/crazy-max/ftpgrab/v7/internal/journal" - "github.com/crazy-max/ftpgrab/v7/internal/model" + "github.com/crazy-max/ftpgrab/v7/internal/grabber" "github.com/crazy-max/ftpgrab/v7/internal/notif" "github.com/crazy-max/ftpgrab/v7/internal/server" - "github.com/crazy-max/ftpgrab/v7/internal/server/ftp" - "github.com/crazy-max/ftpgrab/v7/internal/server/sftp" - "github.com/crazy-max/ftpgrab/v7/pkg/utl" - "github.com/docker/go-units" "github.com/hako/durafmt" "github.com/robfig/cron/v3" "github.com/rs/zerolog/log" @@ -25,32 +15,19 @@ import ( // FtpGrab represents an active ftpgrab object type FtpGrab struct { - meta model.Meta - cfg *config.Config - cron *cron.Cron - srv *server.Client - db *db.Client - notif *notif.Client - jnl *journal.Client - jobID cron.EntryID - locker uint32 + cfg *config.Config + cron *cron.Cron + srv *server.Client + notif *notif.Client + grabber *grabber.Client + jobID cron.EntryID + locker uint32 } -const ( - outdated = model.EntryStatus("Outdated file") - notIncluded = model.EntryStatus("Not included") - excluded = model.EntryStatus("Excluded") - neverDl = model.EntryStatus("Never downloaded") - alreadyDl = model.EntryStatus("Already downloaded") - sizeDiff = model.EntryStatus("Exists but size is different") - hashExists = model.EntryStatus("Hash sum exists") -) - // New creates new ftpgrab instance -func New(meta model.Meta, cfg *config.Config, location *time.Location) (*FtpGrab, error) { +func New(cfg *config.Config, location *time.Location) (*FtpGrab, error) { return &FtpGrab{ - meta: meta, - cfg: cfg, + cfg: cfg, cron: cron.New(cron.WithLocation(location), cron.WithParser(cron.NewParser( cron.SecondOptional|cron.Minute|cron.Hour|cron.Dom|cron.Month|cron.Dow|cron.Descriptor), )), @@ -65,14 +42,13 @@ func (fg *FtpGrab) Start() error { fg.Run() // Init scheduler if defined - if len(fg.cfg.Schedule) == 0 { + if len(fg.cfg.Cli.Schedule) == 0 { return nil } - fg.jobID, err = fg.cron.AddJob(fg.cfg.Schedule, fg) - if err != nil { + if fg.jobID, err = fg.cron.AddJob(fg.cfg.Cli.Schedule, fg); err != nil { return err } - log.Info().Msgf("Cron initialized with schedule %s", fg.cfg.Schedule) + log.Info().Msgf("Cron initialized with schedule %s", fg.cfg.Cli.Schedule) // Start scheduler fg.cron.Start() @@ -99,264 +75,46 @@ func (fg *FtpGrab) Run() { start := time.Now() var err error - // Journal client - fg.jnl = journal.New() - - // Server client - if fg.cfg.Server.FTP != nil { - fg.srv, err = ftp.New(fg.cfg.Server.FTP) - fg.jnl.ServerHost = fg.cfg.Server.FTP.Host - } else if fg.cfg.Server.SFTP != nil { - fg.srv, err = sftp.New(fg.cfg.Server.SFTP) - fg.jnl.ServerHost = fg.cfg.Server.SFTP.Host - } else { - log.Fatal().Err(err).Msg("No server defined") - } - if err != nil { - log.Fatal().Err(err).Msg("Cannot connect to server") - } - - // DB client - if fg.db, err = db.New(fg.cfg.Db); err != nil { - log.Fatal().Err(err).Msg("Cannot open database") - } - // Notification client - if fg.notif, err = notif.New(fg.cfg.Notif, fg.meta); err != nil { + if fg.notif, err = notif.New(fg.cfg.Notif, fg.cfg.Meta); err != nil { log.Fatal().Err(err).Msg("Cannot create notifiers") } - // Iterate sources - for _, src := range fg.srv.Common().Sources { - log.Info().Str("source", src).Msg("Grabbing") - - // Check basedir - dest := fg.cfg.Download.Output - if src != "/" && *fg.cfg.Download.CreateBaseDir { - dest = path.Join(dest, src) - } - - // Retrieve recursively - fg.retrieveRecursive(src, src, dest) + // Grabber client + if fg.grabber, err = grabber.New(fg.cfg.Download, fg.cfg.Db, fg.cfg.Server); err != nil { + log.Fatal().Err(err).Msg("Cannot create grabber") } + defer fg.grabber.Close() - if err := fg.srv.Close(); err != nil { - log.Warn().Err(err).Msg("Cannot close server connection") - } - if err := fg.db.Close(); err != nil { - log.Warn().Err(err).Msg("Cannot close database") + // List files + files := fg.grabber.ListFiles() + if len(files) == 0 { + log.Warn().Msg("No file found from the provided sources") + return } - fg.jnl.Duration = time.Since(start) + log.Info().Msgf("%d file(s) found", len(files)) + // Grab + jnl := fg.grabber.Grab(files) + jnl.Duration = time.Since(start) log.Info(). Str("duration", time.Since(start).Round(time.Millisecond).String()). Msg("Finished") // Check journal before sending report - if fg.jnl.IsEmpty() { + if jnl.IsEmpty() { log.Warn().Msg("Journal empty, skip sending report") return } // Send notifications - fg.notif.Send(*fg.jnl) + fg.notif.Send(jnl) } // Close closes ftpgrab func (fg *FtpGrab) Close() { - if err := fg.srv.Close(); err != nil { - log.Warn().Err(err).Msg("Cannot close server connection") - } - if err := fg.db.Close(); err != nil { - log.Warn().Err(err).Msg("Cannot close database") - } + fg.grabber.Close() if fg.cron != nil { fg.cron.Stop() } } - -func (fg *FtpGrab) retrieveRecursive(base string, source string, dest string) { - // Check source dir exists - files, err := fg.srv.ReadDir(source) - if err != nil { - log.Error().Err(err).Str("source", base). - Msgf("Cannot read directory %s", source) - return - } - - for _, file := range files { - if jnlEntry := fg.retrieve(base, source, dest, file, 0); jnlEntry != nil { - fg.jnl.AddEntry(*jnlEntry) - } - } -} - -func (fg *FtpGrab) retrieve(base string, src string, dest string, file os.FileInfo, retry int) *model.Entry { - srcpath := path.Join(src, file.Name()) - destpath := path.Join(dest, file.Name()) - - if file.IsDir() { - fg.retrieveRecursive(base, srcpath, destpath) - return nil - } - - status := fg.fileStatus(base, src, dest, file) - jnlEntry := &model.Entry{ - File: srcpath, - StatusText: string(status), - } - - sublogger := log.With(). - Str("file", jnlEntry.File). - Str("size", units.HumanSize(float64(file.Size()))). - Logger() - - if status == alreadyDl && !fg.db.HasHash(base, src, file) { - if err := fg.db.PutHash(base, src, file); err != nil { - sublogger.Error().Err(err).Msg("Cannot add hash into db") - } - } - if fg.isSkipped(status) { - if !*fg.cfg.Download.HideSkipped { - sublogger.Warn().Str(".status", jnlEntry.StatusText).Msg("Skipped") - jnlEntry.StatusType = "skip" - return jnlEntry - } - return nil - } - - retrieveStart := time.Now() - sublogger.Info().Str("dest", destpath).Msg("Downloading...") - - destfolder := path.Dir(destpath) - if err := os.MkdirAll(destfolder, os.ModePerm); err != nil { - sublogger.Error().Err(err).Msg("Cannot create destination dir") - jnlEntry.StatusType = "error" - jnlEntry.StatusText = fmt.Sprintf("Cannot create destination dir: %v", err) - return jnlEntry - } - if err := fg.fixPerms(destfolder); err != nil { - sublogger.Warn().Err(err).Msg("Cannot fix parent folder permissions") - } - - destfile, err := os.Create(destpath) - if err != nil { - sublogger.Error().Err(err).Msg("Cannot create destination file") - jnlEntry.StatusType = "error" - jnlEntry.StatusText = fmt.Sprintf("Cannot create destination file: %v", err) - return jnlEntry - } - - err = fg.srv.Retrieve(srcpath, destfile) - if err != nil { - retry++ - sublogger.Error().Err(err).Msgf("Error downloading, retry %d/%d", retry, fg.cfg.Download.Retry) - if retry == fg.cfg.Download.Retry { - sublogger.Error().Err(err).Msg("Cannot download file") - jnlEntry.StatusType = "error" - jnlEntry.StatusText = fmt.Sprintf("Cannot download file: %v", err) - } else { - fg.retrieve(base, src, dest, file, retry) - return nil - } - } else { - sublogger.Info(). - Str("duration", time.Since(retrieveStart).Round(time.Millisecond).String()). - Msg("File successfully downloaded") - jnlEntry.StatusType = "success" - jnlEntry.StatusText = fmt.Sprintf("%s successfully downloaded in %s", - units.HumanSize(float64(file.Size())), - time.Since(retrieveStart).Round(time.Millisecond).String(), - ) - if err := fg.fixPerms(destpath); err != nil { - sublogger.Warn().Err(err).Msg("Cannot fix file permissions") - } - if err := fg.db.PutHash(base, src, file); err != nil { - sublogger.Error().Err(err).Msg("Cannot add hash into db") - jnlEntry.StatusType = "warning" - jnlEntry.StatusText = fmt.Sprintf("Successfully downloaded but cannot add hash into db: %v", err) - } - if err = os.Chtimes(destpath, file.ModTime(), file.ModTime()); err != nil { - sublogger.Warn().Err(err).Msg("Cannot change modtime of destination file") - } - } - - return jnlEntry -} - -func (fg *FtpGrab) fileStatus(base string, src string, dest string, file os.FileInfo) model.EntryStatus { - if !fg.isIncluded(file.Name()) { - return notIncluded - } else if fg.isExcluded(file.Name()) { - return excluded - } else if file.ModTime().Before(fg.cfg.Download.SinceTime) { - return outdated - } else if destfile, err := os.Stat(path.Join(dest, file.Name())); err == nil { - if destfile.Size() == file.Size() { - return alreadyDl - } - return sizeDiff - } else if fg.db.HasHash(base, src, file) { - return hashExists - } - - return neverDl -} - -func (fg *FtpGrab) fixPerms(filepath string) error { - if runtime.GOOS == "windows" { - return nil - } - - fileinfo, err := os.Stat(filepath) - if err != nil { - return err - } - - chmod := os.FileMode(fg.cfg.Download.ChmodFile) - if fileinfo.IsDir() { - chmod = os.FileMode(fg.cfg.Download.ChmodDir) - } - - if err := os.Chmod(filepath, chmod); err != nil { - return err - } - - if err := os.Chown(filepath, fg.cfg.Download.UID, fg.cfg.Download.GID); err != nil { - return err - } - - return nil -} - -func (fg *FtpGrab) isIncluded(filename string) bool { - if len(fg.cfg.Download.Include) == 0 { - return true - } - for _, include := range fg.cfg.Download.Include { - if utl.MatchString(include, filename) { - return true - } - } - return false -} - -func (fg *FtpGrab) isExcluded(filename string) bool { - if len(fg.cfg.Download.Exclude) == 0 { - return false - } - for _, exclude := range fg.cfg.Download.Exclude { - if utl.MatchString(exclude, filename) { - return true - } - } - return false -} - -func (fg *FtpGrab) isSkipped(status model.EntryStatus) bool { - return status == alreadyDl || - status == hashExists || - status == outdated || - status == notIncluded || - status == excluded -} diff --git a/internal/model/cli.go b/internal/config/cli.go similarity index 97% rename from internal/model/cli.go rename to internal/config/cli.go index c7b8c4b1..3ff67062 100644 --- a/internal/model/cli.go +++ b/internal/config/cli.go @@ -1,4 +1,4 @@ -package model +package config import "github.com/alecthomas/kong" diff --git a/internal/config/config.go b/internal/config/config.go index a8d186fa..fa7fa481 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,7 +7,6 @@ import ( "regexp" "time" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/gonfig" "github.com/go-playground/validator/v10" "github.com/pkg/errors" @@ -16,22 +15,24 @@ import ( // Config holds configuration details type Config struct { - Schedule string `yaml:"schedule,omitempty" json:"schedule,omitempty"` - Db *model.Db `yaml:"db,omitempty" json:"db,omitempty" validate:"omitempty"` - Server *model.Server `yaml:"server,omitempty" json:"server,omitempty" validate:"required"` - Download *model.Download `yaml:"download,omitempty" json:"download,omitempty" validate:"required"` - Notif *model.Notif `yaml:"notif,omitempty" json:"notif,omitempty"` + Cli Cli `yaml:"-" json:"-" label:"-" file:"-"` + Meta Meta `yaml:"-" json:"-" label:"-" file:"-"` + Db *Db `yaml:"db,omitempty" json:"db,omitempty" validate:"omitempty"` + Server *Server `yaml:"server,omitempty" json:"server,omitempty" validate:"required"` + Download *Download `yaml:"download,omitempty" json:"download,omitempty" validate:"required"` + Notif *Notif `yaml:"notif,omitempty" json:"notif,omitempty"` } // Load returns Config struct -func Load(cfgfile string, schedule string) (*Config, error) { +func Load(cli Cli, meta Meta) (*Config, error) { cfg := Config{ - Schedule: schedule, - Db: (&model.Db{}).GetDefaults(), + Cli: cli, + Meta: meta, + Db: (&Db{}).GetDefaults(), } fileLoader := gonfig.NewFileLoader(gonfig.FileLoaderConfig{ - Filename: cfgfile, + Filename: cli.Cfgfile, Finder: gonfig.Finder{ BasePaths: []string{"/etc/ftpgrab/ftpgrab", "$XDG_CONFIG_HOME/ftpgrab", "$HOME/.config/ftpgrab", "./ftpgrab"}, Extensions: []string{"yaml", "yml"}, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0eb0b049..857e545d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/pkg/utl" "github.com/crazy-max/gonfig/env" "github.com/stretchr/testify/assert" @@ -17,27 +16,33 @@ import ( func TestLoadFile(t *testing.T) { cases := []struct { name string - cfgfile string + cli Cli wantData *Config wantErr bool }{ { name: "Failed on non-existing file", - cfgfile: "", wantErr: true, }, { - name: "Fail on wrong file format", - cfgfile: "./fixtures/config.invalid.yml", + name: "Fail on wrong file format", + cli: Cli{ + Cfgfile: "./fixtures/config.invalid.yml", + }, wantErr: true, }, { - name: "Success", - cfgfile: "./fixtures/config.test.yml", + name: "Success", + cli: Cli{ + Cfgfile: "./fixtures/config.test.yml", + }, wantData: &Config{ - Db: (&model.Db{}).GetDefaults(), - Server: &model.Server{ - FTP: &model.ServerFTP{ + Cli: Cli{ + Cfgfile: "./fixtures/config.test.yml", + }, + Db: (&Db{}).GetDefaults(), + Server: &Server{ + FTP: &ServerFTP{ Host: "test.rebex.net", Port: 21, Username: "demo", @@ -52,7 +57,7 @@ func TestLoadFile(t *testing.T) { LogTrace: utl.NewFalse(), }, }, - Download: &model.Download{ + Download: &Download{ Output: "./fixtures/downloads", UID: os.Getuid(), GID: os.Getgid(), @@ -64,8 +69,8 @@ func TestLoadFile(t *testing.T) { HideSkipped: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, - Notif: &model.Notif{ - Mail: &model.NotifMail{ + Notif: &Notif{ + Mail: &NotifMail{ Host: "localhost", Port: 25, SSL: utl.NewFalse(), @@ -73,10 +78,10 @@ func TestLoadFile(t *testing.T) { From: "ftpgrab@example.com", To: "webmaster@example.com", }, - Slack: &model.NotifSlack{ + Slack: &NotifSlack{ WebhookURL: "https://hooks.slack.com/services/ABCD12EFG/HIJK34LMN/01234567890abcdefghij", }, - Webhook: &model.NotifWebhook{ + Webhook: &NotifWebhook{ Endpoint: "http://webhook.foo.com/sd54qad89azd5a", Method: "GET", Headers: map[string]string{ @@ -91,7 +96,7 @@ func TestLoadFile(t *testing.T) { } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - cfg, err := Load(tt.cfgfile, "") + cfg, err := Load(tt.cli, Meta{}) if tt.wantErr { require.Error(t, err) return @@ -110,7 +115,7 @@ func TestLoadEnv(t *testing.T) { testCases := []struct { desc string - cfgfile string + cli Cli environ []string expected interface{} wantErr bool @@ -131,9 +136,9 @@ func TestLoadEnv(t *testing.T) { "FTPGRAB_DOWNLOAD_OUTPUT=./fixtures/downloads", }, expected: &Config{ - Db: (&model.Db{}).GetDefaults(), - Server: &model.Server{ - FTP: &model.ServerFTP{ + Db: (&Db{}).GetDefaults(), + Server: &Server{ + FTP: &ServerFTP{ Host: "test.rebex.net", Port: 21, Username: "demo", @@ -148,7 +153,7 @@ func TestLoadEnv(t *testing.T) { LogTrace: utl.NewFalse(), }, }, - Download: &model.Download{ + Download: &Download{ Output: "./fixtures/downloads", UID: os.Getuid(), GID: os.Getgid(), @@ -171,9 +176,9 @@ func TestLoadEnv(t *testing.T) { "FTPGRAB_DOWNLOAD_OUTPUT=./fixtures/downloads", }, expected: &Config{ - Db: (&model.Db{}).GetDefaults(), - Server: &model.Server{ - SFTP: &model.ServerSFTP{ + Db: (&Db{}).GetDefaults(), + Server: &Server{ + SFTP: &ServerSFTP{ Host: "10.0.0.1", Port: 22, UsernameFile: "./fixtures/run_secrets_username", @@ -185,7 +190,7 @@ func TestLoadEnv(t *testing.T) { MaxPacketSize: 32768, }, }, - Download: &model.Download{ + Download: &Download{ Output: "./fixtures/downloads", UID: os.Getuid(), GID: os.Getgid(), @@ -227,7 +232,7 @@ func TestLoadEnv(t *testing.T) { } } - cfg, err := Load(tt.cfgfile, "") + cfg, err := Load(tt.cli, Meta{}) if tt.wantErr { require.Error(t, err) return @@ -244,14 +249,16 @@ func TestLoadMixed(t *testing.T) { testCases := []struct { desc string - cfgfile string + cli Cli environ []string expected interface{} wantErr bool }{ { - desc: "env vars and invalid file", - cfgfile: "./fixtures/config.invalid.yml", + desc: "env vars and invalid file", + cli: Cli{ + Cfgfile: "./fixtures/config.invalid.yml", + }, environ: []string{ "FTPGRAB_SERVER_FTP_HOST=test.rebex.net", "FTPGRAB_SERVER_FTP_USERNAME=demo", @@ -263,8 +270,10 @@ func TestLoadMixed(t *testing.T) { wantErr: true, }, { - desc: "ftp server (file) and notif mails (envs)", - cfgfile: "./fixtures/config.ftp.yml", + desc: "ftp server (file) and notif mails (envs)", + cli: Cli{ + Cfgfile: "./fixtures/config.ftp.yml", + }, environ: []string{ "FTPGRAB_NOTIF_MAIL_HOST=127.0.0.1", "FTPGRAB_NOTIF_MAIL_PORT=25", @@ -274,11 +283,14 @@ func TestLoadMixed(t *testing.T) { "FTPGRAB_NOTIF_MAIL_TO=webmaster@foo.com", }, expected: &Config{ - Db: &model.Db{ + Cli: Cli{ + Cfgfile: "./fixtures/config.ftp.yml", + }, + Db: &Db{ Path: "./fixtures/db/ftpgrab.db", }, - Server: &model.Server{ - FTP: &model.ServerFTP{ + Server: &Server{ + FTP: &ServerFTP{ Host: "test.rebex.net", Port: 21, Username: "demo", @@ -293,7 +305,7 @@ func TestLoadMixed(t *testing.T) { LogTrace: utl.NewFalse(), }, }, - Download: &model.Download{ + Download: &Download{ Output: "./fixtures/downloads", UID: os.Getuid(), GID: os.Getgid(), @@ -303,8 +315,8 @@ func TestLoadMixed(t *testing.T) { HideSkipped: utl.NewFalse(), CreateBaseDir: utl.NewFalse(), }, - Notif: &model.Notif{ - Mail: &model.NotifMail{ + Notif: &Notif{ + Mail: &NotifMail{ Host: "127.0.0.1", Port: 25, SSL: utl.NewFalse(), @@ -317,17 +329,22 @@ func TestLoadMixed(t *testing.T) { wantErr: false, }, { - desc: "sftp server (file) and notif slack (envs)", - cfgfile: "./fixtures/config.sftp.yml", + desc: "sftp server (file) and notif slack (envs)", + cli: Cli{ + Cfgfile: "./fixtures/config.sftp.yml", + }, environ: []string{ "FTPGRAB_NOTIF_SLACK_WEBHOOKURL=https://hooks.slack.com/services/ABCD12EFG/HIJK34LMN/01234567890abcdefghij", }, expected: &Config{ - Db: &model.Db{ + Cli: Cli{ + Cfgfile: "./fixtures/config.sftp.yml", + }, + Db: &Db{ Path: "./fixtures/db/ftpgrab.db", }, - Server: &model.Server{ - SFTP: &model.ServerSFTP{ + Server: &Server{ + SFTP: &ServerSFTP{ Host: "10.0.0.1", Port: 22, Username: "foo", @@ -339,7 +356,7 @@ func TestLoadMixed(t *testing.T) { MaxPacketSize: 32768, }, }, - Download: &model.Download{ + Download: &Download{ Output: "./fixtures/downloads", UID: os.Getuid(), GID: os.Getgid(), @@ -349,8 +366,8 @@ func TestLoadMixed(t *testing.T) { HideSkipped: utl.NewTrue(), CreateBaseDir: utl.NewFalse(), }, - Notif: &model.Notif{ - Slack: &model.NotifSlack{ + Notif: &Notif{ + Slack: &NotifSlack{ WebhookURL: "https://hooks.slack.com/services/ABCD12EFG/HIJK34LMN/01234567890abcdefghij", }, }, @@ -370,7 +387,7 @@ func TestLoadMixed(t *testing.T) { } } - cfg, err := Load(tt.cfgfile, "") + cfg, err := Load(tt.cli, Meta{}) if tt.wantErr { require.Error(t, err) return @@ -384,17 +401,19 @@ func TestLoadMixed(t *testing.T) { func TestValidation(t *testing.T) { cases := []struct { - name string - cfgfile string + name string + cli Cli }{ { - name: "Success", - cfgfile: "./fixtures/config.validate.yml", + name: "Success", + cli: Cli{ + Cfgfile: "./fixtures/config.validate.yml", + }, }, } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - cfg, err := Load(tt.cfgfile, "") + cfg, err := Load(tt.cli, Meta{}) require.NoError(t, err) dec, err := env.Encode(cfg) diff --git a/internal/model/db.go b/internal/config/db.go similarity index 96% rename from internal/model/db.go rename to internal/config/db.go index d4369ef5..3419b977 100644 --- a/internal/model/db.go +++ b/internal/config/db.go @@ -1,4 +1,4 @@ -package model +package config // Db holds data necessary for database configuration type Db struct { diff --git a/internal/model/download.go b/internal/config/download.go similarity index 98% rename from internal/model/download.go rename to internal/config/download.go index f1e11dd1..65f5fd9c 100644 --- a/internal/model/download.go +++ b/internal/config/download.go @@ -1,4 +1,4 @@ -package model +package config import ( "os" diff --git a/internal/config/file_finder.go b/internal/config/file_finder.go deleted file mode 100644 index 68bd90b5..00000000 --- a/internal/config/file_finder.go +++ /dev/null @@ -1,51 +0,0 @@ -package config - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// Finder holds a list of file paths. -type Finder struct { - BasePaths []string - Extensions []string -} - -// Find returns the first valid existing file among configFile -// and the paths already registered with Finder. -func (f Finder) Find(configFile string) (string, error) { - paths := f.getPaths(configFile) - - for _, filePath := range paths { - fp := os.ExpandEnv(filePath) - - _, err := os.Stat(fp) - if os.IsNotExist(err) { - continue - } - if err != nil { - return "", err - } - - return filepath.Abs(fp) - } - - return "", nil -} - -func (f Finder) getPaths(configFile string) []string { - var paths []string - if len(strings.TrimSpace(configFile)) > 0 { - paths = append(paths, configFile) - } - - for _, basePath := range f.BasePaths { - for _, ext := range f.Extensions { - paths = append(paths, fmt.Sprintf("%s.%s", basePath, ext)) - } - } - - return paths -} diff --git a/internal/config/file_finder_test.go b/internal/config/file_finder_test.go deleted file mode 100644 index e5809c5b..00000000 --- a/internal/config/file_finder_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package config - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFinderFind(t *testing.T) { - configFile, err := ioutil.TempFile("", "ftpgrab-file-finder-test-*.yml") - require.NoError(t, err) - - defer func() { - _ = os.Remove(configFile.Name()) - }() - - dir, err := ioutil.TempDir("", "ftpgrab-file-finder-test") - require.NoError(t, err) - - defer func() { - _ = os.RemoveAll(dir) - }() - - fooFile, err := os.Create(filepath.Join(dir, "foo.yml")) - require.NoError(t, err) - - _, err = os.Create(filepath.Join(dir, "bar.yml")) - require.NoError(t, err) - - type expected struct { - error bool - path string - } - - testCases := []struct { - desc string - basePaths []string - configFile string - expected expected - }{ - { - desc: "not found: no config file", - configFile: "", - expected: expected{path: ""}, - }, - { - desc: "not found: no config file, no other paths available", - configFile: "", - basePaths: []string{"/my/path/ftpgrab", "$HOME/my/path/ftpgrab", "./my-ftpgrab"}, - expected: expected{path: ""}, - }, - { - desc: "not found: with non existing config file", - configFile: "/my/path/config.yml", - expected: expected{path: ""}, - }, - { - desc: "found: with config file", - configFile: configFile.Name(), - expected: expected{path: configFile.Name()}, - }, - { - desc: "found: no config file, first base path", - configFile: "", - basePaths: []string{filepath.Join(dir, "foo"), filepath.Join(dir, "bar")}, - expected: expected{path: fooFile.Name()}, - }, - { - desc: "found: no config file, base path", - configFile: "", - basePaths: []string{"/my/path/ftpgrab", "$HOME/my/path/ftpgrab", filepath.Join(dir, "foo")}, - expected: expected{path: fooFile.Name()}, - }, - { - desc: "found: config file over base path", - configFile: configFile.Name(), - basePaths: []string{filepath.Join(dir, "foo"), filepath.Join(dir, "bar")}, - expected: expected{path: configFile.Name()}, - }, - } - - for _, test := range testCases { - t.Run(test.desc, func(t *testing.T) { - finder := Finder{ - BasePaths: test.basePaths, - Extensions: []string{"yaml", "yml"}, - } - - path, err := finder.Find(test.configFile) - - if test.expected.error { - require.Error(t, err) - } else { - require.NoError(t, err) - assert.Equal(t, test.expected.path, path) - } - }) - } -} - -func TestFinderGetPaths(t *testing.T) { - testCases := []struct { - desc string - basePaths []string - configFile string - expected []string - }{ - { - desc: "no config file", - basePaths: []string{"/etc/ftpgrab/ftpgrab", "$HOME/.config/ftpgrab", "./ftpgrab"}, - configFile: "", - expected: []string{ - "/etc/ftpgrab/ftpgrab.yaml", - "/etc/ftpgrab/ftpgrab.yml", - "$HOME/.config/ftpgrab.yaml", - "$HOME/.config/ftpgrab.yml", - "./ftpgrab.yaml", - "./ftpgrab.yml", - }, - }, - { - desc: "with config file", - basePaths: []string{"/etc/ftpgrab/ftpgrab", "$HOME/.config/ftpgrab", "./ftpgrab"}, - configFile: "/my/path/config.yml", - expected: []string{ - "/my/path/config.yml", - "/etc/ftpgrab/ftpgrab.yaml", - "/etc/ftpgrab/ftpgrab.yml", - "$HOME/.config/ftpgrab.yaml", - "$HOME/.config/ftpgrab.yml", - "./ftpgrab.yaml", - "./ftpgrab.yml", - }, - }, - } - - for _, test := range testCases { - test := test - t.Run(test.desc, func(t *testing.T) { - t.Parallel() - - finder := Finder{ - BasePaths: test.basePaths, - Extensions: []string{"yaml", "yml"}, - } - paths := finder.getPaths(test.configFile) - - assert.Equal(t, test.expected, paths) - }) - } -} diff --git a/internal/model/meta.go b/internal/config/meta.go similarity index 93% rename from internal/model/meta.go rename to internal/config/meta.go index d9580f0a..4a9c206f 100644 --- a/internal/model/meta.go +++ b/internal/config/meta.go @@ -1,4 +1,4 @@ -package model +package config // Meta holds application details type Meta struct { diff --git a/internal/model/notif.go b/internal/config/notif.go similarity index 96% rename from internal/model/notif.go rename to internal/config/notif.go index 236f4c01..a38bc229 100644 --- a/internal/model/notif.go +++ b/internal/config/notif.go @@ -1,4 +1,4 @@ -package model +package config // Notif holds data necessary for notification configuration type Notif struct { diff --git a/internal/model/notif_mail.go b/internal/config/notif_mail.go similarity index 98% rename from internal/model/notif_mail.go rename to internal/config/notif_mail.go index d8c1fb0f..a8dbf139 100644 --- a/internal/model/notif_mail.go +++ b/internal/config/notif_mail.go @@ -1,4 +1,4 @@ -package model +package config import ( "github.com/crazy-max/ftpgrab/v7/pkg/utl" diff --git a/internal/model/notif_slack.go b/internal/config/notif_slack.go similarity index 96% rename from internal/model/notif_slack.go rename to internal/config/notif_slack.go index 4d2dd68a..38136846 100644 --- a/internal/model/notif_slack.go +++ b/internal/config/notif_slack.go @@ -1,4 +1,4 @@ -package model +package config // NotifSlack holds slack notification configuration details type NotifSlack struct { diff --git a/internal/model/notif_webhook.go b/internal/config/notif_webhook.go similarity index 98% rename from internal/model/notif_webhook.go rename to internal/config/notif_webhook.go index 9dc42746..c30f3177 100644 --- a/internal/model/notif_webhook.go +++ b/internal/config/notif_webhook.go @@ -1,4 +1,4 @@ -package model +package config import ( "time" diff --git a/internal/model/server.go b/internal/config/server.go similarity index 97% rename from internal/model/server.go rename to internal/config/server.go index ec03bbbe..be7f07bd 100644 --- a/internal/model/server.go +++ b/internal/config/server.go @@ -1,4 +1,4 @@ -package model +package config // Server represents a server configuration type Server struct { diff --git a/internal/model/server_ftp.go b/internal/config/server_ftp.go similarity index 99% rename from internal/model/server_ftp.go rename to internal/config/server_ftp.go index d39e4971..18fab76d 100644 --- a/internal/model/server_ftp.go +++ b/internal/config/server_ftp.go @@ -1,4 +1,4 @@ -package model +package config import ( "time" diff --git a/internal/model/server_sftp.go b/internal/config/server_sftp.go similarity index 99% rename from internal/model/server_sftp.go rename to internal/config/server_sftp.go index f0569bed..bb64bc8b 100644 --- a/internal/model/server_sftp.go +++ b/internal/config/server_sftp.go @@ -1,4 +1,4 @@ -package model +package config import ( "time" diff --git a/internal/db/client.go b/internal/db/client.go index 9c92d7b0..8b575989 100644 --- a/internal/db/client.go +++ b/internal/db/client.go @@ -2,14 +2,14 @@ package db import ( "encoding/json" - "fmt" "os" "path" "strings" "time" - "github.com/crazy-max/ftpgrab/v7/internal/model" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/pkg/utl" + "github.com/pkg/errors" "github.com/rs/zerolog/log" bolt "go.etcd.io/bbolt" ) @@ -17,7 +17,7 @@ import ( // Client represents an active db object type Client struct { *bolt.DB - cfg *model.Db + cfg *config.Db bucket string } @@ -28,7 +28,7 @@ type entry struct { } // New creates new db instance -func New(cfg *model.Db) (c *Client, err error) { +func New(cfg *config.Db) (c *Client, err error) { var db *bolt.DB var bucket = "ftpgrab" @@ -59,7 +59,7 @@ func New(cfg *model.Db) (c *Client, err error) { log.Debug().Msgf("%d entries found in database", stats.KeyN) return nil }); err != nil { - return nil, fmt.Errorf("cannot count entries in database, %v", err) + return nil, errors.Wrap(err, "Cannot count entries in database") } return &Client{db, cfg, bucket}, nil diff --git a/internal/grabber/file.go b/internal/grabber/file.go new file mode 100644 index 00000000..e57d5f8f --- /dev/null +++ b/internal/grabber/file.go @@ -0,0 +1,69 @@ +package grabber + +import ( + "os" + "path" + + "github.com/rs/zerolog/log" +) + +// File represents a file to grab +type File struct { + Base string + SrcDir string + DestDir string + Info os.FileInfo +} + +func (c *Client) ListFiles() []File { + var files []File + + // Iterate sources + for _, src := range c.server.Common().Sources { + log.Debug().Str("source", src).Msg("Listing files") + + // Check basedir + dest := c.config.Output + if src != "/" && *c.config.CreateBaseDir { + dest = path.Join(dest, src) + } + + files = append(files, c.readDir(src, src, dest)...) + } + + return files +} + +func (c *Client) readDir(base string, srcdir string, destdir string) []File { + var files []File + + items, err := c.server.ReadDir(srcdir) + if err != nil { + log.Error().Err(err).Str("source", base).Msgf("Cannot read directory %s", srcdir) + return []File{} + } + + for _, item := range items { + files = append(files, c.readFile(base, srcdir, destdir, item)...) + } + + return files +} + +func (c *Client) readFile(base string, srcdir string, destdir string, file os.FileInfo) []File { + srcfile := path.Join(srcdir, file.Name()) + destfile := path.Join(destdir, file.Name()) + + if file.IsDir() { + return c.readDir(base, srcfile, destfile) + } + + return []File{ + { + Base: base, + SrcDir: srcdir, + DestDir: destdir, + Info: file, + }, + } +} diff --git a/internal/grabber/grabber.go b/internal/grabber/grabber.go new file mode 100644 index 00000000..e0562c91 --- /dev/null +++ b/internal/grabber/grabber.go @@ -0,0 +1,238 @@ +package grabber + +import ( + "fmt" + "os" + "path" + "runtime" + "time" + + "github.com/crazy-max/ftpgrab/v7/internal/config" + "github.com/crazy-max/ftpgrab/v7/internal/db" + "github.com/crazy-max/ftpgrab/v7/internal/journal" + "github.com/crazy-max/ftpgrab/v7/internal/server" + "github.com/crazy-max/ftpgrab/v7/internal/server/ftp" + "github.com/crazy-max/ftpgrab/v7/internal/server/sftp" + "github.com/crazy-max/ftpgrab/v7/pkg/utl" + "github.com/docker/go-units" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +// Client represents an active grabber object +type Client struct { + config *config.Download + db *db.Client + server *server.Client +} + +// New creates new grabber instance +func New(dlConfig *config.Download, dbConfig *config.Db, serverConfig *config.Server) (*Client, error) { + var dbCli *db.Client + var serverCli *server.Client + var err error + + // DB client + if dbCli, err = db.New(dbConfig); err != nil { + return nil, errors.Wrap(err, "Cannot open database") + } + + // Server client + if serverConfig.FTP != nil { + serverCli, err = ftp.New(serverConfig.FTP) + } else if serverConfig.SFTP != nil { + serverCli, err = sftp.New(serverConfig.SFTP) + } else { + return nil, errors.New("No server defined") + } + if err != nil { + return nil, errors.Wrap(err, "Cannot connect to server") + } + + return &Client{ + config: dlConfig, + db: dbCli, + server: serverCli, + }, nil +} + +func (c *Client) Grab(files []File) journal.Journal { + jnl := journal.New() + jnl.ServerHost = c.server.Common().Host + + for _, file := range files { + if entry := c.download(file, 0); entry != nil { + jnl.Add(*entry) + } + } + + return jnl.Journal +} + +func (c *Client) download(file File, retry int) *journal.Entry { + srcpath := path.Join(file.SrcDir, file.Info.Name()) + destpath := path.Join(file.DestDir, file.Info.Name()) + + entry := &journal.Entry{ + File: srcpath, + Status: c.getStatus(file), + } + + sublogger := log.With(). + Str("file", entry.File). + Str("size", units.HumanSize(float64(file.Info.Size()))). + Logger() + + if entry.Status == journal.EntryStatusAlreadyDl && !c.db.HasHash(file.Base, file.SrcDir, file.Info) { + if err := c.db.PutHash(file.Base, file.SrcDir, file.Info); err != nil { + sublogger.Error().Err(err).Msg("Cannot add hash into db") + entry.Level = journal.EntryLevelWarning + entry.Text = fmt.Sprintf("Already downloaded but cannot add hash into db: %v", err) + return entry + } + } + + if entry.Status.IsSkipped() { + if !*c.config.HideSkipped { + sublogger.Warn().Msgf("Skipped (%s)", entry.Status) + entry.Level = journal.EntryLevelSkip + return entry + } + return nil + } + + retrieveStart := time.Now() + sublogger.Info().Str("dest", destpath).Msg("Downloading...") + + destfolder := path.Dir(destpath) + if err := os.MkdirAll(destfolder, os.ModePerm); err != nil { + sublogger.Error().Err(err).Msg("Cannot create destination dir") + entry.Level = journal.EntryLevelError + entry.Text = fmt.Sprintf("Cannot create destination dir: %v", err) + return entry + } + if err := c.fixPerms(destfolder); err != nil { + sublogger.Warn().Err(err).Msg("Cannot fix parent folder permissions") + } + + destfile, err := os.Create(destpath) + if err != nil { + sublogger.Error().Err(err).Msg("Cannot create destination file") + entry.Level = journal.EntryLevelError + entry.Text = fmt.Sprintf("Cannot create destination file: %v", err) + return entry + } + + err = c.server.Retrieve(srcpath, destfile) + if err != nil { + retry++ + sublogger.Error().Err(err).Msgf("Error downloading, retry %d/%d", retry, c.config.Retry) + if retry == c.config.Retry { + sublogger.Error().Err(err).Msg("Cannot download file") + entry.Level = journal.EntryLevelError + entry.Text = fmt.Sprintf("Cannot download file: %v", err) + } else { + return c.download(file, retry) + } + } else { + sublogger.Info(). + Str("duration", time.Since(retrieveStart).Round(time.Millisecond).String()). + Msg("File successfully downloaded") + entry.Level = journal.EntryLevelSuccess + entry.Text = fmt.Sprintf("%s successfully downloaded in %s", + units.HumanSize(float64(file.Info.Size())), + time.Since(retrieveStart).Round(time.Millisecond).String(), + ) + if err := c.fixPerms(destpath); err != nil { + sublogger.Warn().Err(err).Msg("Cannot fix file permissions") + } + if err := c.db.PutHash(file.Base, file.SrcDir, file.Info); err != nil { + sublogger.Error().Err(err).Msg("Cannot add hash into db") + entry.Level = journal.EntryLevelWarning + entry.Text = fmt.Sprintf("Successfully downloaded but cannot add hash into db: %v", err) + } + if err = os.Chtimes(destpath, file.Info.ModTime(), file.Info.ModTime()); err != nil { + sublogger.Warn().Err(err).Msg("Cannot change modtime of destination file") + } + } + + return entry +} + +func (c *Client) getStatus(file File) journal.EntryStatus { + if !c.isIncluded(file) { + return journal.EntryStatusNotIncluded + } else if c.isExcluded(file) { + return journal.EntryStatusExcluded + } else if file.Info.ModTime().Before(c.config.SinceTime) { + return journal.EntryStatusOutdated + } else if destfile, err := os.Stat(path.Join(file.DestDir, file.Info.Name())); err == nil { + if destfile.Size() == file.Info.Size() { + return journal.EntryStatusAlreadyDl + } + return journal.EntryStatusSizeDiff + } else if c.db.HasHash(file.Base, file.SrcDir, file.Info) { + return journal.EntryStatusHashExists + } + return journal.EntryStatusNeverDl +} + +func (c *Client) isIncluded(file File) bool { + if len(c.config.Include) == 0 { + return true + } + for _, include := range c.config.Include { + if utl.MatchString(include, file.Info.Name()) { + return true + } + } + return false +} + +func (c *Client) isExcluded(file File) bool { + if len(c.config.Exclude) == 0 { + return false + } + for _, exclude := range c.config.Exclude { + if utl.MatchString(exclude, file.Info.Name()) { + return true + } + } + return false +} + +func (c *Client) fixPerms(filepath string) error { + if runtime.GOOS == "windows" { + return nil + } + + fileinfo, err := os.Stat(filepath) + if err != nil { + return err + } + + chmod := os.FileMode(c.config.ChmodFile) + if fileinfo.IsDir() { + chmod = os.FileMode(c.config.ChmodDir) + } + + if err := os.Chmod(filepath, chmod); err != nil { + return err + } + + if err := os.Chown(filepath, c.config.UID, c.config.GID); err != nil { + return err + } + + return nil +} + +// Close closes grabber +func (c *Client) Close() { + if err := c.db.Close(); err != nil { + log.Warn().Err(err).Msg("Cannot close database") + } + if err := c.server.Close(); err != nil { + log.Warn().Err(err).Msg("Cannot close server connection") + } +} diff --git a/internal/journal/client.go b/internal/journal/client.go index 7730bcca..9135dc49 100644 --- a/internal/journal/client.go +++ b/internal/journal/client.go @@ -1,32 +1,29 @@ package journal -import ( - "github.com/crazy-max/ftpgrab/v7/internal/model" -) - // Client represents an active journal object type Client struct { - model.Journal + Journal } // New creates new journal instance func New() *Client { - return &Client{model.Journal{}} + return &Client{Journal{}} } -// AddEntry adds an entry in the journal -func (c *Client) AddEntry(entry model.Entry) { +// Add adds an entry in the journal +func (c *Client) Add(entry Entry) { c.Entries = append(c.Entries, entry) - if entry.StatusType == "error" { + switch entry.Level { + case EntryLevelError: c.Count.Error++ - } else if entry.StatusType == "skip" { + case EntryLevelSkip: c.Count.Skip++ - } else if entry.StatusType == "success" { + case EntryLevelSuccess: c.Count.Success++ } } -// IsEmpty verifies if journal is empty +// IsEmpty checks if journal is empty func (c *Client) IsEmpty() bool { - return c.Count.Error == 0 && c.Count.Success == 0 + return c.Entries == nil || len(c.Entries) == 0 } diff --git a/internal/journal/entry.go b/internal/journal/entry.go new file mode 100644 index 00000000..b38135a3 --- /dev/null +++ b/internal/journal/entry.go @@ -0,0 +1,40 @@ +package journal + +// Entry represents a journal entry +type Entry struct { + File string `json:"file,omitempty"` + Status EntryStatus `json:"status,omitempty"` + Level EntryLevel `json:"level,omitempty"` + Text string `json:"text,omitempty"` +} + +// EntryLevel represents an entry kevek +type EntryLevel string + +const ( + EntryLevelError = EntryLevel("error") + EntryLevelWarning = EntryLevel("warning") + EntryLevelSkip = EntryLevel("skip") + EntryLevelSuccess = EntryLevel("success") +) + +// EntryStatus represents entry status +type EntryStatus string + +const ( + EntryStatusOutdated = EntryStatus("Outdated file") + EntryStatusNotIncluded = EntryStatus("Not included") + EntryStatusExcluded = EntryStatus("Excluded") + EntryStatusNeverDl = EntryStatus("Never downloaded") + EntryStatusAlreadyDl = EntryStatus("Already downloaded") + EntryStatusSizeDiff = EntryStatus("Exists but size is different") + EntryStatusHashExists = EntryStatus("Hash sum exists") +) + +func (es *EntryStatus) IsSkipped() bool { + return *es == EntryStatusAlreadyDl || + *es == EntryStatusHashExists || + *es == EntryStatusOutdated || + *es == EntryStatusNotIncluded || + *es == EntryStatusExcluded +} diff --git a/internal/model/journal.go b/internal/journal/journal.go similarity index 67% rename from internal/model/journal.go rename to internal/journal/journal.go index b9e9bc6e..16b9d32c 100644 --- a/internal/model/journal.go +++ b/internal/journal/journal.go @@ -1,4 +1,4 @@ -package model +package journal import ( "encoding/json" @@ -7,7 +7,7 @@ import ( "github.com/hako/durafmt" ) -// Journal holds ftpgrab entries and status +// Journal holds journal entries type Journal struct { ServerHost string `json:"-"` Entries []Entry `json:"entries,omitempty"` @@ -20,16 +20,6 @@ type Journal struct { Duration time.Duration `json:"duration,omitempty"` } -// Entry represents a journal entry -type Entry struct { - File string `json:"file,omitempty"` - StatusType string `json:"status_type,omitempty"` - StatusText string `json:"status_text,omitempty"` -} - -// EntryStatus represents entry status -type EntryStatus string - func (j Journal) MarshalJSON() ([]byte, error) { type Alias Journal return json.Marshal(&struct { @@ -40,3 +30,8 @@ func (j Journal) MarshalJSON() ([]byte, error) { Duration: durafmt.ParseShort(j.Duration).String(), }) } + +// IsEmpty checks if journal is empty +func (j Journal) IsEmpty() bool { + return j.Entries == nil || len(j.Entries) == 0 +} diff --git a/internal/logging/logger.go b/internal/logging/logger.go index 891f20e9..2f1cb0cf 100644 --- a/internal/logging/logger.go +++ b/internal/logging/logger.go @@ -8,14 +8,14 @@ import ( "syscall" "time" - "github.com/crazy-max/ftpgrab/v7/internal/model" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/ilya1st/rotatewriter" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) // Configure configures logger -func Configure(cli *model.Cli, location *time.Location) { +func Configure(cli config.Cli, location *time.Location) { var err error var w io.Writer diff --git a/internal/notif/client.go b/internal/notif/client.go index 8ce211c0..833612d8 100644 --- a/internal/notif/client.go +++ b/internal/notif/client.go @@ -1,8 +1,8 @@ package notif import ( + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/journal" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/internal/notif/mail" "github.com/crazy-max/ftpgrab/v7/internal/notif/notifier" "github.com/crazy-max/ftpgrab/v7/internal/notif/slack" @@ -10,35 +10,35 @@ import ( "github.com/rs/zerolog/log" ) -// Client represents an active webhook notification object +// Client represents an active notification object type Client struct { - cfg *model.Notif - meta model.Meta + cfg *config.Notif + meta config.Meta notifiers []notifier.Notifier } // New creates a new notification instance -func New(config *model.Notif, meta model.Meta) (*Client, error) { +func New(cfg *config.Notif, meta config.Meta) (*Client, error) { var c = &Client{ - cfg: config, + cfg: cfg, meta: meta, notifiers: []notifier.Notifier{}, } - if config == nil { + if cfg == nil { log.Warn().Msg("No notifier available") return c, nil } // Add notifiers - if config.Mail != nil { - c.notifiers = append(c.notifiers, mail.New(config.Mail, meta)) + if cfg.Mail != nil { + c.notifiers = append(c.notifiers, mail.New(cfg.Mail, meta)) } - if config.Slack != nil { - c.notifiers = append(c.notifiers, slack.New(config.Slack, meta)) + if cfg.Slack != nil { + c.notifiers = append(c.notifiers, slack.New(cfg.Slack, meta)) } - if config.Webhook != nil { - c.notifiers = append(c.notifiers, webhook.New(config.Webhook, meta)) + if cfg.Webhook != nil { + c.notifiers = append(c.notifiers, webhook.New(cfg.Webhook, meta)) } log.Debug().Msgf("%d notifier(s) created", len(c.notifiers)) @@ -46,7 +46,7 @@ func New(config *model.Notif, meta model.Meta) (*Client, error) { } // Send creates and sends notifications to notifiers -func (c *Client) Send(jnl journal.Client) { +func (c *Client) Send(jnl journal.Journal) { for _, n := range c.notifiers { log.Debug().Msgf("Sending %s notification...", n.Name()) if err := n.Send(jnl); err != nil { diff --git a/internal/notif/mail/client.go b/internal/notif/mail/client.go index cece5f19..72921326 100644 --- a/internal/notif/mail/client.go +++ b/internal/notif/mail/client.go @@ -5,28 +5,29 @@ import ( "fmt" "time" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/journal" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/internal/notif/notifier" "github.com/crazy-max/ftpgrab/v7/pkg/utl" "github.com/go-gomail/gomail" "github.com/hako/durafmt" "github.com/matcornic/hermes/v2" + "github.com/pkg/errors" "github.com/rs/zerolog/log" ) // Client represents an active mail notification object type Client struct { *notifier.Notifier - cfg *model.NotifMail - meta model.Meta + cfg *config.NotifMail + meta config.Meta } // New creates a new mail notification instance -func New(config *model.NotifMail, meta model.Meta) notifier.Notifier { +func New(cfg *config.NotifMail, meta config.Meta) notifier.Notifier { return notifier.Notifier{ Handler: &Client{ - cfg: config, + cfg: cfg, meta: meta, }, } @@ -38,7 +39,7 @@ func (c *Client) Name() string { } // Send creates and sends an email notification with journal entries -func (c *Client) Send(jnl journal.Client) error { +func (c *Client) Send(jnl journal.Journal) error { h := hermes.Hermes{ Theme: new(Theme), Product: hermes.Product{ @@ -56,8 +57,8 @@ func (c *Client) Send(jnl journal.Client) error { var entriesData [][]hermes.Entry for _, entry := range jnl.Entries { entriesData = append(entriesData, []hermes.Entry{ - {Key: "Status", Value: entry.StatusType}, - {Key: "Info", Value: string(entry.StatusText)}, + {Key: "Status", Value: string(entry.Level)}, + {Key: "Info", Value: entry.Text}, {Key: "File", Value: entry.File}, }) } @@ -90,13 +91,13 @@ func (c *Client) Send(jnl journal.Client) error { // Generate an HTML email with the provided contents (for modern clients) htmlpart, err := h.GenerateHTML(email) if err != nil { - return fmt.Errorf("hermes: %v", err) + return errors.Wrap(err, "Cannot generate HTML content for email notification") } // Generate the plaintext version of the e-mail (for clients that do not support xHTML) textpart, err := h.GeneratePlainText(email) if err != nil { - return fmt.Errorf("hermes: %v", err) + return errors.Wrap(err, "Cannot generate plaintext content for email notification") } msg := gomail.NewMessage() diff --git a/internal/notif/notifier/notifier.go b/internal/notif/notifier/notifier.go index 2da54391..68a1d6e7 100644 --- a/internal/notif/notifier/notifier.go +++ b/internal/notif/notifier/notifier.go @@ -5,7 +5,7 @@ import "github.com/crazy-max/ftpgrab/v7/internal/journal" // Handler is a notifier interface type Handler interface { Name() string - Send(jnl journal.Client) error + Send(jnl journal.Journal) error } // Notifier represents an active notifier object diff --git a/internal/notif/slack/slack.go b/internal/notif/slack/slack.go index 58dc9ee8..64473ec5 100644 --- a/internal/notif/slack/slack.go +++ b/internal/notif/slack/slack.go @@ -8,8 +8,8 @@ import ( "strconv" "time" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/journal" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/internal/notif/notifier" "github.com/hako/durafmt" "github.com/nlopes/slack" @@ -18,15 +18,15 @@ import ( // Client represents an active slack notification object type Client struct { *notifier.Notifier - cfg *model.NotifSlack - meta model.Meta + cfg *config.NotifSlack + meta config.Meta } // New creates a new slack notification instance -func New(config *model.NotifSlack, meta model.Meta) notifier.Notifier { +func New(cfg *config.NotifSlack, meta config.Meta) notifier.Notifier { return notifier.Notifier{ Handler: &Client{ - cfg: config, + cfg: cfg, meta: meta, }, } @@ -38,7 +38,7 @@ func (c *Client) Name() string { } // Send creates and sends a slack notification with journal entries -func (c *Client) Send(jnl journal.Client) error { +func (c *Client) Send(jnl journal.Journal) error { var textBuf bytes.Buffer textTpl := template.Must(template.New("text").Parse("FTPGrab has successfully downloaded *{{ .Success }}* files in *{{ .Duration }}*.\n*{{ .Skip }}* have been skipped and *{{ .Error }}* errors occurred.")) if err := textTpl.Execute(&textBuf, struct { diff --git a/internal/notif/webhook/client.go b/internal/notif/webhook/client.go index 6bd99605..4516567d 100644 --- a/internal/notif/webhook/client.go +++ b/internal/notif/webhook/client.go @@ -5,20 +5,20 @@ import ( "encoding/json" "net/http" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/journal" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/internal/notif/notifier" ) // Client represents an active webhook notification object type Client struct { *notifier.Notifier - cfg *model.NotifWebhook - meta model.Meta + cfg *config.NotifWebhook + meta config.Meta } // New creates a new webhook notification instance -func New(config *model.NotifWebhook, meta model.Meta) notifier.Notifier { +func New(config *config.NotifWebhook, meta config.Meta) notifier.Notifier { return notifier.Notifier{ Handler: &Client{ cfg: config, @@ -33,21 +33,21 @@ func (c *Client) Name() string { } // Send creates and sends a webhook notification with journal entries -func (c *Client) Send(jnl journal.Client) error { +func (c *Client) Send(jnl journal.Journal) error { hc := http.Client{ Timeout: *c.cfg.Timeout, } body, err := json.Marshal(struct { - Version string `json:"ftpgrab_version,omitempty"` - ServerIP string `json:"server_ip,omitempty"` - Dest string `json:"dest_hostname,omitempty"` - Journal model.Journal `json:"journal,omitempty"` + Version string `json:"ftpgrab_version,omitempty"` + ServerIP string `json:"server_ip,omitempty"` + Dest string `json:"dest_hostname,omitempty"` + Journal journal.Journal `json:"journal,omitempty"` }{ Version: c.meta.Version, ServerIP: jnl.ServerHost, Dest: c.meta.Hostname, - Journal: jnl.Journal, + Journal: jnl, }) if err != nil { return err diff --git a/internal/server/client.go b/internal/server/client.go index fa63b5e0..57bf726a 100644 --- a/internal/server/client.go +++ b/internal/server/client.go @@ -4,12 +4,12 @@ import ( "io" "os" - "github.com/crazy-max/ftpgrab/v7/internal/model" + "github.com/crazy-max/ftpgrab/v7/internal/config" ) // Handler is a server interface type Handler interface { - Common() model.ServerCommon + Common() config.ServerCommon ReadDir(source string) ([]os.FileInfo, error) Retrieve(path string, dest io.Writer) error Close() error diff --git a/internal/server/ftp/client.go b/internal/server/ftp/client.go index 7be89763..928ed89a 100644 --- a/internal/server/ftp/client.go +++ b/internal/server/ftp/client.go @@ -7,8 +7,8 @@ import ( "os" "regexp" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/logging" - "github.com/crazy-max/ftpgrab/v7/internal/model" "github.com/crazy-max/ftpgrab/v7/internal/server" "github.com/crazy-max/ftpgrab/v7/pkg/utl" "github.com/jlaffaye/ftp" @@ -18,39 +18,39 @@ import ( // Client represents an active ftp object type Client struct { *server.Client - config *model.ServerFTP - ftp *ftp.ServerConn + cfg *config.ServerFTP + ftp *ftp.ServerConn } // New creates new ftp instance -func New(config *model.ServerFTP) (*server.Client, error) { +func New(cfg *config.ServerFTP) (*server.Client, error) { var err error - var client = &Client{config: config} + var client = &Client{cfg: cfg} ftpConfig := []ftp.DialOption{ - ftp.DialWithTimeout(*config.Timeout), - ftp.DialWithDisabledEPSV(*config.DisableEPSV), + ftp.DialWithTimeout(*cfg.Timeout), + ftp.DialWithDisabledEPSV(*cfg.DisableEPSV), ftp.DialWithDebugOutput(&logging.FtpWriter{ - Enabled: *config.LogTrace, + Enabled: *cfg.LogTrace, }), } - if *config.TLS { + if *cfg.TLS { ftpConfig = append(ftpConfig, ftp.DialWithTLS(&tls.Config{ - ServerName: config.Host, - InsecureSkipVerify: *config.InsecureSkipVerify, + ServerName: cfg.Host, + InsecureSkipVerify: *cfg.InsecureSkipVerify, })) } - if client.ftp, err = ftp.Dial(fmt.Sprintf("%s:%d", config.Host, config.Port), ftpConfig...); err != nil { + if client.ftp, err = ftp.Dial(fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), ftpConfig...); err != nil { return nil, err } - username, err := utl.GetSecret(config.Username, config.UsernameFile) + username, err := utl.GetSecret(cfg.Username, cfg.UsernameFile) if err != nil { log.Warn().Err(err).Msg("Cannot retrieve username secret for ftp server") } - password, err := utl.GetSecret(config.Password, config.PasswordFile) + password, err := utl.GetSecret(cfg.Password, cfg.PasswordFile) if err != nil { log.Warn().Err(err).Msg("Cannot retrieve password secret for ftp server") } @@ -65,11 +65,11 @@ func New(config *model.ServerFTP) (*server.Client, error) { } // Common return common configuration -func (c *Client) Common() model.ServerCommon { - return model.ServerCommon{ - Host: c.config.Host, - Port: c.config.Port, - Sources: c.config.Sources, +func (c *Client) Common() config.ServerCommon { + return config.ServerCommon{ + Host: c.cfg.Host, + Port: c.cfg.Port, + Sources: c.cfg.Sources, } } diff --git a/internal/server/sftp/client.go b/internal/server/sftp/client.go index cd6a02ef..8397e410 100644 --- a/internal/server/sftp/client.go +++ b/internal/server/sftp/client.go @@ -6,9 +6,10 @@ import ( "io/ioutil" "os" - "github.com/crazy-max/ftpgrab/v7/internal/model" + "github.com/crazy-max/ftpgrab/v7/internal/config" "github.com/crazy-max/ftpgrab/v7/internal/server" "github.com/crazy-max/ftpgrab/v7/pkg/utl" + "github.com/pkg/errors" "github.com/pkg/sftp" "github.com/rs/zerolog/log" "golang.org/x/crypto/ssh" @@ -17,13 +18,13 @@ import ( // Client represents an active sftp object type Client struct { *server.Client - config *model.ServerSFTP + config *config.ServerSFTP sftp *sftp.Client ssh *ssh.Client } // New creates new ftp instance -func New(config *model.ServerSFTP) (*server.Client, error) { +func New(config *config.ServerSFTP) (*server.Client, error) { var err error var client = &Client{config: config} var sshConf *ssh.ClientConfig @@ -36,7 +37,7 @@ func New(config *model.ServerSFTP) (*server.Client, error) { log.Warn().Err(err).Msg("Cannot retrieve key passphrase secret for sftp server") } if sshAuth, err = client.readPublicKey(config.KeyFile, keyPassphrase); err != nil { - return nil, fmt.Errorf("unable to read SFTP public key, %v", err) + return nil, errors.Wrap(err, "Unable to read SFTP public key") } } else if len(config.Password) > 0 || len(config.PasswordFile) > 0 { password, err := utl.GetSecret(config.Password, config.PasswordFile) @@ -63,7 +64,7 @@ func New(config *model.ServerSFTP) (*server.Client, error) { sshConf.SetDefaults() client.ssh, err = ssh.Dial("tcp", fmt.Sprintf("%s:%d", config.Host, config.Port), sshConf) if err != nil { - return nil, fmt.Errorf("cannot open ssh connection, %v", err) + return nil, errors.Wrap(err, "Cannot open ssh connection") } if client.sftp, err = sftp.NewClient(client.ssh, sftp.MaxPacket(config.MaxPacketSize)); err != nil { @@ -74,8 +75,8 @@ func New(config *model.ServerSFTP) (*server.Client, error) { } // Common return common configuration -func (c *Client) Common() model.ServerCommon { - return model.ServerCommon{ +func (c *Client) Common() config.ServerCommon { + return config.ServerCommon{ Host: c.config.Host, Port: c.config.Port, Sources: c.config.Sources,