Skip to content

Commit

Permalink
Config: Allow scheduling of indexing and backup tasks #2495 #2608 #4243
Browse files Browse the repository at this point in the history
Note that this is "bleeding edge" functionality and that the newly added
config option PHOTOPRISM_BACKUP_RETAIN can be set, but does not have any
effect yet. Feedback welcome!

Signed-off-by: Michael Mayer <[email protected]>
  • Loading branch information
lastzero committed May 11, 2024
1 parent 6a5826e commit 0e7c91f
Show file tree
Hide file tree
Showing 41 changed files with 553 additions and 197 deletions.
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ require golang.org/x/text v0.15.0

require (
github.com/davidbyttow/govips/v2 v2.14.0
github.com/go-co-op/gocron/v2 v2.5.0
github.com/pquerna/otp v1.4.0
github.com/robfig/cron/v3 v3.0.1
)

require (
Expand All @@ -111,6 +113,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mandykoh/go-parallel v0.1.0 // indirect
Expand All @@ -126,7 +129,7 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/zitadel/logging v0.5.0 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sys v0.20.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
Expand Down
15 changes: 12 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-co-op/gocron/v2 v2.5.0 h1:ff/TJX9GdTJBDL1il9cyd/Sj3WnS+BB7ZzwHKSNL5p8=
github.com/go-co-op/gocron/v2 v2.5.0/go.mod h1:ckPQw96ZuZLRUGu88vVpd9a6d9HakI14KWahFZtGvNw=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
Expand Down Expand Up @@ -244,6 +246,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
Expand All @@ -262,8 +266,9 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leandro-lugaresi/hub v1.1.1 h1:zqp0HzFvj4HtqjMBXM2QF17o6PNmR8MJOChgeKl/aw8=
github.com/leandro-lugaresi/hub v1.1.1/go.mod h1:XEFWanhHv6Rt3XlteHMxuNDYi8dJcpJjodpqkU+BtIo=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
Expand Down Expand Up @@ -327,6 +332,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg=
github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
Expand Down Expand Up @@ -386,6 +393,8 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
Expand Down Expand Up @@ -414,8 +423,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
Expand Down
4 changes: 2 additions & 2 deletions internal/auto/auto.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func Start(conf *config.Config) {
}()
}

// Stop stops waiting for indexing & importing opportunities.
func Stop() {
// Shutdown the auto indexing watchers.
func Shutdown() {
stop <- true
}
2 changes: 1 addition & 1 deletion internal/auto/auto_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,5 @@ func TestStart(t *testing.T) {
ResetImport()
ResetIndex()

Stop()
Shutdown()
}
4 changes: 2 additions & 2 deletions internal/auto/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@ func mustImport(delay time.Duration) bool {
importMutex.Lock()
defer importMutex.Unlock()

return !autoImport.IsZero() && autoImport.Sub(time.Now()) < -1*delay && !mutex.MainWorker.Running()
return !autoImport.IsZero() && autoImport.Sub(time.Now()) < -1*delay && !mutex.IndexWorker.Running()
}

// Import starts importing originals e.g. after WebDAV uploads.
func Import() error {
if mutex.MainWorker.Running() {
if mutex.IndexWorker.Running() {
return nil
}

Expand Down
74 changes: 6 additions & 68 deletions internal/auto/index.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
package auto

import (
"path/filepath"
"sync"
"time"

"github.com/photoprism/photoprism/internal/api"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/get"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/pkg/i18n"
"github.com/photoprism/photoprism/internal/workers"
)

var autoIndex = time.Time{}
Expand Down Expand Up @@ -42,79 +39,20 @@ func mustIndex(delay time.Duration) bool {
indexMutex.Lock()
defer indexMutex.Unlock()

return !autoIndex.IsZero() && autoIndex.Sub(time.Now()) < -1*delay && !mutex.MainWorker.Running()
return !autoIndex.IsZero() && autoIndex.Sub(time.Now()) < -1*delay && !mutex.IndexWorker.Running()
}

// Index starts indexing originals e.g. after WebDAV uploads.
func Index() error {
if mutex.MainWorker.Running() {
return nil
}

conf := get.Config()
settings := conf.Settings()

start := time.Now()

path := conf.OriginalsPath()

ind := get.Index()

convert := settings.Index.Convert && conf.SidecarWritable()
indOpt := photoprism.NewIndexOptions(entity.RootPath, false, convert, true, false, true)
indOpt.Action = photoprism.ActionAutoIndex

lastRun, lastFound := ind.LastRun()
found, indexed := ind.Start(indOpt)

if !lastRun.IsZero() && indexed == 0 && len(found) == lastFound {
func Index() (err error) {
if mutex.IndexWorker.Running() {
return nil
}

api.RemoveFromFolderCache(entity.RootOriginals)

prg := get.Purge()

prgOpt := photoprism.PurgeOptions{
Path: filepath.Clean(entity.RootPath),
Ignore: found,
Force: true,
}

if files, photos, updated, err := prg.Start(prgOpt); err != nil {
return err
} else if updated > 0 {
event.InfoMsg(i18n.MsgRemovedFilesAndPhotos, len(files), len(photos))
}

event.Publish("index.updating", event.Data{
"uid": indOpt.UID,
"action": indOpt.Action,
"step": "moments",
})

moments := get.Moments()

if err := moments.Start(); err != nil {
log.Warnf("moments: %s", err)
}

elapsed := int(time.Since(start).Seconds())

msg := i18n.Msg(i18n.MsgIndexingCompletedIn, elapsed)

event.Success(msg)

eventData := event.Data{
"uid": indOpt.UID,
"action": indOpt.Action,
"path": path,
"seconds": elapsed,
}

event.Publish("index.completed", eventData)
err = workers.NewIndex(get.Config()).Start()

api.UpdateClientConfig()

return nil
return err
}
2 changes: 1 addition & 1 deletion internal/commands/show_config_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func showConfigOptionsAction(ctx *cli.Context) error {
{Start: "PHOTOPRISM_ADMIN_PASSWORD", Title: "Authentication"},
{Start: "PHOTOPRISM_LOG_LEVEL", Title: "Logging"},
{Start: "PHOTOPRISM_CONFIG_PATH", Title: "Storage"},
{Start: "PHOTOPRISM_WORKERS", Title: "Index Workers"},
{Start: "PHOTOPRISM_INDEX_WORKERS, PHOTOPRISM_WORKERS", Title: "Index Workers"},
{Start: "PHOTOPRISM_READONLY", Title: "Feature Flags"},
{Start: "PHOTOPRISM_DEFAULT_LOCALE", Title: "Customization"},
{Start: "PHOTOPRISM_SITE_URL", Title: "Site Information"},
Expand Down
2 changes: 1 addition & 1 deletion internal/commands/show_config_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func showConfigYamlAction(ctx *cli.Context) error {
{Start: "AuthMode", Title: "Authentication"},
{Start: "LogLevel", Title: "Logging"},
{Start: "ConfigPath", Title: "Storage"},
{Start: "Workers", Title: "Index Workers"},
{Start: "IndexWorkers", Title: "Index Workers"},
{Start: "ReadOnly", Title: "Feature Flags"},
{Start: "DefaultTheme", Title: "Customization"},
{Start: "SiteUrl", Title: "Site Information"},
Expand Down
4 changes: 2 additions & 2 deletions internal/commands/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ func startAction(ctx *cli.Context) error {
sig := <-quit

// Stop all background activity.
auto.Stop()
workers.Stop()
auto.Shutdown()
workers.Shutdown()
session.Shutdown()
mutex.CancelAll()

Expand Down
26 changes: 19 additions & 7 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ import (
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/sqlite"

"github.com/klauspost/cpuid/v2"
"github.com/pbnjay/memory"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"

Expand Down Expand Up @@ -720,8 +720,8 @@ func (c *Config) Shutdown() {
}
}

// Workers returns the number of workers e.g. for indexing files.
func (c *Config) Workers() int {
// IndexWorkers returns the number of indexing workers.
func (c *Config) IndexWorkers() int {
// Use one worker on systems with less than the recommended amount of memory.
if TotalMem < RecommendedMem {
return 1
Expand All @@ -736,15 +736,15 @@ func (c *Config) Workers() int {
}

// Limit number of workers when using SQLite3 to avoid database locking issues.
if c.DatabaseDriver() == SQLite3 && (cores >= 8 && c.options.Workers <= 0 || c.options.Workers > 4) {
if c.DatabaseDriver() == SQLite3 && (cores >= 8 && c.options.IndexWorkers <= 0 || c.options.IndexWorkers > 4) {
return 4
}

// Return explicit value if set and not too large.
if c.options.Workers > runtime.NumCPU() {
if c.options.IndexWorkers > runtime.NumCPU() {
return runtime.NumCPU()
} else if c.options.Workers > 0 {
return c.options.Workers
} else if c.options.IndexWorkers > 0 {
return c.options.IndexWorkers
}

// Use half the available cores by default.
Expand All @@ -755,6 +755,18 @@ func (c *Config) Workers() int {
return 1
}

// IndexSchedule returns the indexing schedule in cron format, e.g. "0 */3 * * *" to start indexing every 3 hours.
func (c *Config) IndexSchedule() string {
if c.options.IndexSchedule == "" {
return ""
} else if _, err := cron.ParseStandard(c.options.IndexSchedule); err != nil {
log.Tracef("config: invalid auto indexing schedule (%s)", err)
return ""
}

return c.options.IndexSchedule
}

// WakeupInterval returns the duration between background worker runs
// required for face recognition and index maintenance(1-86400s).
func (c *Config) WakeupInterval() time.Duration {
Expand Down
56 changes: 56 additions & 0 deletions internal/config/config_backup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package config

import (
"path/filepath"

"github.com/robfig/cron/v3"

"github.com/photoprism/photoprism/pkg/fs"
)

const (
DefaultBackupSchedule = "0 12 * * *"
DefaultBackupRetain = 14
)

// BackupPath returns the backup storage path.
func (c *Config) BackupPath() string {
if fs.PathWritable(c.options.BackupPath) {
return fs.Abs(c.options.BackupPath)
}

return filepath.Join(c.StoragePath(), "backup")
}

// BackupIndex checks if index SQL database dumps should be created based on the configured schedule.
func (c *Config) BackupIndex() bool {
return c.options.BackupIndex
}

// BackupAlbums checks if album YAML file backups should be created based on the configured schedule.
func (c *Config) BackupAlbums() bool {
return c.options.BackupAlbums
}

// BackupRetain returns the maximum number of SQL database dumps to keep, or -1 to keep all.
func (c *Config) BackupRetain() int {
if c.options.BackupRetain == 0 {
return DefaultBackupRetain
} else if c.options.BackupRetain < -1 {
return -1
}

return c.options.BackupRetain
}

// BackupSchedule returns the backup schedule in cron format, e.g. "0 12 * * *" for daily at noon.
func (c *Config) BackupSchedule() string {
if c.options.BackupSchedule == "" {
return ""
} else if _, err := cron.ParseStandard(c.options.BackupSchedule); err != nil {
log.Tracef("config: invalid backup schedule (%s)", err)
return ""
}

return c.options.BackupSchedule
}
Loading

1 comment on commit 0e7c91f

@lastzero
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.