Merge pull request #8 from rbtr/movies

plugin redesign and movies support

.drone.yml

@@ -13,10 +13,10 @@ steps:
   commands:
   - make test
 
-# - name: lint
-#   image: golangci/golangci-lint
-#   commands:
-#   - golangci-lint run -v
+- name: lint
+  image: golangci/golangci-lint
+  commands:
+  - make lint
 
 - name: release
   image: golang:latest

.golangci.yml

@@ -4,11 +4,12 @@ run:
 linters:
   enable-all: true
   disable:
-    - wsl
-    - lll
-    - funlen
-    - godox
-    - gomnd
-    - gocognit
-    - gochecknoglobals
-    - gochecknoinits
+  - wsl
+  - lll
+  - funlen
+  - godox
+  - gomnd
+  - gocognit
+  - gochecknoglobals
+  - gochecknoinits
+  - maligned

.goreleaser.yml

@@ -0,0 +1,16 @@
+nfpms:
+- name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}'
+  bindir: "/usr/local/bin"
+  replacements:
+    amd64: x86_64
+  formats:
+  - deb
+  - rpm
+  empty_folders:
+  - /etc/pachinko
+changelog:
+  sort: asc
+  filters:
+    exclude:
+    - '^docs:'
+    - '^test:'

Makefile

@@ -4,6 +4,9 @@ MODULE = pachinko
 GCFLAGS = -gcflags "all=-trimpath=$(PWD)" -asmflags "all=-trimpath=$(PWD)"
 GO_BUILD_ENV_VARS := CGO_ENABLED=0 GOOS=linux GOARCH=amd64
 
+lint: ## lint
+	@golangci-lint run -v
+
 test: ## run tests
 	@go test ./...
 

README.md

@@ -8,7 +8,7 @@
 |_|
 
 modular, plugin based media sorter  
-written in go by @rbtr  
+created by @rbtr  
 ```
 ---
 
@@ -20,32 +20,60 @@ written in go by @rbtr
 
 
 ### what it is
-pachinko is a media sorter. it integrates with the tvdb and the moviedb to, given a directory of reasonably named mix media, organize that media into a clean hierarchal directory directory structure ideal for use in media servers like plex, kodi/xbmc, etc.
+pachinko is a media sorter. it integrates with the tvdb and the moviedb to, given a directory of reasonably named mix media, organize that media into a clean hierarchal directory structure ideal for use in media servers like plex, kodi/xbmc, etc.
 
 unlike some of the prior implementations of this idea, pachinko was designed from inception to be automation and container-friendly.  
 it has no heavy gui - configure it through the config file or via flags, then execute it and walk away.
 
-it is written in go so that it is approachable for anyone interested in contributing without sacrificing too much performance.  
-the plugin-style architecture makes the codebase clear and efficient.
+it is written in go so that it is approachable for anyone interested in contributing without sacrificing performance.  
+the plugin-style architecture keeps the core codebase clear and efficient.
 
 ### design
 
+#### plugins
 pachinko has a plugin based pipeline design. the base plugin types are:
 - input - add data from a datasource to the stream
 - processor - modify the datastream in-flight
-- output -  write data from the stream to a datastore
+- output - deal with the processed data by e.g. moving files from their original dir to their sorted path.
 
-these plugin types make pachinko very flexible. composing a plugin pipeline of any combination of plugins is possible.
+these base plugin types make pachinko flexible. composing a pipeline of many combination of plugins is possible.
 
 additionally there are subtypes of `processor` plugins:
-- extractor - parse data already present in the datastream to classify, clean, or add information to the data
-- decorator - use external datasources to add information to items in the datastream
+- preprocessor - parse data already present in the datastream to classify, clean, or add information to the data before main processor plugins run. the preprocessors make modifications to the datastream based only on the data already present in the objects in the datastream.
+- (intra)processor - the main working processors, where external datasources may be queried to enrich the datastream and significant modifications made. 
+- postprocessor - last chance to modify the datastream before it is sent to outputs, but after the rich data has been added. the postprocessors make final modifications that shouldn't be the responsibility of the intraprocessors but may depend on the data enrichments that those have added.
+
+they subtypes exist mainly to allow ordering of plugin flow. 
+
+#### datatypes
+pachinko currently supports these data types: 
+- tv and 
+- movie video files
+
+other datatypes planned include: images (and whatever you would like to contribute!)
+
+#### inputs
+pachinko currently supports these inputs: 
+- local filesystem (`path`). 
+
+other datastore types planned include : s3 (and whatever you would like to contribute!)
 
-they differ primarily in that decorators supplement the datastream from external sources, and extractors use only the extant data in the stream.
+#### outputs
+pachinko currently supports these outputs:
+- local filesystem (`path_mover`)
+- stdout (`logger`)
 
-pachinko currently supports two data types: tv and movie video files. other datatypes planned include: images (and whatever you would like to contribute!)
+#### processors
+pachinko has the following required processors:
+- categorizer (internal) 
 
-pachinko currently supports one input and one output datastore: local filesystem (aka `path`). other datastore types planned include : s3 (and whatever you would like to contribute!)
+pachinko has the following optional processors:
+- tv identifier (pre-tv)
+- movie identifier (pre-movie)
+- tvdb (intra-tvdb)
+- tmdb (intra-tmdb)
+- tv path solver (post-tv_path_solver)
+- movie path solver (post-movie_path_solver)
 
 ### how to run it
 pachinko is distributed as a container and as a cross-platform binary.  
@@ -69,7 +97,7 @@ dry-run: true
 log-level: debug
 inputs: []
 outputs: []
-processors: []
+processors: {}
 ```
 
 the full, current list of options is available by running `./pachinko config` on the commandline.  
@@ -114,4 +142,4 @@ read the full license terms [here](https://www.mozilla.org/en-US/MPL/2.0/FAQ/).
 
 created by @rbtr  
 inspired by the functionality and frustrating user experience of: [sorttv by cliffe](https://sourceforge.net/projects/sorttv/), filebot, tinymediamanager, and others  
-and the excellent architecture patterns of coredns, telegraf
+and the excellent architecture patterns of telegraf, caddy, coredns, and others

cmd/config.go

@@ -51,15 +51,14 @@ and the pachinko pipeline.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
 		log.SetLevel(log.TraceLevel)
-		cfg, err := config.LoadConfigCmdConfig()
+		cfg, err := config.LoadCmdConfig()
 		if err != nil {
 			log.Fatal(err)
 		}
 		if err := cfg.Validate(); err != nil {
 			log.Fatal(err)
 		}
-		pipelineCfg := &config.SortCmdConfig{RootConfig: cfg.RootConfig}
-
+		pipelineCfg := config.NewCmdSort()
 		if err := cfg.DefaultConfig(pipelineCfg); err != nil {
 			log.Fatal(err)
 		}

cmd/root.go

@@ -15,10 +15,12 @@ import (
 	"syscall"
 
 	homedir "github.com/mitchellh/go-homedir"
-	_ "github.com/rbtr/pachinko/plugin"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	// include plugins
+	_ "github.com/rbtr/pachinko/plugin"
 )
 
 var cfgFile string

cmd/sort.go

@@ -29,7 +29,7 @@ not do anything useful.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
 		log.SetLevel(log.TraceLevel)
-		cfg, err := config.LoadSortCmdConfig()
+		cfg, err := config.LoadCmdSort()
 		if err != nil {
 			log.Fatal(err)
 		}

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/cyruzin/golang-tmdb v1.2.1
 	github.com/lithammer/fuzzysearch v1.1.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.1.2

go.sum

@@ -24,6 +24,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cyruzin/golang-tmdb v1.2.1 h1:jrp2vjn12e+BgD9j0f7yHhXmvqSBEwcGQiexyfthGjw=
+github.com/cyruzin/golang-tmdb v1.2.1/go.mod h1:ka1Gufj/wBUkU4phN6Ixx3a+a4UDwnZjK4eXLCjxkkI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/config/config.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/rbtr/pachinko/internal/pipeline"
+	internalpre "github.com/rbtr/pachinko/internal/plugin/processor/pre"
 	"github.com/rbtr/pachinko/plugin/input"
 	"github.com/rbtr/pachinko/plugin/output"
 	"github.com/rbtr/pachinko/plugin/processor"
@@ -25,20 +26,20 @@ type RootConfig struct {
 	LogFormat string `mapstructure:"log-format"`
 }
 
-type ConfigCmdConfig struct {
+type CmdConfig struct {
 	RootConfig `mapstructure:",squash"`
-	Format     string   `mapstructure:"format"`
-	Inputs     []string `mapstructure:"inputs"`
-	Outputs    []string `mapstructure:"outputs"`
-	Processors []string `mapstructure:"processors"`
+	Format     string                      `mapstructure:"format"`
+	Inputs     []string                    `mapstructure:"inputs"`
+	Outputs    []string                    `mapstructure:"outputs"`
+	Processors map[processor.Type][]string `mapstructure:"processors"`
 }
 
-type SortCmdConfig struct {
+type CmdSort struct {
 	RootConfig `mapstructure:",squash"`
-	Pipeline   pipeline.Config          `mapstructure:"pipeline"`
-	Inputs     []map[string]interface{} `mapstructure:"inputs"`
-	Outputs    []map[string]interface{} `mapstructure:"outputs"`
-	Processors []map[string]interface{} `mapstructure:"processors"`
+	Pipeline   pipeline.Config                             `mapstructure:"pipeline"`
+	Inputs     []map[string]interface{}                    `mapstructure:"inputs"`
+	Outputs    []map[string]interface{}                    `mapstructure:"outputs"`
+	Processors map[processor.Type][]map[string]interface{} `mapstructure:"processors"`
 }
 
 func (c *RootConfig) configLogger() {
@@ -68,7 +69,7 @@ func (c *RootConfig) Validate() error {
 	return nil
 }
 
-func (c *SortCmdConfig) ConfigurePipeline(pipe *pipeline.Pipeline) error {
+func (c *CmdSort) ConfigurePipeline(pipe *pipeline.Pipeline) error {
 	if err := mapstructure.Decode(c.Pipeline, pipe); err != nil {
 		return err
 	}
@@ -102,34 +103,52 @@ func (c *SortCmdConfig) ConfigurePipeline(pipe *pipeline.Pipeline) error {
 		}
 	}
 
-	for _, p := range c.Processors {
-		if name, ok := p["name"]; ok {
-			if initializer, ok := processor.Registry[name.(string)]; ok {
-				plugin := initializer()
-				if err := mapstructure.Decode(p, plugin); err != nil {
-					return err
-				}
-				if err := plugin.Init(); err != nil {
-					return err
+	categorizer := internalpre.NewCategorizer()
+	if err := categorizer.Init(); err != nil {
+		return err
+	}
+	pipe.WithProcessors(categorizer)
+
+	for _, t := range processor.Types {
+		for _, p := range c.Processors[t] {
+			if name, ok := p["name"]; ok {
+				if initializer, ok := processor.Registry[t][name.(string)]; ok {
+					plugin := initializer()
+					if err := mapstructure.Decode(p, plugin); err != nil {
+						return err
+					}
+					if err := plugin.Init(); err != nil {
+						return err
+					}
+					pipe.WithProcessors(plugin)
 				}
-				pipe.WithProcessors(plugin)
 			}
 		}
 	}
 
 	return nil
 }
 
+func NewCmdSort() *CmdSort {
+	return &CmdSort{
+		Processors: map[processor.Type][]map[string]interface{}{
+			processor.Pre:   {},
+			processor.Intra: {},
+			processor.Post:  {},
+		},
+	}
+}
+
 // LoadConfig loadconfig
-func LoadSortCmdConfig() (*SortCmdConfig, error) {
-	cfg := &SortCmdConfig{}
+func LoadCmdSort() (*CmdSort, error) {
+	cfg := NewCmdSort()
 	viper.SetEnvKeyReplacer(strings.NewReplacer("_", "-"))
 	viper.AutomaticEnv()
 	err := viper.Unmarshal(cfg)
 	return cfg, err
 }
 
-func (c *ConfigCmdConfig) DefaultConfig(p *SortCmdConfig) error {
+func (c *CmdConfig) DefaultConfig(p *CmdSort) error {
 	if len(c.Inputs) == 0 && len(c.Outputs) == 0 && len(c.Processors) == 0 {
 		// no plugins specified, dump configs for them all
 		for k := range input.Registry {
@@ -138,8 +157,10 @@ func (c *ConfigCmdConfig) DefaultConfig(p *SortCmdConfig) error {
 		for k := range output.Registry {
 			c.Outputs = append(c.Outputs, k)
 		}
-		for k := range processor.Registry {
-			c.Processors = append(c.Processors, k)
+		for _, t := range []processor.Type{processor.Pre, processor.Post, processor.Intra} {
+			for k := range processor.Registry[t] {
+				c.Processors[t] = append(c.Processors[t], k)
+			}
 		}
 	}
 
@@ -167,23 +188,25 @@ func (c *ConfigCmdConfig) DefaultConfig(p *SortCmdConfig) error {
 		}
 	}
 
-	for _, name := range c.Processors {
-		log.Tracef("making default config for plugin %s", name)
-		if initializer, ok := processor.Registry[name]; ok {
-			var out map[string]interface{}
-			if err := mapstructure.Decode(initializer(), &out); err != nil {
-				return err
+	for t, names := range c.Processors {
+		for _, name := range names {
+			log.Tracef("making default config for plugin %s", name)
+			if initializer, ok := processor.Registry[t][name]; ok {
+				var out map[string]interface{}
+				if err := mapstructure.Decode(initializer(), &out); err != nil {
+					return err
+				}
+				out["name"] = name
+				p.Processors[t] = append(p.Processors[t], out)
 			}
-			out["name"] = name
-			p.Processors = append(p.Processors, out)
 		}
 	}
 
 	return nil
 }
 
-func LoadConfigCmdConfig() (*ConfigCmdConfig, error) {
-	cfg := &ConfigCmdConfig{}
+func LoadCmdConfig() (*CmdConfig, error) {
+	cfg := &CmdConfig{}
 	viper.SetEnvKeyReplacer(strings.NewReplacer("_", "-"))
 	viper.AutomaticEnv()
 	err := viper.Unmarshal(cfg)