Merge pull request #22 from coldbrewcloud/logging

Container logging support

VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

aws/client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/coldbrewcloud/coldbrew-cli/aws/ecs"
 	"github.com/coldbrewcloud/coldbrew-cli/aws/elb"
 	"github.com/coldbrewcloud/coldbrew-cli/aws/iam"
+	"github.com/coldbrewcloud/coldbrew-cli/aws/logs"
 	"github.com/coldbrewcloud/coldbrew-cli/aws/sns"
 )
 
@@ -24,6 +25,7 @@ type Client struct {
 	ecrClient         *ecr.Client
 	iamClient         *iam.Client
 	snsClient         *sns.Client
+	logsClient        *logs.Client
 }
 
 func NewClient(region, accessKey, secretKey string) *Client {
@@ -86,3 +88,10 @@ func (c *Client) SNS() *sns.Client {
 	}
 	return c.snsClient
 }
+
+func (c *Client) CloudWatchLogs() *logs.Client {
+	if c.logsClient == nil {
+		c.logsClient = logs.New(c.session, c.config)
+	}
+	return c.logsClient
+}

aws/consts.go

@@ -11,4 +11,12 @@ const (
 	AWSRegionAPSouthEast1 = "ap-southeast-1"
 	AWSRegionAPSouthEast2 = "ap-southeast-2"
 	AWSRegionSAEast1      = "sa-east-1"
+
+	ECSTaskDefinitionLogDriverJSONFile = "json-file"
+	ECSTaskDefinitionLogDriverAWSLogs  = "awslogs"
+	ECSTaskDefinitionLogDriverSyslog   = "syslog"
+	ECSTaskDefinitionLogDriverJournald = "journald"
+	ECSTaskDefinitionLogDriverGelf     = "gelf"
+	ECSTaskDefinitionLogDriverFluentd  = "fluentd"
+	ECSTaskDefinitionLogDriverSplunk   = "splunk"
 )

aws/ecs/client.go

@@ -71,7 +71,7 @@ func (c *Client) DeleteCluster(clusterName string) error {
 	return err
 }
 
-func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerName string, cpu, memory uint64, envs map[string]string, portMappings []PortMapping, cloudWatchLogs bool) (*_ecs.TaskDefinition, error) {
+func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerName string, cpu, memory uint64, envs map[string]string, portMappings []PortMapping, logDriver string, logDriverOptions map[string]string) (*_ecs.TaskDefinition, error) {
 	if taskDefinitionName == "" {
 		return nil, errors.New("taskDefinitionName is empty")
 	}
@@ -96,14 +96,10 @@ func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerNa
 		Family: _aws.String(taskDefinitionName),
 	}
 
-	// TODO: move this out of this function
-	if cloudWatchLogs {
+	if logDriver != "" {
 		params.ContainerDefinitions[0].LogConfiguration = &_ecs.LogConfiguration{
-			LogDriver: _aws.String(_ecs.LogDriverAwslogs),
-			Options: _aws.StringMap(map[string]string{
-				"awslogs-group":  "coldbrewcloud-deploy-logs",
-				"awslogs-region": c.awsRegion,
-			}),
+			LogDriver: _aws.String(logDriver),
+			Options:   _aws.StringMap(logDriverOptions),
 		}
 	}
 

aws/logs/client.go

@@ -0,0 +1,59 @@
+package logs
+
+import (
+	_aws "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	_logs "github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+)
+
+type Client struct {
+	svc       *_logs.CloudWatchLogs
+	awsRegion string
+}
+
+func New(session *session.Session, config *_aws.Config) *Client {
+	return &Client{
+		awsRegion: *config.Region,
+		svc:       _logs.New(session, config),
+	}
+}
+
+func (c *Client) CreateGroup(groupName string) error {
+	params := &_logs.CreateLogGroupInput{
+		LogGroupName: _aws.String(groupName),
+	}
+
+	_, err := c.svc.CreateLogGroup(params)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) ListGroups(groupNamePrefix string) ([]*_logs.LogGroup, error) {
+	var nextToken *string
+	groups := []*_logs.LogGroup{}
+
+	for {
+		params := &_logs.DescribeLogGroupsInput{
+			LogGroupNamePrefix: _aws.String(groupNamePrefix),
+			NextToken:          nextToken,
+		}
+
+		res, err := c.svc.DescribeLogGroups(params)
+		if err != nil {
+			return nil, err
+		}
+
+		groups = append(groups, res.LogGroups...)
+
+		if res.NextToken == nil {
+			break
+		} else {
+			nextToken = res.NextToken
+		}
+	}
+
+	return groups, nil
+}

commands/deploy/aws_ecs.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"math"
 
+	"github.com/coldbrewcloud/coldbrew-cli/aws"
 	"github.com/coldbrewcloud/coldbrew-cli/aws/ecs"
 	"github.com/coldbrewcloud/coldbrew-cli/console"
 	"github.com/coldbrewcloud/coldbrew-cli/core"
+	"github.com/coldbrewcloud/coldbrew-cli/utils"
 	"github.com/coldbrewcloud/coldbrew-cli/utils/conv"
 )
 
@@ -31,7 +33,30 @@ func (c *Command) updateECSTaskDefinition(dockerImageFullURI string) (string, er
 		return "", err
 	}
 	memory /= 1000 * 1000
-	useCloudWatchLogs := false
+
+	// logging
+	loggingDriver := conv.S(c.conf.Logging.Driver)
+	if c.conf.Logging.Options == nil {
+		c.conf.Logging.Options = make(map[string]string)
+	}
+	switch loggingDriver {
+	case aws.ECSTaskDefinitionLogDriverAWSLogs:
+		// test if group needs to be created
+		awsLogsGroupName, ok := c.conf.Logging.Options["awslogs-group"]
+		if !ok || utils.IsBlank(awsLogsGroupName) {
+			awsLogsGroupName = core.DefaultCloudWatchLogsGroupName(conv.S(c.conf.Name), conv.S(c.conf.ClusterName))
+			c.conf.Logging.Options["awslogs-group"] = awsLogsGroupName
+		}
+		if err := c.PrepareCloudWatchLogsGroup(awsLogsGroupName); err != nil {
+			return "", err
+		}
+
+		// assign region if not provided
+		awsLogsRegionName, ok := c.conf.Logging.Options["awslogs-region"]
+		if !ok || utils.IsBlank(awsLogsRegionName) {
+			c.conf.Logging.Options["awslogs-region"] = conv.S(c.globalFlags.AWSRegion)
+		}
+	}
 
 	console.UpdatingResource("Updating ECS Task Definition", ecsTaskDefinitionName, false)
 	ecsTaskDef, err := c.awsClient.ECS().UpdateTaskDefinition(
@@ -42,7 +67,7 @@ func (c *Command) updateECSTaskDefinition(dockerImageFullURI string) (string, er
 		memory,
 		c.conf.Env,
 		portMappings,
-		useCloudWatchLogs)
+		loggingDriver, c.conf.Logging.Options)
 	if err != nil {
 		return "", fmt.Errorf("Failed to update ECS Task Definition [%s]: %s", ecsTaskDefinitionName, err.Error())
 	}
@@ -136,3 +161,25 @@ func (c *Command) updateECSService(ecsClusterName, ecsServiceName, ecsTaskDefini
 
 	return nil
 }
+
+func (c *Command) PrepareCloudWatchLogsGroup(groupName string) error {
+	groups, err := c.awsClient.CloudWatchLogs().ListGroups(groupName)
+	if err != nil {
+		return fmt.Errorf("Failed to list CloudWatch Logs Group [%s]: %s", groupName, err.Error())
+	}
+
+	for _, group := range groups {
+		if conv.S(group.LogGroupName) == groupName {
+			// log group exists; return with no error
+			return nil
+		}
+	}
+
+	// log group does not exist; create a new group
+	console.AddingResource("Creating CloudWatch Logs Group", groupName, false)
+	if err := c.awsClient.CloudWatchLogs().CreateGroup(groupName); err != nil {
+		return fmt.Errorf("Failed to create CloudWatch Logs Group [%s]: %s", groupName, err.Error())
+	}
+
+	return nil
+}

config/config.go

@@ -9,6 +9,7 @@ type Config struct {
 	Units        *uint16            `json:"units,omitempty" yaml:"units,omitempty"`
 	Env          map[string]string  `json:"env,omitempty" yaml:"env,omitempty"`
 	LoadBalancer ConfigLoadBalancer `json:"load_balancer" yaml:"load_balancer"`
+	Logging      ConfigLogging      `json:"logging" yaml:"logging"`
 	AWS          ConfigAWS          `json:"aws" yaml:"aws"`
 	Docker       ConfigDocker       `json:"docker" yaml:"docker"`
 }
@@ -29,6 +30,11 @@ type ConfigLoadBalancerHealthCheck struct {
 	UnhealthyLimit *uint16 `json:"unhealthy_limit,omitempty" yaml:"unhealthy_limit,omitempty"`
 }
 
+type ConfigLogging struct {
+	Driver  *string           `json:"driver,omitempty" yaml:"driver,omitempty"`
+	Options map[string]string `json:"options" yaml:"options"`
+}
+
 type ConfigAWS struct {
 	ELBLoadBalancerName  *string `json:"elb_name,omitempty" yaml:"elb_name,omitempty"`
 	ELBTargetGroupName   *string `json:"elb_target_group_name,omitempty" yaml:"elb_target_group_name,omitempty"`

config/config_test.go

@@ -1,8 +1,6 @@
 package config
 
-import (
-	"github.com/coldbrewcloud/coldbrew-cli/utils/conv"
-)
+import "github.com/coldbrewcloud/coldbrew-cli/utils/conv"
 
 const refConfigYAML = `
 name: echo
@@ -29,6 +27,12 @@ load_balancer:
     healthy_limit: 5
     unhealthy_limit: 2
 
+logging:
+  driver: json-file
+  options:
+    logopt1: value1
+    logopt2: value2
+
 aws:
   elb_name: echo-lb
   elb_target_group_name: echo-target
@@ -65,6 +69,13 @@ const refConfigJSON = `
 			"unhealthy_limit": 2
 		}
 	},
+	"logging": {
+	    "driver": "json-file",
+	    "options": {
+	        "logopt1": "value1",
+	        "logopt2": "value2"
+	    }
+	},
 	"aws": {
 		"elb_name": "echo-lb",
 		"elb_target_group_name": "echo-target",
@@ -101,6 +112,13 @@ var refConfig = &Config{
 			UnhealthyLimit: conv.U16P(2),
 		},
 	},
+	Logging: ConfigLogging{
+		Driver: conv.SP("json-file"),
+		Options: map[string]string{
+			"logopt1": "value1",
+			"logopt2": "value2",
+		},
+	},
 	AWS: ConfigAWS{
 		ELBLoadBalancerName:  conv.SP("echo-lb"),
 		ELBTargetGroupName:   conv.SP("echo-target"),

config/default_config.go

@@ -19,16 +19,24 @@ func DefaultConfig(appName string) *Config {
 	conf.Env = make(map[string]string)
 
 	// load balancer
-	conf.LoadBalancer.Enabled = conv.BP(false)
-	conf.LoadBalancer.Port = conv.U16P(80)
+	{
+		conf.LoadBalancer.Enabled = conv.BP(false)
+		conf.LoadBalancer.Port = conv.U16P(80)
+
+		// health check
+		conf.LoadBalancer.HealthCheck.Path = conv.SP("/")
+		conf.LoadBalancer.HealthCheck.Status = conv.SP("200-299")
+		conf.LoadBalancer.HealthCheck.Interval = conv.SP("15s")
+		conf.LoadBalancer.HealthCheck.Timeout = conv.SP("10s")
+		conf.LoadBalancer.HealthCheck.HealthyLimit = conv.U16P(3)
+		conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(3)
+	}
 
-	// health check
-	conf.LoadBalancer.HealthCheck.Path = conv.SP("/")
-	conf.LoadBalancer.HealthCheck.Status = conv.SP("200-299")
-	conf.LoadBalancer.HealthCheck.Interval = conv.SP("15s")
-	conf.LoadBalancer.HealthCheck.Timeout = conv.SP("10s")
-	conf.LoadBalancer.HealthCheck.HealthyLimit = conv.U16P(3)
-	conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(3)
+	// logging
+	{
+		conf.Logging.Driver = nil
+		conf.Logging.Options = make(map[string]string)
+	}
 
 	// AWS
 	{

config/load.go

@@ -76,6 +76,18 @@ func (c *Config) Defaults(source *Config) {
 	defU16(&c.LoadBalancer.HealthCheck.HealthyLimit, source.LoadBalancer.HealthCheck.HealthyLimit)
 	defU16(&c.LoadBalancer.HealthCheck.UnhealthyLimit, source.LoadBalancer.HealthCheck.UnhealthyLimit)
 
+	// logging
+	if conv.S(c.Logging.Driver) == "" {
+		// logging option is copied only when logging driver was copied
+		defS(&c.Logging.Driver, source.Logging.Driver)
+		if source.Logging.Options != nil {
+			c.Logging.Options = make(map[string]string)
+			for k, v := range source.Logging.Options {
+				c.Logging.Options[k] = v
+			}
+		}
+	}
+
 	// AWS
 	defS(&c.AWS.ELBLoadBalancerName, source.AWS.ELBLoadBalancerName)
 	defS(&c.AWS.ELBTargetGroupName, source.AWS.ELBTargetGroupName)

config/validate.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/coldbrewcloud/coldbrew-cli/aws"
 	"github.com/coldbrewcloud/coldbrew-cli/core"
 	"github.com/coldbrewcloud/coldbrew-cli/utils"
 	"github.com/coldbrewcloud/coldbrew-cli/utils/conv"
@@ -90,6 +91,20 @@ func (c *Config) Validate() error {
 		return fmt.Errorf("Invalid ELB Security Group name [%s]", conv.S(c.AWS.ELBSecurityGroupName))
 	}
 
+	switch conv.S(c.Logging.Driver) {
+	case "",
+		aws.ECSTaskDefinitionLogDriverAWSLogs,
+		aws.ECSTaskDefinitionLogDriverJSONFile,
+		aws.ECSTaskDefinitionLogDriverSyslog,
+		aws.ECSTaskDefinitionLogDriverFluentd,
+		aws.ECSTaskDefinitionLogDriverGelf,
+		aws.ECSTaskDefinitionLogDriverJournald,
+		aws.ECSTaskDefinitionLogDriverSplunk:
+		// need more validation for other driver types
+	default:
+		return fmt.Errorf("Log driver [%s] not supported.", conv.S(c.Logging.Driver))
+	}
+
 	if utils.IsBlank(conv.S(c.Docker.Bin)) {
 		return fmt.Errorf("Invalid docker executable path [%s]", conv.S(c.Docker.Bin))
 	}

config/validate_test.go

@@ -245,6 +245,29 @@ func TestConfig_Validate(t *testing.T) {
 	conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(0)
 	assert.NotNil(t, conf.Validate())
 
+	// Logging Driver
+	conf = DefaultConfig("app1")
+	conf.Logging.Driver = nil
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("json-file")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("syslog")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("journald")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("gelf")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("fluentd")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("splunk")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("awslogs")
+	assert.Nil(t, conf.Validate())
+	conf.Logging.Driver = conv.SP("unknowndriver")
+	assert.NotNil(t, conf.Validate())
+
 	// AWS ECR Repository Name
 	conf = DefaultConfig("app1")
 	conf.AWS.ECRRepositoryName = nil