From 1e9b0e36be0f2350f7a4dcd8b287932619f71077 Mon Sep 17 00:00:00 2001 From: Daniel Kang Date: Sat, 29 Oct 2016 20:25:39 -0700 Subject: [PATCH 1/2] - added logging support - for "awslogs", CloudWatch Logs group will be created if needed --- aws/client.go | 9 ++++++ aws/consts.go | 8 ++++++ aws/ecs/client.go | 12 +++----- aws/logs/client.go | 59 ++++++++++++++++++++++++++++++++++++++ commands/deploy/aws_ecs.go | 51 ++++++++++++++++++++++++++++++-- config/config.go | 6 ++++ config/config_test.go | 24 ++++++++++++++-- config/default_config.go | 26 +++++++++++------ config/load.go | 12 ++++++++ config/validate.go | 15 ++++++++++ config/validate_test.go | 23 +++++++++++++++ core/apps.go | 4 +++ 12 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 aws/logs/client.go diff --git a/aws/client.go b/aws/client.go index ee18d09..dfe3103 100644 --- a/aws/client.go +++ b/aws/client.go @@ -10,6 +10,7 @@ import ( "github.com/coldbrewcloud/coldbrew-cli/aws/ecs" "github.com/coldbrewcloud/coldbrew-cli/aws/elb" "github.com/coldbrewcloud/coldbrew-cli/aws/iam" + "github.com/coldbrewcloud/coldbrew-cli/aws/logs" "github.com/coldbrewcloud/coldbrew-cli/aws/sns" ) @@ -24,6 +25,7 @@ type Client struct { ecrClient *ecr.Client iamClient *iam.Client snsClient *sns.Client + logsClient *logs.Client } func NewClient(region, accessKey, secretKey string) *Client { @@ -86,3 +88,10 @@ func (c *Client) SNS() *sns.Client { } return c.snsClient } + +func (c *Client) CloudWatchLogs() *logs.Client { + if c.logsClient == nil { + c.logsClient = logs.New(c.session, c.config) + } + return c.logsClient +} diff --git a/aws/consts.go b/aws/consts.go index 86e2243..0feaff9 100644 --- a/aws/consts.go +++ b/aws/consts.go @@ -11,4 +11,12 @@ const ( AWSRegionAPSouthEast1 = "ap-southeast-1" AWSRegionAPSouthEast2 = "ap-southeast-2" AWSRegionSAEast1 = "sa-east-1" + + ECSTaskDefinitionLogDriverJSONFile = "json-file" + ECSTaskDefinitionLogDriverAWSLogs = "awslogs" + ECSTaskDefinitionLogDriverSyslog = "syslog" + ECSTaskDefinitionLogDriverJournald = "journald" + ECSTaskDefinitionLogDriverGelf = "gelf" + ECSTaskDefinitionLogDriverFluentd = "fluentd" + ECSTaskDefinitionLogDriverSplunk = "splunk" ) diff --git a/aws/ecs/client.go b/aws/ecs/client.go index af03778..662b00d 100644 --- a/aws/ecs/client.go +++ b/aws/ecs/client.go @@ -71,7 +71,7 @@ func (c *Client) DeleteCluster(clusterName string) error { return err } -func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerName string, cpu, memory uint64, envs map[string]string, portMappings []PortMapping, cloudWatchLogs bool) (*_ecs.TaskDefinition, error) { +func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerName string, cpu, memory uint64, envs map[string]string, portMappings []PortMapping, logDriver string, logDriverOptions map[string]string) (*_ecs.TaskDefinition, error) { if taskDefinitionName == "" { return nil, errors.New("taskDefinitionName is empty") } @@ -96,14 +96,10 @@ func (c *Client) UpdateTaskDefinition(taskDefinitionName, image, taskContainerNa Family: _aws.String(taskDefinitionName), } - // TODO: move this out of this function - if cloudWatchLogs { + if logDriver != "" { params.ContainerDefinitions[0].LogConfiguration = &_ecs.LogConfiguration{ - LogDriver: _aws.String(_ecs.LogDriverAwslogs), - Options: _aws.StringMap(map[string]string{ - "awslogs-group": "coldbrewcloud-deploy-logs", - "awslogs-region": c.awsRegion, - }), + LogDriver: _aws.String(logDriver), + Options: _aws.StringMap(logDriverOptions), } } diff --git a/aws/logs/client.go b/aws/logs/client.go new file mode 100644 index 0000000..f5065d0 --- /dev/null +++ b/aws/logs/client.go @@ -0,0 +1,59 @@ +package logs + +import ( + _aws "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + _logs "github.com/aws/aws-sdk-go/service/cloudwatchlogs" +) + +type Client struct { + svc *_logs.CloudWatchLogs + awsRegion string +} + +func New(session *session.Session, config *_aws.Config) *Client { + return &Client{ + awsRegion: *config.Region, + svc: _logs.New(session, config), + } +} + +func (c *Client) CreateGroup(groupName string) error { + params := &_logs.CreateLogGroupInput{ + LogGroupName: _aws.String(groupName), + } + + _, err := c.svc.CreateLogGroup(params) + if err != nil { + return err + } + + return nil +} + +func (c *Client) ListGroups(groupNamePrefix string) ([]*_logs.LogGroup, error) { + var nextToken *string + groups := []*_logs.LogGroup{} + + for { + params := &_logs.DescribeLogGroupsInput{ + LogGroupNamePrefix: _aws.String(groupNamePrefix), + NextToken: nextToken, + } + + res, err := c.svc.DescribeLogGroups(params) + if err != nil { + return nil, err + } + + groups = append(groups, res.LogGroups...) + + if res.NextToken == nil { + break + } else { + nextToken = res.NextToken + } + } + + return groups, nil +} diff --git a/commands/deploy/aws_ecs.go b/commands/deploy/aws_ecs.go index 37c5f01..e1feca7 100644 --- a/commands/deploy/aws_ecs.go +++ b/commands/deploy/aws_ecs.go @@ -5,9 +5,11 @@ import ( "fmt" "math" + "github.com/coldbrewcloud/coldbrew-cli/aws" "github.com/coldbrewcloud/coldbrew-cli/aws/ecs" "github.com/coldbrewcloud/coldbrew-cli/console" "github.com/coldbrewcloud/coldbrew-cli/core" + "github.com/coldbrewcloud/coldbrew-cli/utils" "github.com/coldbrewcloud/coldbrew-cli/utils/conv" ) @@ -31,7 +33,30 @@ func (c *Command) updateECSTaskDefinition(dockerImageFullURI string) (string, er return "", err } memory /= 1000 * 1000 - useCloudWatchLogs := false + + // logging + loggingDriver := conv.S(c.conf.Logging.Driver) + if c.conf.Logging.Options == nil { + c.conf.Logging.Options = make(map[string]string) + } + switch loggingDriver { + case aws.ECSTaskDefinitionLogDriverAWSLogs: + // test if group needs to be created + awsLogsGroupName, ok := c.conf.Logging.Options["awslogs-group"] + if !ok || utils.IsBlank(awsLogsGroupName) { + awsLogsGroupName = core.DefaultCloudWatchLogsGroupName(conv.S(c.conf.Name), conv.S(c.conf.ClusterName)) + c.conf.Logging.Options["awslogs-group"] = awsLogsGroupName + } + if err := c.PrepareCloudWatchLogsGroup(awsLogsGroupName); err != nil { + return "", err + } + + // assign region if not provided + awsLogsRegionName, ok := c.conf.Logging.Options["awslogs-region"] + if !ok || utils.IsBlank(awsLogsRegionName) { + c.conf.Logging.Options["awslogs-region"] = conv.S(c.globalFlags.AWSRegion) + } + } console.UpdatingResource("Updating ECS Task Definition", ecsTaskDefinitionName, false) ecsTaskDef, err := c.awsClient.ECS().UpdateTaskDefinition( @@ -42,7 +67,7 @@ func (c *Command) updateECSTaskDefinition(dockerImageFullURI string) (string, er memory, c.conf.Env, portMappings, - useCloudWatchLogs) + loggingDriver, c.conf.Logging.Options) if err != nil { return "", fmt.Errorf("Failed to update ECS Task Definition [%s]: %s", ecsTaskDefinitionName, err.Error()) } @@ -136,3 +161,25 @@ func (c *Command) updateECSService(ecsClusterName, ecsServiceName, ecsTaskDefini return nil } + +func (c *Command) PrepareCloudWatchLogsGroup(groupName string) error { + groups, err := c.awsClient.CloudWatchLogs().ListGroups(groupName) + if err != nil { + return fmt.Errorf("Failed to list CloudWatch Logs Group [%s]: %s", groupName, err.Error()) + } + + for _, group := range groups { + if conv.S(group.LogGroupName) == groupName { + // log group exists; return with no error + return nil + } + } + + // log group does not exist; create a new group + console.AddingResource("Creating CloudWatch Logs Group", groupName, false) + if err := c.awsClient.CloudWatchLogs().CreateGroup(groupName); err != nil { + return fmt.Errorf("Failed to create CloudWatch Logs Group [%s]: %s", groupName, err.Error()) + } + + return nil +} diff --git a/config/config.go b/config/config.go index f38bd7f..070fef1 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ type Config struct { Units *uint16 `json:"units,omitempty" yaml:"units,omitempty"` Env map[string]string `json:"env,omitempty" yaml:"env,omitempty"` LoadBalancer ConfigLoadBalancer `json:"load_balancer" yaml:"load_balancer"` + Logging ConfigLogging `json:"logging" yaml:"logging"` AWS ConfigAWS `json:"aws" yaml:"aws"` Docker ConfigDocker `json:"docker" yaml:"docker"` } @@ -29,6 +30,11 @@ type ConfigLoadBalancerHealthCheck struct { UnhealthyLimit *uint16 `json:"unhealthy_limit,omitempty" yaml:"unhealthy_limit,omitempty"` } +type ConfigLogging struct { + Driver *string `json:"driver,omitempty" yaml:"driver,omitempty"` + Options map[string]string `json:"options" yaml:"options"` +} + type ConfigAWS struct { ELBLoadBalancerName *string `json:"elb_name,omitempty" yaml:"elb_name,omitempty"` ELBTargetGroupName *string `json:"elb_target_group_name,omitempty" yaml:"elb_target_group_name,omitempty"` diff --git a/config/config_test.go b/config/config_test.go index f2c8860..0db99ac 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,8 +1,6 @@ package config -import ( - "github.com/coldbrewcloud/coldbrew-cli/utils/conv" -) +import "github.com/coldbrewcloud/coldbrew-cli/utils/conv" const refConfigYAML = ` name: echo @@ -29,6 +27,12 @@ load_balancer: healthy_limit: 5 unhealthy_limit: 2 +logging: + driver: json-file + options: + logopt1: value1 + logopt2: value2 + aws: elb_name: echo-lb elb_target_group_name: echo-target @@ -65,6 +69,13 @@ const refConfigJSON = ` "unhealthy_limit": 2 } }, + "logging": { + "driver": "json-file", + "options": { + "logopt1": "value1", + "logopt2": "value2" + } + }, "aws": { "elb_name": "echo-lb", "elb_target_group_name": "echo-target", @@ -101,6 +112,13 @@ var refConfig = &Config{ UnhealthyLimit: conv.U16P(2), }, }, + Logging: ConfigLogging{ + Driver: conv.SP("json-file"), + Options: map[string]string{ + "logopt1": "value1", + "logopt2": "value2", + }, + }, AWS: ConfigAWS{ ELBLoadBalancerName: conv.SP("echo-lb"), ELBTargetGroupName: conv.SP("echo-target"), diff --git a/config/default_config.go b/config/default_config.go index 7b0b5bd..f1be3be 100644 --- a/config/default_config.go +++ b/config/default_config.go @@ -19,16 +19,24 @@ func DefaultConfig(appName string) *Config { conf.Env = make(map[string]string) // load balancer - conf.LoadBalancer.Enabled = conv.BP(false) - conf.LoadBalancer.Port = conv.U16P(80) + { + conf.LoadBalancer.Enabled = conv.BP(false) + conf.LoadBalancer.Port = conv.U16P(80) + + // health check + conf.LoadBalancer.HealthCheck.Path = conv.SP("/") + conf.LoadBalancer.HealthCheck.Status = conv.SP("200-299") + conf.LoadBalancer.HealthCheck.Interval = conv.SP("15s") + conf.LoadBalancer.HealthCheck.Timeout = conv.SP("10s") + conf.LoadBalancer.HealthCheck.HealthyLimit = conv.U16P(3) + conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(3) + } - // health check - conf.LoadBalancer.HealthCheck.Path = conv.SP("/") - conf.LoadBalancer.HealthCheck.Status = conv.SP("200-299") - conf.LoadBalancer.HealthCheck.Interval = conv.SP("15s") - conf.LoadBalancer.HealthCheck.Timeout = conv.SP("10s") - conf.LoadBalancer.HealthCheck.HealthyLimit = conv.U16P(3) - conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(3) + // logging + { + conf.Logging.Driver = nil + conf.Logging.Options = make(map[string]string) + } // AWS { diff --git a/config/load.go b/config/load.go index 7135dba..1452dbd 100644 --- a/config/load.go +++ b/config/load.go @@ -76,6 +76,18 @@ func (c *Config) Defaults(source *Config) { defU16(&c.LoadBalancer.HealthCheck.HealthyLimit, source.LoadBalancer.HealthCheck.HealthyLimit) defU16(&c.LoadBalancer.HealthCheck.UnhealthyLimit, source.LoadBalancer.HealthCheck.UnhealthyLimit) + // logging + if conv.S(c.Logging.Driver) == "" { + // logging option is copied only when logging driver was copied + defS(&c.Logging.Driver, source.Logging.Driver) + if source.Logging.Options != nil { + c.Logging.Options = make(map[string]string) + for k, v := range source.Logging.Options { + c.Logging.Options[k] = v + } + } + } + // AWS defS(&c.AWS.ELBLoadBalancerName, source.AWS.ELBLoadBalancerName) defS(&c.AWS.ELBTargetGroupName, source.AWS.ELBTargetGroupName) diff --git a/config/validate.go b/config/validate.go index 7686fd8..0a3f259 100644 --- a/config/validate.go +++ b/config/validate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/coldbrewcloud/coldbrew-cli/aws" "github.com/coldbrewcloud/coldbrew-cli/core" "github.com/coldbrewcloud/coldbrew-cli/utils" "github.com/coldbrewcloud/coldbrew-cli/utils/conv" @@ -90,6 +91,20 @@ func (c *Config) Validate() error { return fmt.Errorf("Invalid ELB Security Group name [%s]", conv.S(c.AWS.ELBSecurityGroupName)) } + switch conv.S(c.Logging.Driver) { + case "", + aws.ECSTaskDefinitionLogDriverAWSLogs, + aws.ECSTaskDefinitionLogDriverJSONFile, + aws.ECSTaskDefinitionLogDriverSyslog, + aws.ECSTaskDefinitionLogDriverFluentd, + aws.ECSTaskDefinitionLogDriverGelf, + aws.ECSTaskDefinitionLogDriverJournald, + aws.ECSTaskDefinitionLogDriverSplunk: + // need more validation for other driver types + default: + return fmt.Errorf("Log driver [%s] not supported.", conv.S(c.Logging.Driver)) + } + if utils.IsBlank(conv.S(c.Docker.Bin)) { return fmt.Errorf("Invalid docker executable path [%s]", conv.S(c.Docker.Bin)) } diff --git a/config/validate_test.go b/config/validate_test.go index 94dae1a..628e4aa 100644 --- a/config/validate_test.go +++ b/config/validate_test.go @@ -245,6 +245,29 @@ func TestConfig_Validate(t *testing.T) { conf.LoadBalancer.HealthCheck.UnhealthyLimit = conv.U16P(0) assert.NotNil(t, conf.Validate()) + // Logging Driver + conf = DefaultConfig("app1") + conf.Logging.Driver = nil + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("json-file") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("syslog") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("journald") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("gelf") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("fluentd") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("splunk") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("awslogs") + assert.Nil(t, conf.Validate()) + conf.Logging.Driver = conv.SP("unknowndriver") + assert.NotNil(t, conf.Validate()) + // AWS ECR Repository Name conf = DefaultConfig("app1") conf.AWS.ECRRepositoryName = nil diff --git a/core/apps.go b/core/apps.go index 58c1020..c814f53 100644 --- a/core/apps.go +++ b/core/apps.go @@ -58,3 +58,7 @@ func DefaultELBLoadBalancerSecurityGroupName(appName string) string { func DefaultECRRepository(appName string) string { return fmt.Sprintf("coldbrew/%s", appName) } + +func DefaultCloudWatchLogsGroupName(appName, clusterName string) string { + return fmt.Sprintf("coldbrew-%s-%s", clusterName, appName) +} From bc6eeb23da92ee30673463f994b612a8cf04e456 Mon Sep 17 00:00:00 2001 From: Daniel Kang Date: Sat, 29 Oct 2016 20:47:10 -0700 Subject: [PATCH 2/2] v1.3.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 867e524..589268e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 \ No newline at end of file +1.3.0 \ No newline at end of file