Merge pull request #19 from coldbrewcloud/custom-ami

Support custom ECS Container Instance images

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

aws/autoscaling/client.go

@@ -30,6 +30,7 @@ func (c *Client) CreateLaunchConfiguration(launchConfigurationName, instanceType
 		LaunchConfigurationName: _aws.String(launchConfigurationName),
 		SecurityGroups:          _aws.StringSlice(securityGroupIDs),
 		UserData:                _aws.String(userData),
+		InstanceMonitoring:      &_autoscaling.InstanceMonitoring{Enabled: _aws.Bool(false)},
 	}
 
 	if !utils.IsBlank(keyPairName) {

aws/consts.go

@@ -2,6 +2,7 @@ package aws
 
 const (
 	AWSRegionUSEast1      = "us-east-1"
+	AWSRegionUSEast2      = "us-east-2"
 	AWSRegionUSWest1      = "us-west-1"
 	AWSRegionUSWest2      = "us-west-2"
 	AWSRegionEUWest1      = "eu-west-1"

aws/ec2/client.go

@@ -408,3 +408,22 @@ func (c *Client) RetrieveInstances(instanceIDs []string) ([]*_ec2.Instance, erro
 
 	return instances, nil
 }
+
+func (c *Client) FindImage(ownerID, tagName string) ([]*_ec2.Image, error) {
+	params := &_ec2.DescribeImagesInput{
+		Owners: _aws.StringSlice([]string{ownerID}),
+		Filters: []*_ec2.Filter{
+			{
+				Name:   _aws.String("tag-key"),
+				Values: _aws.StringSlice([]string{tagName}),
+			},
+		},
+	}
+
+	res, err := c.svc.DescribeImages(params)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Images, nil
+}

commands/clustercreate/aws.go

@@ -3,13 +3,33 @@ package clustercreate
 import (
 	"encoding/base64"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/coldbrewcloud/coldbrew-cli/aws"
+	"github.com/coldbrewcloud/coldbrew-cli/console"
 	"github.com/coldbrewcloud/coldbrew-cli/core"
 	"github.com/coldbrewcloud/coldbrew-cli/utils/conv"
 )
 
+const (
+	defaultECSContainerInstanceImageIDBaseURL = "https://s3-us-west-2.amazonaws.com/files.coldbrewcloud.com/coldbrew-cli/ecs-ci-ami/default/"
+	defaultECSContainerInstanceImageOwnerID   = "865092420289"
+)
+
+var defaultECSContainerInstanceAmazonImageID = map[string]string{
+	aws.AWSRegionUSEast1:      "ami-1924770e",
+	aws.AWSRegionUSEast2:      "ami-bd3e64d8",
+	aws.AWSRegionUSWest1:      "ami-7f004b1f",
+	aws.AWSRegionUSWest2:      "ami-56ed4936",
+	aws.AWSRegionEUWest1:      "ami-c8337dbb",
+	aws.AWSRegionEUCentral1:   "ami-dd12ebb2",
+	aws.AWSRegionAPNorthEast1: "ami-c8b016a9",
+	aws.AWSRegionAPSouthEast1: "ami-6d22840e",
+	aws.AWSRegionAPSouthEast2: "ami-73407d10",
+}
+
 func (c *Command) getAWSInfo() (string, string, []string, error) {
 	regionName, vpcID, err := c.globalFlags.GetAWSRegionAndVPCID()
 	if err != nil {
@@ -28,30 +48,45 @@ func (c *Command) getAWSInfo() (string, string, []string, error) {
 	return regionName, vpcID, subnetIDs, nil
 }
 
-func (c *Command) getClusterImageID(region string) string {
-	switch region {
-	case aws.AWSRegionUSEast1:
-		return "ami-40286957"
-	case aws.AWSRegionUSWest1:
-		return "ami-20fab440"
-	case aws.AWSRegionUSWest2:
-		return "ami-562cf236"
-	case aws.AWSRegionEUWest1:
-		return "ami-175f1964"
-	case aws.AWSRegionEUCentral1:
-		return "ami-c55ea2aa"
-	case aws.AWSRegionAPNorthEast1:
-		return "ami-010ed160"
-	case aws.AWSRegionAPSouthEast1:
-		return "ami-438b2f20"
-	case aws.AWSRegionAPSouthEast2:
-		return "ami-862211e5"
-	default:
-		return ""
+func (c *Command) retrieveDefaultECSContainerInstancesImageID(region string) string {
+	defaultImages, err := c.awsClient.EC2().FindImage(defaultECSContainerInstanceImageOwnerID, core.AWSTagNameCreatedTimestamp)
+	if err == nil {
+		var latestImage *ec2.Image
+		var latestImageCreationTime string
+
+		for _, image := range defaultImages {
+			if conv.S(image.OwnerId) == defaultECSContainerInstanceImageOwnerID {
+				if latestImage == nil {
+					latestImageCreationTime = getCreationTimeFromTags(image.Tags)
+					if latestImageCreationTime != "" {
+						latestImage = image
+					}
+				} else {
+					creationTime := getCreationTimeFromTags(image.Tags)
+					if creationTime != "" {
+						if strings.Compare(latestImageCreationTime, creationTime) < 0 {
+							latestImage = image
+							latestImageCreationTime = creationTime
+						}
+					}
+				}
+			}
+		}
+
+		if latestImage != nil {
+			return conv.S(latestImage.ImageId)
+		}
 	}
+
+	// if failed to find coldbrew-cli default image, use Amazon ECS optimized image as fallback
+	console.Error("Failed to retrieve default image ID for ECS Container Instances. Amazon ECS Optimized AMI will be used instead.")
+	if imageID, ok := defaultECSContainerInstanceAmazonImageID[region]; ok {
+		return imageID
+	}
+	return ""
 }
 
-func (c *Command) getInstanceUserData(ecsClusterName string) string {
+func (c *Command) getDefaultInstanceUserData(ecsClusterName string) string {
 	userData := fmt.Sprintf(`#!/bin/bash
 echo ECS_CLUSTER=%s >> /etc/ecs/ecs.config`, ecsClusterName)
 	return base64.StdEncoding.EncodeToString([]byte(userData))
@@ -107,3 +142,13 @@ func (c *Command) waitAutoScalingGroupDeletion(autoScalingGroupName string) erro
 	}
 	return nil
 }
+
+func getCreationTimeFromTags(tags []*ec2.Tag) string {
+	for _, tag := range tags {
+		if conv.S(tag.Key) == core.AWSTagNameCreatedTimestamp {
+			return conv.S(tag.Value)
+			break
+		}
+	}
+	return ""
+}

commands/clustercreate/command.go

@@ -1,7 +1,9 @@
 package clustercreate
 
 import (
+	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"strings"
 	"time"
 
@@ -245,12 +247,26 @@ func (c *Command) Run() error {
 		}
 
 		// container instance image ID
-		imageID := c.getClusterImageID(conv.S(c.globalFlags.AWSRegion))
-		if imageID == "" {
-			return console.ExitWithErrorString("No defatul instance image found")
+		imageID := conv.S(c.commandFlags.InstanceImageID)
+		if utils.IsBlank(imageID) {
+			// if not provided, use coldbrew-cli default images
+			imageID = c.retrieveDefaultECSContainerInstancesImageID(conv.S(c.globalFlags.AWSRegion))
+			if imageID == "" {
+				return console.ExitWithErrorString("No default instance image found")
+			}
 		}
 
-		instanceUserData := c.getInstanceUserData(ecsClusterName)
+		// container instance userdata
+		instanceUserData := c.getDefaultInstanceUserData(ecsClusterName)
+		instanceUserDataFile := conv.S(c.commandFlags.InstanceUserDataFile)
+		if !utils.IsBlank(instanceUserDataFile) {
+			// if user provided custom user data file, use it instead
+			fileData, err := ioutil.ReadFile(instanceUserDataFile)
+			if err != nil {
+				return console.ExitWithErrorString("Failed to read userdata file [%s]: %s", instanceUserDataFile, err.Error())
+			}
+			instanceUserData = base64.StdEncoding.EncodeToString(fileData)
+		}
 
 		// NOTE: sometimes resources created (e.g. InstanceProfile) do not become available immediately.
 		err = utils.Retry(func() (bool, error) {

commands/clustercreate/flags.go

@@ -6,21 +6,25 @@ import (
 )
 
 type Flags struct {
-	InstanceType    *string `json:"instance_type"`
-	InitialCapacity *uint16 `json:"initial_capacity"`
-	NoKeyPair       *bool   `json:"no-keypair"`
-	KeyPairName     *string `json:"keypair_name"`
-	InstanceProfile *string `json:"instance_profile"`
-	ForceCreate     *bool   `json:"force"`
+	InstanceType         *string `json:"instance_type"`
+	InitialCapacity      *uint16 `json:"initial_capacity"`
+	NoKeyPair            *bool   `json:"no-keypair"`
+	KeyPairName          *string `json:"keypair_name"`
+	InstanceProfile      *string `json:"instance_profile"`
+	InstanceImageID      *string `json:"instance_image_id"`
+	InstanceUserDataFile *string `json:"instance_user_data_file"`
+	ForceCreate          *bool   `json:"force"`
 }
 
 func NewFlags(kc *kingpin.CmdClause) *Flags {
 	return &Flags{
-		InstanceType:    kc.Flag("instance-type", "Container instance type").Default(core.DefaultContainerInstanceType()).String(),
-		InitialCapacity: kc.Flag("instance-count", "Initial number of container instances").Default("1").Uint16(),
-		NoKeyPair:       kc.Flag("disable-keypair", "Do not assign EC2 keypairs").Bool(),
-		KeyPairName:     kc.Flag("key", "EC2 keypair name").Default("").String(),
-		InstanceProfile: kc.Flag("instance-profile", "IAM instance profile name for container instances").Default("").String(),
-		ForceCreate:     kc.Flag("yes", "Create all resource with no confirmation").Short('y').Default("false").Bool(),
+		InstanceType:         kc.Flag("instance-type", "Container instance type").Default(core.DefaultContainerInstanceType()).String(),
+		InitialCapacity:      kc.Flag("instance-count", "Initial number of container instances").Default("1").Uint16(),
+		NoKeyPair:            kc.Flag("disable-keypair", "Do not assign EC2 keypairs").Bool(),
+		KeyPairName:          kc.Flag("key", "EC2 keypair name").Default("").String(),
+		InstanceProfile:      kc.Flag("instance-profile", "IAM instance profile name for container instances").Default("").String(),
+		InstanceImageID:      kc.Flag("instance-image", "EC2 Image (AMI) ID for ECS Container Instances").Default("").String(),
+		InstanceUserDataFile: kc.Flag("instance-userdata", "File path that contains userdata for ECS Container Instances").Default("").String(),
+		ForceCreate:          kc.Flag("yes", "Create all resource with no confirmation").Short('y').Default("false").Bool(),
 	}
 }