Add support for a custom aws endpoint

.circleci/config.yml

@@ -316,6 +316,52 @@ jobs:
       - store_test_results:
           path: /tmp/logs
 
+  mock_aws_test:
+    <<: *env
+    docker:
+      - image: cimg/python:3.10.2
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+
+      - run:
+          <<: *install_gruntwork_utils
+
+      # The weird way you have to set PATH in Circle 2.0
+      - run: |
+          echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV
+      - run:
+          name: install mock aws server
+          command: |
+            pip install moto[server]
+      - run:
+          name: install aws cli
+          command: |
+            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+            unzip awscliv2.zip
+            sudo ./aws/install
+      - run:
+          name: run mock aws server
+          command: |
+            moto_server | tee /tmp/logs/moto_server_output.log
+          background: true
+      - run:
+          name: wait for mock aws server
+          command: |
+            while ! AWS_REGION=us-west-2 AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy aws --endpoint http://localhost:5000 sts get-caller-identity; do
+              echo "Waiting for local mock AWS server"
+              sleep 1
+            done
+      - run:
+          command: |
+            mkdir -p /tmp/logs
+            run-go-tests --packages "-tags=mockaws ./test/" | tee /tmp/logs/test_output.log
+
+      # Store test result and log artifacts for browsing purposes
+      - store_artifacts:
+          path: /tmp/logs
+      - store_test_results:
+          path: /tmp/logs
 
   deploy:
     <<: *defaults
@@ -386,6 +432,15 @@ workflows:
             tags:
               only: /^v.*/
 
+      - mock_aws_test:
+          context:
+            - Gruntwork Admin
+          requires:
+            - setup
+          filters:
+            tags:
+              only: /^v.*/
+
       - deploy:
           context:
             - AWS__PHXDEVOPS__circle-ci-test

docs/_docs/02_testing-best-practices/tools-and-plugins.md

@@ -16,7 +16,14 @@ This page contains a list of tools and plugins for Terratest to make integration
 
 - [Tools and Plugins](#tools-and-plugins)
   - [Terratest Maven Plugin](#terratest-maven-plugin)
+  - [Custom AWS Endpoints](#custom-aws-endpoints)
 
 ### Terratest Maven Plugin
 
 The Terratest Maven Plugin aims to bring Terratest to the JVM world. Create your Go based tests beside your Java code with Maven and run them together. You can export the results into Json or an HTML page. As the plugin is MIT licensed, it is easy and painless to integrate into any Java+Maven combination. To learn more check out the website: [Terratest Maven Plugin](https://terratest-maven-plugin.github.io) and the [GitHub repository](https://github.com/terratest-maven-plugin/terratest-maven-plugin)
+
+### Custom AWS Endpoints
+
+Terratest can be used with a custom AWS endpoint. One of the use cases for this is testing your automation using a mock AWS server such as [Moto in standalone server mode](https://docs.getmoto.org/en/latest/docs/getting_started.html#stand-alone-server-mode). This allows you to test your AWS based automation without an AWS account.
+
+To configure this, set `TERRATEST_CUSTOM_AWS_ENDPOINT` to the endpoint that you want Terratest to use for accessing AWS resources. You must also configure your automation to use the same endpoint, for example [Terraform supports custom AWS endpoints as well](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints).

examples/terraform-aws-endpoint-example/README.md

@@ -0,0 +1,28 @@
+# Terraform AWS Endpoint Example
+
+This folder contains a simple Terraform module to demonstrate using custom
+endpoints. It's deploying some AWS resources to `http://localhost:5000`, which
+is the default port for [moto running in server
+mode](http://docs.getmoto.org/en/latest/docs/server_mode.html). This allows for
+testing terraform modules locally with no connection to AWS.
+
+Check out
+[test/terraform_aws_endpoint_example_test.go](/test/terraform_aws_endpoint_example_test.go)
+to see how you can write automated tests for this module.
+
+## Running this module manually
+
+1. Run [Moto locally in server mode](http://docs.getmoto.org/en/latest/docs/server_mode.html)
+1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`.
+1. Run `terraform init`.
+1. Run `terraform apply`.
+1. When you're done, run `terraform destroy`.
+
+## Running automated tests against this module
+
+1. Run [Moto locally in server mode](http://docs.getmoto.org/en/latest/docs/server_mode.html)
+1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`.
+1. Install [Golang](https://golang.org/) and make sure this code is checked out into your `GOPATH`.
+1. `cd test`
+1. `dep ensure`
+1. `go test -v -run TestTerraformAwsEndpointExample`

examples/terraform-aws-endpoint-example/main.tf

@@ -0,0 +1,131 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# PIN TERRAFORM VERSION TO >= 0.12
+# The examples have been upgraded to 0.12 syntax
+# ---------------------------------------------------------------------------------------------------------------------
+provider "aws" {
+  region     = var.region
+  access_key = "dummy"
+  secret_key = "dummy"
+
+  endpoints {
+    sts = "http://localhost:5000"
+    s3  = "http://localhost:5000"
+  }
+}
+
+terraform {
+  # This module is now only being tested with Terraform 0.13.x. However, to make upgrading easier, we are setting
+  # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it
+  # forwards compatible with 0.13.x code.
+  required_version = ">= 0.12.26"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY A S3 BUCKET WITH VERSIONING ENABLED INCLUDING TAGS TO A LOCAL ENDPOINT
+# See test/terraform_aws_endpoint_example_test.go for how to write automated tests for this code.
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Deploy and configure test S3 bucket with versioning and access log
+resource "aws_s3_bucket" "test_bucket" {
+  bucket = "${local.aws_account_id}-${var.tag_bucket_name}"
+
+  tags = {
+    Name        = var.tag_bucket_name
+    Environment = var.tag_bucket_environment
+  }
+}
+
+resource "aws_s3_bucket_logging" "test_bucket" {
+  bucket        = aws_s3_bucket.test_bucket.id
+  target_bucket = aws_s3_bucket.test_bucket_logs.id
+  target_prefix = "TFStateLogs/"
+}
+
+resource "aws_s3_bucket_versioning" "test_bucket" {
+  bucket = aws_s3_bucket.test_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_acl" "test_bucket" {
+  bucket = aws_s3_bucket.test_bucket.id
+  acl    = "private"
+}
+
+
+# Deploy S3 bucket to collect access logs for test bucket
+resource "aws_s3_bucket" "test_bucket_logs" {
+  bucket = "${local.aws_account_id}-${var.tag_bucket_name}-logs"
+
+  tags = {
+    Name        = "${local.aws_account_id}-${var.tag_bucket_name}-logs"
+    Environment = var.tag_bucket_environment
+  }
+
+  force_destroy = true
+}
+
+resource "aws_s3_bucket_acl" "test_bucket_logs" {
+  bucket = aws_s3_bucket.test_bucket_logs.id
+  acl    = "log-delivery-write"
+}
+
+# Configure bucket access policies
+
+resource "aws_s3_bucket_policy" "bucket_access_policy" {
+  count  = var.with_policy ? 1 : 0
+  bucket = aws_s3_bucket.test_bucket.id
+  policy = data.aws_iam_policy_document.s3_bucket_policy.json
+}
+
+data "aws_iam_policy_document" "s3_bucket_policy" {
+  statement {
+    effect = "Allow"
+    principals {
+      # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
+      # force an interpolation expression to be interpreted as a list by wrapping it
+      # in an extra set of list brackets. That form was supported for compatibility in
+      # v0.11, but is no longer supported in Terraform v0.12.
+      #
+      # If the expression in the following list itself returns a list, remove the
+      # brackets to avoid interpretation as a list of lists. If the expression
+      # returns a single list item then leave it as-is and remove this TODO comment.
+      identifiers = [local.aws_account_id]
+      type        = "AWS"
+    }
+    actions   = ["*"]
+    resources = ["${aws_s3_bucket.test_bucket.arn}/*"]
+  }
+
+  statement {
+    effect = "Deny"
+    principals {
+      identifiers = ["*"]
+      type        = "AWS"
+    }
+    actions   = ["*"]
+    resources = ["${aws_s3_bucket.test_bucket.arn}/*"]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values = [
+        "false",
+      ]
+    }
+  }
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# LOCALS
+# Used to represent any data that requires complex expressions/interpolations
+# ---------------------------------------------------------------------------------------------------------------------
+
+data "aws_caller_identity" "current" {
+}
+
+locals {
+  aws_account_id = data.aws_caller_identity.current.account_id
+}
+

examples/terraform-aws-endpoint-example/outputs.tf

@@ -0,0 +1,15 @@
+output "bucket_id" {
+  value = aws_s3_bucket.test_bucket.id
+}
+
+output "bucket_arn" {
+  value = aws_s3_bucket.test_bucket.arn
+}
+
+output "logging_target_bucket" {
+  value = aws_s3_bucket_logging.test_bucket.target_bucket
+}
+
+output "logging_target_prefix" {
+  value = aws_s3_bucket_logging.test_bucket.target_prefix
+}

examples/terraform-aws-endpoint-example/variables.tf

@@ -0,0 +1,40 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# You must provide a value for each of these parameters.
+# ---------------------------------------------------------------------------------------------------------------------
+variable "region" {
+  description = "The AWS region to deploy to"
+  type        = string
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These parameters have reasonable defaults.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "with_policy" {
+  description = "If set to `true`, the bucket will be created with a bucket policy."
+  type        = bool
+  default     = false
+}
+
+variable "tag_bucket_name" {
+  description = "The Name tag to set for the S3 Bucket."
+  type        = string
+  default     = "Test Bucket"
+}
+
+variable "tag_bucket_environment" {
+  description = "The Environment tag to set for the S3 Bucket."
+  type        = string
+  default     = "Test"
+}
+

modules/aws/auth.go

@@ -15,7 +15,8 @@ import (
 )
 
 const (
-	AuthAssumeRoleEnvVar = "TERRATEST_IAM_ROLE" // OS environment variable name through which Assume Role ARN may be passed for authentication
+	AuthAssumeRoleEnvVar = "TERRATEST_IAM_ROLE"            // OS environment variable name through which Assume Role ARN may be passed for authentication
+	CustomEndpointEnvVar = "TERRATEST_CUSTOM_AWS_ENDPOINT" // Custom endpoint to use as aws service
 )
 
 // NewAuthenticatedSession creates an AWS session following to standard AWS authentication workflow.
@@ -32,6 +33,11 @@ func NewAuthenticatedSession(region string) (*session.Session, error) {
 func NewAuthenticatedSessionFromDefaultCredentials(region string) (*session.Session, error) {
 	awsConfig := aws.NewConfig().WithRegion(region)
 
+	if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok {
+		awsConfig.WithEndpoint(customEndpoint)
+	}
+	awsConfig.WithEndpoint(os.Getenv(CustomEndpointEnvVar))
+
 	sessionOptions := session.Options{
 		Config:            *awsConfig,
 		SharedConfigState: session.SharedConfigEnable,
@@ -68,7 +74,13 @@ func NewAuthenticatedSessionFromRole(region string, roleARN string) (*session.Se
 // CreateAwsSessionFromRole returns a new AWS session after assuming the role
 // whose ARN is provided in roleARN.
 func CreateAwsSessionFromRole(region string, roleARN string) (*session.Session, error) {
-	sess, err := session.NewSession(aws.NewConfig().WithRegion(region))
+	awsConfig := aws.NewConfig().WithRegion(region)
+
+	if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok {
+		awsConfig.WithEndpoint(customEndpoint)
+	}
+
+	sess, err := session.NewSession(awsConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -86,8 +98,15 @@ func AssumeRole(sess *session.Session, roleARN string) *session.Session {
 // CreateAwsSessionWithCreds creates a new AWS session using explicit credentials. This is useful if you want to create an IAM User dynamically and
 // create an AWS session authenticated as the new IAM User.
 func CreateAwsSessionWithCreds(region string, accessKeyID string, secretAccessKey string) (*session.Session, error) {
-	creds := CreateAwsCredentials(accessKeyID, secretAccessKey)
-	return session.NewSession(aws.NewConfig().WithRegion(region).WithCredentials(creds))
+	awsConfig := aws.NewConfig().WithRegion(region)
+
+	if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok {
+		awsConfig.WithEndpoint(customEndpoint)
+	}
+
+	awsConfig.WithCredentials(CreateAwsCredentials(accessKeyID, secretAccessKey))
+
+	return session.NewSession(awsConfig)
 }
 
 // CreateAwsSessionWithMfa creates a new AWS session authenticated using an MFA token retrieved using the given STS client and MFA Device.
@@ -109,8 +128,15 @@ func CreateAwsSessionWithMfa(region string, stsClient *sts.STS, mfaDevice *iam.V
 	secretAccessKey := *output.Credentials.SecretAccessKey
 	sessionToken := *output.Credentials.SessionToken
 
-	creds := CreateAwsCredentialsWithSessionToken(accessKeyID, secretAccessKey, sessionToken)
-	return session.NewSession(aws.NewConfig().WithRegion(region).WithCredentials(creds))
+	awsConfig := aws.NewConfig().WithRegion(region)
+
+	if customEndpoint, ok := os.LookupEnv(CustomEndpointEnvVar); ok {
+		awsConfig.WithEndpoint(customEndpoint)
+	}
+
+	awsConfig.WithCredentials(CreateAwsCredentialsWithSessionToken(accessKeyID, secretAccessKey, sessionToken))
+
+	return session.NewSession(awsConfig)
 }
 
 // CreateAwsCredentials creates an AWS Credentials configuration with specific AWS credentials.