From 856e0b07c13f72d10edea71ff52f494d5d1f956a Mon Sep 17 00:00:00 2001 From: Dominique Vernier Date: Tue, 3 Oct 2023 19:52:23 -0400 Subject: [PATCH 1/3] Add extra collected resources and stats resources Signed-off-by: Dominique Vernier --- .gitignore | 4 +- cloudmapper.py | 2 +- collect_commands.yaml | 194 +++++++++++++++++++++-------------------- commands/collect.py | 5 +- commands/iam_report.py | 7 +- requirements.txt | 17 ++-- stats_config.yaml | 58 +++++++++--- 7 files changed, 163 insertions(+), 124 deletions(-) diff --git a/.gitignore b/.gitignore index ac50e35b2..1689e4939 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .vscode -config.json +.python-version +!config.json.demo +config.json.* config/ .DS_Store *.pyc diff --git a/cloudmapper.py b/cloudmapper.py index 32c652baa..a19517fa4 100755 --- a/cloudmapper.py +++ b/cloudmapper.py @@ -30,7 +30,7 @@ import pkgutil import importlib -__version__ = "2.10.1" +__version__ = "2.10.2" def show_help(commands): diff --git a/collect_commands.yaml b/collect_commands.yaml index d3b150c74..417dbc157 100644 --- a/collect_commands.yaml +++ b/collect_commands.yaml @@ -20,13 +20,13 @@ - Service: iam Request: get-role # Need to get IAM boundaries Parameters: - - Name: RoleName - Value: iam-get-account-authorization-details.json|.RoleDetailList[]?|.RoleName + - Name: RoleName + Value: iam-get-account-authorization-details.json|.RoleDetailList[]?|.RoleName - Service: iam Request: get-user # Needed to get IAM boundaries Parameters: - - Name: UserName - Value: iam-get-account-authorization-details.json|.UserDetailList[]?|.UserName + - Name: UserName + Value: iam-get-account-authorization-details.json|.UserDetailList[]?|.UserName - Service: iam Request: get-account-password-policy - Service: iam @@ -48,54 +48,54 @@ - Service: iam Request: get-saml-provider Parameters: - - Name: SAMLProviderArn - Value: iam-list-saml-providers.json|.SAMLProviderList[]?|.Arn + - Name: SAMLProviderArn + Value: iam-list-saml-providers.json|.SAMLProviderList[]?|.Arn - Service: iam Request: list-open-id-connect-providers - Service: iam Request: get-open-id-connect-provider Parameters: - - Name: OpenIDConnectProviderArn - Value: iam-list-open-id-connect-providers.json|.OpenIDConnectProviderList[]?|.Arn + - Name: OpenIDConnectProviderArn + Value: iam-list-open-id-connect-providers.json|.OpenIDConnectProviderList[]?|.Arn - Service: s3control Request: get-public-access-block Parameters: - - Name: AccountId - Value: sts-get-caller-identity.json|.Account + - Name: AccountId + Value: sts-get-caller-identity.json|.Account - Service: s3 Request: list-buckets - Service: s3 Request: get-bucket-acl Parameters: - - Name: Bucket - Value: s3-list-buckets.json|.Buckets[]?|.Name + - Name: Bucket + Value: s3-list-buckets.json|.Buckets[]?|.Name - Service: s3 Request: get-bucket-policy Parameters: - - Name: Bucket - Value: s3-list-buckets.json|.Buckets[]?|.Name + - Name: Bucket + Value: s3-list-buckets.json|.Buckets[]?|.Name - Service: s3 Request: get-bucket-logging Parameters: - - Name: Bucket - Value: s3-list-buckets.json|.Buckets[]?|.Name + - Name: Bucket + Value: s3-list-buckets.json|.Buckets[]?|.Name - Service: s3 Request: get-bucket-location Parameters: - - Name: Bucket - Value: s3-list-buckets.json|.Buckets[]?|.Name + - Name: Bucket + Value: s3-list-buckets.json|.Buckets[]?|.Name - Service: s3 Request: get-bucket-encryption Parameters: - - Name: Bucket - Value: s3-list-buckets.json|.Buckets[]?|.Name + - Name: Bucket + Value: s3-list-buckets.json|.Buckets[]?|.Name - Service: route53 Request: list-hosted-zones - Service: route53 Request: list-resource-record-sets Parameters: - - Name: HostedZoneId - Value: route53-list-hosted-zones.json|.HostedZones[]?|[.Id,.Name] + - Name: HostedZoneId + Value: route53-list-hosted-zones.json|.HostedZones[]?|[.Id,.Name] - Service: route53domains Request: list-domains - Service: ec2 @@ -112,13 +112,15 @@ Request: describe-addresses - Service: ec2 Request: describe-internet-gateways +- Service: ec2 + Request: describe-addresses - Service: cloudtrail Request: describe-trails - Service: cloudtrail Request: get-event-selectors Parameters: - - Name: TrailName - Value: cloudtrail-describe-trails.json|.trailList[].TrailARN + - Name: TrailName + Value: cloudtrail-describe-trails.json|.trailList[].TrailARN - Service: rds Request: describe-db-instances - Service: rds @@ -126,49 +128,49 @@ - Service: rds Request: describe-db-snapshot-attributes Parameters: - - Name: DBSnapshotIdentifier - Value: rds-describe-db-snapshots.json|.DBSnapshots[]?|.DBSnapshotIdentifier + - Name: DBSnapshotIdentifier + Value: rds-describe-db-snapshots.json|.DBSnapshots[]?|.DBSnapshotIdentifier - Service: rds Request: list-tags-for-resource Parameters: - - Name: ResourceName - Value: rds-describe-db-instances.json|.DBInstances[]?|.DBInstanceArn + - Name: ResourceName + Value: rds-describe-db-instances.json|.DBInstances[]?|.DBInstanceArn - Service: elb Request: describe-load-balancers - Service: elb Request: describe-load-balancer-attributes Parameters: - - Name: LoadBalancerName - Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[].LoadBalancerName + - Name: LoadBalancerName + Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[].LoadBalancerName - Service: elb Request: describe-load-balancer-policies - Service: elb Request: describe-tags Parameters: - - Name: LoadBalancerNames - Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|[[.LoadBalancerName]] + - Name: LoadBalancerNames + Value: elb-describe-load-balancers.json|.LoadBalancerDescriptions[]?|[[.LoadBalancerName]] - Service: elbv2 Request: describe-load-balancers - Service: elbv2 Request: describe-target-groups Parameters: - - Name: LoadBalancerArn - Value: elbv2-describe-load-balancers.json|.LoadBalancers[].LoadBalancerArn + - Name: LoadBalancerArn + Value: elbv2-describe-load-balancers.json|.LoadBalancers[].LoadBalancerArn - Service: elbv2 Request: describe-target-health Parameters: - - Name: TargetGroupArn - Value: elbv2-describe-target-groups/*|.TargetGroups[].TargetGroupArn + - Name: TargetGroupArn + Value: elbv2-describe-target-groups/*|.TargetGroups[].TargetGroupArn - Service: elbv2 Request: describe-load-balancer-attributes Parameters: - - Name: LoadBalancerArn - Value: elbv2-describe-load-balancers.json|.LoadBalancers[].LoadBalancerArn + - Name: LoadBalancerArn + Value: elbv2-describe-load-balancers.json|.LoadBalancers[].LoadBalancerArn - Service: elbv2 Request: describe-tags Parameters: - - Name: ResourceArns - Value: elbv2-describe-load-balancers.json|.LoadBalancers[]?|[[.LoadBalancerArn]] + - Name: ResourceArns + Value: elbv2-describe-load-balancers.json|.LoadBalancers[]?|[[.LoadBalancerArn]] - Service: redshift Request: describe-clusters - Service: redshift @@ -178,17 +180,17 @@ - Service: sqs Request: get-queue-attributes Parameters: - - Name: QueueUrl - Value: sqs-list-queues.json|.QueueUrls[]? - - Name: AttributeNames - Value: [All] + - Name: QueueUrl + Value: sqs-list-queues.json|.QueueUrls[]? + - Name: AttributeNames + Value: [All] - Service: sns Request: list-topics - Service: sns Request: get-topic-attributes Parameters: - - Name: TopicArn - Value: sns-list-topics.json|.Topics[]?|.TopicArn + - Name: TopicArn + Value: sns-list-topics.json|.Topics[]?|.TopicArn - Service: ec2 Request: describe-security-groups - Service: ec2 @@ -206,13 +208,13 @@ - Service: cloudformation Request: get-template Parameters: - - Name: StackName - Value: cloudformation-describe-stacks.json|.Stacks[]?|.StackName + - Name: StackName + Value: cloudformation-describe-stacks.json|.Stacks[]?|.StackName - Service: cloudformation Request: describe-stack-resources Parameters: - - Name: StackName - Value: cloudformation-describe-stacks.json|.Stacks[]?|.StackName + - Name: StackName + Value: cloudformation-describe-stacks.json|.Stacks[]?|.StackName - Service: cloudfront Request: list-distributions - Service: cloudsearch @@ -220,8 +222,8 @@ - Service: cloudsearch Request: describe-service-access-policies Parameters: - - Name: DomainName - Value: cloudsearch-describe-domains.json|.DomainStatusList[]?|.DomainName + - Name: DomainName + Value: cloudsearch-describe-domains.json|.DomainStatusList[]?|.DomainName - Service: cloudwatch Request: describe-alarms - Service: config @@ -233,9 +235,9 @@ - Service: ec2 Request: describe-images Parameters: - - Name: Owners - Value: - - "self" + - Name: Owners + Value: + - "self" - Service: ec2 Request: describe-network-acls - Service: ec2 @@ -245,16 +247,16 @@ - Service: ec2 Request: describe-snapshots Parameters: - - Name: OwnerIds - Value: - - "self" + - Name: OwnerIds + Value: + - "self" - Service: ec2 Request: describe-snapshot-attribute Parameters: - - Name: SnapshotId - Value: ec2-describe-snapshots.json|.Snapshots[]?|.SnapshotId - - Name: Attribute - Value: 'createVolumePermission' + - Name: SnapshotId + Value: ec2-describe-snapshots.json|.Snapshots[]?|.SnapshotId + - Name: Attribute + Value: 'createVolumePermission' - Service: ec2 Request: describe-vpc-endpoint-connections - Service: ec2 @@ -268,8 +270,8 @@ - Service: ecr Request: get-repository-policy Parameters: - - Name: repositoryName - Value: ecr-describe-repositories.json|.repositories[]|.repositoryName + - Name: repositoryName + Value: ecr-describe-repositories.json|.repositories[]|.repositoryName - Service: elasticache Request: describe-cache-clusters - Service: elasticbeanstalk @@ -281,8 +283,8 @@ - Service: es Request: describe-elasticsearch-domain Parameters: - - Name: DomainName - Value: es-list-domain-names.json|.DomainNames[]?|.DomainName + - Name: DomainName + Value: es-list-domain-names.json|.DomainNames[]?|.DomainName - Service: events Request: describe-event-bus - Service: events @@ -292,65 +294,65 @@ - Service: firehose Request: describe-delivery-stream Parameters: - - Name: DeliveryStreamName - Value: firehose-list-delivery-streams.json|.DeliveryStreamNames[]? + - Name: DeliveryStreamName + Value: firehose-list-delivery-streams.json|.DeliveryStreamNames[]? - Service: glacier Request: list-vaults Parameters: - - Name: accountId - Value: "-" + - Name: accountId + Value: "-" - Service: glacier Request: get-vault-access-policy Parameters: - - Name: vaultName - Value: glacier-list-vaults.json|.VaultList[]?|.VaultName - - Name: accountId - Value: "-" + - Name: vaultName + Value: glacier-list-vaults.json|.VaultList[]?|.VaultName + - Name: accountId + Value: "-" - Service: kms Request: list-keys - Service: kms Request: list-grants Parameters: - - Name: KeyId - Value: kms-list-keys.json|.Keys[]?|.KeyId + - Name: KeyId + Value: kms-list-keys.json|.Keys[]?|.KeyId - Service: kms Request: list-key-policies Parameters: - - Name: KeyId - Value: kms-list-keys.json|.Keys[]?|.KeyId + - Name: KeyId + Value: kms-list-keys.json|.Keys[]?|.KeyId - Service: kms Request: get-key-policy Parameters: - - Name: KeyId - Value: kms-list-keys.json|.Keys[]?|.KeyId - - Name: PolicyName - Value: 'default' + - Name: KeyId + Value: kms-list-keys.json|.Keys[]?|.KeyId + - Name: PolicyName + Value: 'default' - Service: kms Request: get-key-rotation-status Parameters: - - Name: KeyId - Value: kms-list-keys.json|.Keys[]?|.KeyId + - Name: KeyId + Value: kms-list-keys.json|.Keys[]?|.KeyId - Service: lambda Request: list-functions - Service: lambda Request: get-policy Parameters: - - Name: FunctionName - Value: lambda-list-functions.json|.Functions[]?|.FunctionName + - Name: FunctionName + Value: lambda-list-functions.json|.Functions[]?|.FunctionName - Service: lambda Request: list-layers - Service: lambda Request: list-layer-versions Parameters: - - Name: LayerName - Value: lambda-list-layers.json|.Layers[]?|.LayerName + - Name: LayerName + Value: lambda-list-layers.json|.Layers[]?|.LayerName - Service: ecs Request: list-clusters - Service: ecs Request: list-tasks Parameters: - - Name: cluster - Value: ecs-list-clusters.json|.clusterArns[] + - Name: cluster + Value: ecs-list-clusters.json|.clusterArns[] - Service: ecs Request: describe-tasks Custom_collection: True @@ -359,8 +361,8 @@ - Service: eks Request: describe-cluster Parameters: - - Name: name - Value: eks-list-clusters.json|.clusters[] + - Name: name + Value: eks-list-clusters.json|.clusters[] - Service: logs Request: describe-destinations - Service: logs @@ -393,8 +395,8 @@ - Service: guardduty Request: get-detector Parameters: - - Name: DetectorId - Value: guardduty-list-detectors.json|.DetectorIds[]?|. + - Name: DetectorId + Value: guardduty-list-detectors.json|.DetectorIds[]?|. - Service: organizations Request: describe-organization - Service: organizations @@ -406,8 +408,8 @@ - Service: secretsmanager Request: get-resource-policy Parameters: - - Name: SecretId - Value: secretsmanager-list-secrets.json|.SecretList[]?|.Name + - Name: SecretId + Value: secretsmanager-list-secrets.json|.SecretList[]?|.Name - Service: route53 Request: list-hosted-zones-by-vpc Custom_collection: True diff --git a/commands/collect.py b/commands/collect.py index 073914f4e..34bb3f561 100644 --- a/commands/collect.py +++ b/commands/collect.py @@ -234,9 +234,6 @@ def collect(arguments): regions_filter = None if len(arguments.regions_filter) > 0: regions_filter = arguments.regions_filter.lower().split(",") - # Force include of default region -- seems to be required - if default_region not in regions_filter: - regions_filter.append(default_region) session_data = {"region_name": default_region} @@ -295,6 +292,8 @@ def collect(arguments): r for r in region_list["Regions"] if r["RegionName"] in regions_filter ] region_list["Regions"] = filtered_regions + else: + region_list["Regions"] = [ default_region ] with open("account-data/{}/describe-regions.json".format(account_dir), "w+") as f: f.write(json.dumps(region_list, indent=4, sort_keys=True)) diff --git a/commands/iam_report.py b/commands/iam_report.py index 0edd5c3aa..c912f8a9e 100644 --- a/commands/iam_report.py +++ b/commands/iam_report.py @@ -85,12 +85,15 @@ def get_access_advisor(region, principal_stats, json_account_auth_details, args) ]: stats = {} stats["auth"] = principal_auth - job_id = get_parameter_file( + parameter_file = get_parameter_file( region, "iam", "generate-service-last-accessed-details", principal_auth["Arn"], - )["JobId"] + ) + if not parameter_file: + continue + job_id = parameter_file["JobId"] json_last_access_details = get_parameter_file( region, "iam", "get-service-last-accessed-details", job_id ) diff --git a/requirements.txt b/requirements.txt index 69b801796..b679a0239 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ autoflake==1.4 autopep8==1.6.0 boto3==1.19.10 botocore==1.22.10 -certifi==2021.10.8 -chardet==4.0.0 +certifi==2023.7.22 +chardet==3.0.4 charset-normalizer==2.0.7 coverage==6.1.1 cycler==0.11.0 @@ -23,27 +23,26 @@ mccabe==0.6.1 mock==4.0.3 netaddr==0.8.0 nose==1.3.7 -numpy==1.21.3 +numpy==1.25.2 pandas==1.3.4 parliament==1.5.2 -Pillow==9.0.1 +Pillow==9.3.0 platformdirs==2.4.0 policyuniverse==1.4.0.20210819 pycodestyle==2.8.0 pyflakes==2.4.0 -pyjq==2.4.0 +pyjq==2.6.0 pylint==2.11.1 pyparsing==3.0.4 python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 -requests==2.26.0 +requests==2.31.0 s3transfer==0.5.0 -scipy==1.7.1 +scipy==1.11.2 seaborn==0.11.2 six==1.16.0 toml==0.10.2 -typed-ast==1.4.3 -typing-extensions==3.10.0.2 +typing-extensions==4.6.0 urllib3==1.26.7 wrapt==1.13.3 diff --git a/stats_config.yaml b/stats_config.yaml index e0880bc4f..189bbec4d 100644 --- a/stats_config.yaml +++ b/stats_config.yaml @@ -1,9 +1,24 @@ - name: S3 buckets query: .Buckets|length source: s3-list-buckets -- name: EC2 instances +- name: EC2 Elastic IPs + query: '.Addresses | length' + source: ec2-describe-addresses +- name: EC2 Instances query: '[.Reservations[]?.Instances[]?|select(.State.Name == "running")]|length' source: ec2-describe-instances +- name: EC2 Network Interfaces + query: '.InternetGateways|length' + source: ec2-describe-network-interfaces +- name: EC2 Subnets + query: .Subnets|length + source: ec2-describe-subnets +- name: EC2 Volumes + query: .Volumes|length + source: ec2-describe-volumes +- name: EC2 VPCs + query: .Vpcs|length + source: ec2-describe-vpcs - name: ELBs query: .LoadBalancerDescriptions|length source: elb-describe-load-balancers @@ -13,6 +28,9 @@ - name: RDS instances query: .DBInstances|length source: rds-describe-db-instances +- name: RDS Snapshots + query: .DBSnapshots|length + source: rds-describe-db-snapshots - name: Redshift clusters query: .Clusters|length source: redshift-describe-clusters @@ -72,6 +90,26 @@ query: .Images|length source: ec2-describe-images verbose: true +- name: EC2 Availability Zone + query: '.AvailabilityZones | length' + source: ec2-describe-addresses + verbose: true +- name: EC2 snapshots + query: .Snapshots|length + source: ec2-describe-snapshots + verbose: true +- name: EC2 Security Groups + query: .SecurityGroups|length + source: ec2-describe-security-groups + verbose: true +- name: EC2 VPCEndpoints + query: .VPCEndpoints|length + source: ec2-describe-vpc-endpoints + verbose: true +- name: EC2 Internet Gateways + query: '[.InternetGateways[]?.Attachments[]?|select(.State == "available")]|length' + source: describe-internet-gateways + verbose: true - name: Network ACLs query: .NetworkAcls|length source: ec2-describe-network-acls @@ -80,10 +118,6 @@ query: .RouteTables|length source: ec2-describe-route-tables verbose: true -- name: EC2 snapshots - query: .Snapshots|length - source: ec2-describe-snapshots - verbose: true - name: VPC endpoints query: .VpcEndpointConnections|length source: ec2-describe-vpc-endpoint-connections @@ -128,15 +162,15 @@ query: .logGroups|length source: logs-describe-log-groups verbose: true -- name: ACM Certificates +- name: ACM Certificates query: .CertificateSummaryList|length - source: acm-list-certificates + source: acm-list-certificates verbose: true -- name: DynamoDB Tables +- name: DynamoDB Tables query: .TableNames|length - source: dynamodb-list-tables + source: dynamodb-list-tables verbose: true -- name: Internet Gateways - query: '[.InternetGateways[]?.Attachments[]?|select(.State == "available")]|length' - source: describe-internet-gateways +- name: ELBs Policies + query: .PolicyDescriptions|length + source: elb-describe-load-balancer-policies verbose: true From f47dac90fef8bf09ddcbeae87ad29b1591d7d9d9 Mon Sep 17 00:00:00 2001 From: Dominique Vernier Date: Wed, 4 Oct 2023 09:50:12 -0400 Subject: [PATCH 2/3] Add exclude_universale_services Signed-off-by: Dominique Vernier --- commands/collect.py | 457 +++++++++++++++++++++++--------------------- 1 file changed, 243 insertions(+), 214 deletions(-) diff --git a/commands/collect.py b/commands/collect.py index 34bb3f561..d745d3226 100644 --- a/commands/collect.py +++ b/commands/collect.py @@ -10,6 +10,7 @@ import yaml import pyjq import urllib.parse +import copy from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError from shared.common import get_account, custom_serializer from botocore.config import Config @@ -223,19 +224,19 @@ def collect(arguments): make_directory("account-data/{}".format(account_dir)) # Identify the default region used by global services such as IAM - default_region = os.environ.get("AWS_REGION", "us-east-1") - if "gov-" in default_region: - default_region = "us-gov-west-1" - elif "cn-" in default_region: - default_region = "cn-north-1" + default_region_name = os.environ.get("AWS_REGION", "us-east-1") + if "gov-" in default_region_name: + default_region_name = "us-gov-west-1" + elif "cn-" in default_region_name: + default_region_name = "cn-north-1" else: - default_region = "us-east-1" + default_region_name = "us-east-1" regions_filter = None if len(arguments.regions_filter) > 0: regions_filter = arguments.regions_filter.lower().split(",") - session_data = {"region_name": default_region} + session_data = {"region_name": default_region_name} if arguments.profile_name: session_data["profile_name"] = arguments.profile_name @@ -286,17 +287,22 @@ def collect(arguments): print("* Getting region names", flush=True) ec2 = session.client("ec2") region_list = ec2.describe_regions() + + default_region_array = [ r for r in region_list["Regions"] if r["RegionName"] == default_region_name ] + default_region = default_region_array[0] + region_list_services = copy.deepcopy(region_list) if regions_filter is not None: filtered_regions = [ r for r in region_list["Regions"] if r["RegionName"] in regions_filter ] region_list["Regions"] = filtered_regions - else: - region_list["Regions"] = [ default_region ] + region_list_services = copy.deepcopy(region_list) + if not arguments.exclude_universal_services and default_region_name not in regions_filter: + region_list_services["Regions"].append(default_region) with open("account-data/{}/describe-regions.json".format(account_dir), "w+") as f: - f.write(json.dumps(region_list, indent=4, sort_keys=True)) + f.write(json.dumps(region_list_services, indent=4, sort_keys=True)) print("* Creating directory for each region name", flush=True) for region in region_list["Regions"]: @@ -331,11 +337,19 @@ def collect(arguments): parameters = {} for region in region_list["Regions"]: - dynamic_parameter = None + region_service = copy.deepcopy(region) # Only call universal services in default region if runner["Service"] in universal_services: - if region["RegionName"] != default_region: - continue + if region["RegionName"] != default_region_name: + if not arguments.exclude_universal_services: + region_service["RegionName"] = default_region_name + make_directory( + "account-data/{}/{}".format( + account_dir, region_service.get("RegionName", "Unknown") + ) + ) + else: + continue elif region["RegionName"] not in session.get_available_regions( runner["Service"] ): @@ -345,232 +359,241 @@ def collect(arguments): ) ) continue - handler = session.client( - runner["Service"], - region_name=region["RegionName"], - config=Config(retries={"max_attempts": arguments.max_attempts}), - ) - filepath = "account-data/{}/{}/{}-{}".format( - account_dir, region["RegionName"], runner["Service"], runner["Request"] + summary_rep, cont = gather(runner, region_service, arguments, session, account_dir, parameters) + summary.append(summary_rep) + if cont: + continue + + # Print summary + print("--------------------------------------------------------------------") + failures = [] + for call_summary in summary: + if "exception" in call_summary: + failures.append(call_summary) + + print("Summary: {} APIs called. {} errors".format(len(summary), len(failures))) + if len(failures) > 0: + print("Failures:") + for call_summary in failures: + print( + " {}.{}({}): {}".format( + call_summary["service"], + call_summary["action"], + call_summary["parameters"], + call_summary["exception"], + ) ) + # Ensure errors can be detected + exit(-1) - method_to_call = snakecase(runner["Request"]) +def gather(runner, region, arguments, session, account_dir, parameters): + dynamic_parameter = None + summary = [] + handler = session.client( + runner["Service"], + region_name=region["RegionName"], + config=Config(retries={"max_attempts": arguments.max_attempts}), + ) - # Identify any parameters - if runner.get("Parameters", False): - for parameter in runner["Parameters"]: - parameters[parameter["Name"]] = parameter["Value"] + filepath = "account-data/{}/{}/{}-{}".format( + account_dir, region["RegionName"], runner["Service"], runner["Request"] + ) - # Look for any dynamic values (ones that jq parse a file) - if "|" in parameter["Value"]: - dynamic_parameter = parameter["Name"] + method_to_call = snakecase(runner["Request"]) - if runner.get("Custom_collection", False): - # The data to collect for this function is too complicated for my existing code, - # so I have to write custom code. - if runner["Service"] == "ecs" and runner["Request"] == "describe-tasks": - action_path = filepath - make_directory(action_path) + # Identify any parameters + if runner.get("Parameters", False): + for parameter in runner["Parameters"]: + parameters[parameter["Name"]] = parameter["Value"] - # Read the ecs-list-clusters.json file - list_clusters_file = "account-data/{}/{}/{}".format( - account_dir, region["RegionName"], "ecs-list-clusters.json" - ) + # Look for any dynamic values (ones that jq parse a file) + if "|" in parameter["Value"]: + dynamic_parameter = parameter["Name"] + + if runner.get("Custom_collection", False): + # The data to collect for this function is too complicated for my existing code, + # so I have to write custom code. + if runner["Service"] == "ecs" and runner["Request"] == "describe-tasks": + action_path = filepath + make_directory(action_path) - if os.path.isfile(list_clusters_file): - with open(list_clusters_file, "r") as f: - list_clusters = json.load(f) + # Read the ecs-list-clusters.json file + list_clusters_file = "account-data/{}/{}/{}".format( + account_dir, region["RegionName"], "ecs-list-clusters.json" + ) + + if os.path.isfile(list_clusters_file): + with open(list_clusters_file, "r") as f: + list_clusters = json.load(f) + + # For each cluster, read the `ecs list-tasks` + for clusterArn in list_clusters["clusterArns"]: + cluster_path = ( + action_path + + "/" + + urllib.parse.quote_plus(clusterArn) + ) + make_directory(cluster_path) + + list_tasks_file = "account-data/{}/{}/{}/{}".format( + account_dir, + region["RegionName"], + "ecs-list-tasks", + urllib.parse.quote_plus(clusterArn), + ) - # For each cluster, read the `ecs list-tasks` - for clusterArn in list_clusters["clusterArns"]: - cluster_path = ( + with open(list_tasks_file, "r") as f2: + list_tasks = json.load(f2) + + # For each task, call `ecs describe-tasks` using the `cluster` and `task` as arguments + for taskArn in list_tasks["taskArns"]: + outputfile = ( action_path + "/" + urllib.parse.quote_plus(clusterArn) + + "/" + + urllib.parse.quote_plus(taskArn) ) - make_directory(cluster_path) - list_tasks_file = "account-data/{}/{}/{}/{}".format( - account_dir, - region["RegionName"], - "ecs-list-tasks", - urllib.parse.quote_plus(clusterArn), + call_parameters = {} + call_parameters["cluster"] = clusterArn + call_parameters["tasks"] = [taskArn] + + call_function( + outputfile, + handler, + method_to_call, + call_parameters, + runner.get("Check", None), + summary, ) + elif ( + runner["Service"] == "route53" + and runner["Request"] == "list-hosted-zones-by-vpc" + ): + action_path = filepath + make_directory(action_path) - with open(list_tasks_file, "r") as f2: - list_tasks = json.load(f2) - - # For each task, call `ecs describe-tasks` using the `cluster` and `task` as arguments - for taskArn in list_tasks["taskArns"]: - outputfile = ( - action_path - + "/" - + urllib.parse.quote_plus(clusterArn) - + "/" - + urllib.parse.quote_plus(taskArn) - ) - - call_parameters = {} - call_parameters["cluster"] = clusterArn - call_parameters["tasks"] = [taskArn] - - call_function( - outputfile, - handler, - method_to_call, - call_parameters, - runner.get("Check", None), - summary, - ) - elif ( - runner["Service"] == "route53" - and runner["Request"] == "list-hosted-zones-by-vpc" - ): - action_path = filepath - make_directory(action_path) - - # Read the regions file - regions_file = "account-data/{}/{}".format( - account_dir, "describe-regions.json" + # Read the regions file + regions_file = "account-data/{}/{}".format( + account_dir, "describe-regions.json" + ) + with open(regions_file, "r") as f: + describe_regions = json.load(f) + + # For each region + for collect_region in describe_regions["Regions"]: + cluster_path = ( + action_path + + "/" + + urllib.parse.quote_plus(collect_region["RegionName"]) ) - with open(regions_file, "r") as f: - describe_regions = json.load(f) - - # For each region - for collect_region in describe_regions["Regions"]: - cluster_path = ( - action_path - + "/" - + urllib.parse.quote_plus(collect_region["RegionName"]) - ) - make_directory(cluster_path) + make_directory(cluster_path) - # Read the VPC file - describe_vpcs_file = "account-data/{}/{}/{}".format( - account_dir, - collect_region["RegionName"], - "ec2-describe-vpcs.json", - ) + # Read the VPC file + describe_vpcs_file = "account-data/{}/{}/{}".format( + account_dir, + collect_region["RegionName"], + "ec2-describe-vpcs.json", + ) - if os.path.isfile(describe_vpcs_file): - with open(describe_vpcs_file, "r") as f2: - describe_vpcs = json.load(f2) - - for vpc in describe_vpcs["Vpcs"]: - outputfile = ( - action_path - + "/" - + urllib.parse.quote_plus( - collect_region["RegionName"] - ) - + "/" - + urllib.parse.quote_plus(vpc["VpcId"]) - ) - - call_parameters = {} - call_parameters["VPCRegion"] = collect_region[ - "RegionName" - ] - call_parameters["VPCId"] = vpc["VpcId"] - call_function( - outputfile, - handler, - method_to_call, - call_parameters, - runner.get("Check", None), - summary, - ) - - elif dynamic_parameter is not None: - # Set up directory for the dynamic value - make_directory(filepath) - - # The dynamic parameter must always be the first value - parameter_file = parameters[dynamic_parameter].split("|")[0] - parameter_file = "account-data/{}/{}/{}".format( - account_dir, region["RegionName"], parameter_file - ) + if os.path.isfile(describe_vpcs_file): + with open(describe_vpcs_file, "r") as f2: + describe_vpcs = json.load(f2) - # Get array if a globbing pattern is used (ex. "*.json") - parameter_files = glob.glob(parameter_file) - - for parameter_file in parameter_files: - if not os.path.isfile(parameter_file): - # The file where parameters are obtained from does not exist - # Need to manually add the failure to our list of calls made as this failure - # occurs before the call is attempted. - call_summary = { - "service": handler.meta.service_model.service_name, - "action": method_to_call, - "parameters": parameters, - "exception": "Parameter file does not exist: {}".format( - parameter_file - ), - } - summary.append(call_summary) - print( - " The file where parameters are obtained from does not exist: {}".format( - parameter_file - ), - flush=True, - ) - continue + for vpc in describe_vpcs["Vpcs"]: + outputfile = ( + action_path + + "/" + + urllib.parse.quote_plus( + collect_region["RegionName"] + ) + + "/" + + urllib.parse.quote_plus(vpc["VpcId"]) + ) - with open(parameter_file, "r") as f: - parameter_values = json.load(f) - pyjq_parse_string = "|".join( - parameters[dynamic_parameter].split("|")[1:] - ) - for parameter in pyjq.all(pyjq_parse_string, parameter_values): - filename = get_filename_from_parameter(parameter) - identifier = get_identifier_from_parameter(parameter) - call_parameters = dict(parameters) - call_parameters[dynamic_parameter] = identifier - - outputfile = "{}/{}".format(filepath, filename) - - call_function( - outputfile, - handler, - method_to_call, - call_parameters, - runner.get("Check", None), - summary, - ) - else: - filepath = filepath + ".json" - call_function( - filepath, - handler, - method_to_call, - parameters, - runner.get("Check", None), - summary, - ) + call_parameters = {} + call_parameters["VPCRegion"] = collect_region[ + "RegionName" + ] + call_parameters["VPCId"] = vpc["VpcId"] + call_function( + outputfile, + handler, + method_to_call, + call_parameters, + runner.get("Check", None), + summary, + ) - # Print summary - print("--------------------------------------------------------------------") - failures = [] - for call_summary in summary: - if "exception" in call_summary: - failures.append(call_summary) + elif dynamic_parameter is not None: + # Set up directory for the dynamic value + make_directory(filepath) - print("Summary: {} APIs called. {} errors".format(len(summary), len(failures))) - if len(failures) > 0: - print("Failures:") - for call_summary in failures: - print( - " {}.{}({}): {}".format( - call_summary["service"], - call_summary["action"], - call_summary["parameters"], - call_summary["exception"], + # The dynamic parameter must always be the first value + parameter_file = parameters[dynamic_parameter].split("|")[0] + parameter_file = "account-data/{}/{}/{}".format( + account_dir, region["RegionName"], parameter_file + ) + + # Get array if a globbing pattern is used (ex. "*.json") + parameter_files = glob.glob(parameter_file) + + for parameter_file in parameter_files: + if not os.path.isfile(parameter_file): + # The file where parameters are obtained from does not exist + # Need to manually add the failure to our list of calls made as this failure + # occurs before the call is attempted. + call_summary = { + "service": handler.meta.service_model.service_name, + "action": method_to_call, + "parameters": parameters, + "exception": "Parameter file does not exist: {}".format( + parameter_file + ), + } + summary.append(call_summary) + print( + " The file where parameters are obtained from does not exist: {}".format( + parameter_file + ), + flush=True, ) - ) - # Ensure errors can be detected - exit(-1) + return summary, True + with open(parameter_file, "r") as f: + parameter_values = json.load(f) + pyjq_parse_string = "|".join( + parameters[dynamic_parameter].split("|")[1:] + ) + for parameter in pyjq.all(pyjq_parse_string, parameter_values): + filename = get_filename_from_parameter(parameter) + identifier = get_identifier_from_parameter(parameter) + call_parameters = dict(parameters) + call_parameters[dynamic_parameter] = identifier + + outputfile = "{}/{}".format(filepath, filename) + + call_function( + outputfile, + handler, + method_to_call, + call_parameters, + runner.get("Check", None), + summary, + ) + else: + filepath = filepath + ".json" + call_function( + filepath, + handler, + method_to_call, + parameters, + runner.get("Check", None), + summary, + ) + return summary, False def run(arguments): parser = argparse.ArgumentParser() @@ -596,6 +619,12 @@ def run(arguments): help="Remove any existing local, previously collected data for the account before gathering", action="store_true", ) + parser.add_argument( + "--exclude-universal-services", + help="Do not gather the universal services for region different than default region", + action="store_true", + dest="exclude_universal_services" + ) parser.add_argument( "--max-attempts", help="Override Botocore config max_attempts (default 4)", From 2afeab050cb091496d5fa8e08ee502911227330c Mon Sep 17 00:00:00 2001 From: Dominique Vernier Date: Wed, 4 Oct 2023 12:44:13 -0400 Subject: [PATCH 3/3] Add secrets and order stats Signed-off-by: Dominique Vernier --- stats_config.yaml | 257 +++++++++++++++++++++++----------------------- 1 file changed, 130 insertions(+), 127 deletions(-) diff --git a/stats_config.yaml b/stats_config.yaml index 189bbec4d..c090132f1 100644 --- a/stats_config.yaml +++ b/stats_config.yaml @@ -1,90 +1,37 @@ -- name: S3 buckets - query: .Buckets|length - source: s3-list-buckets -- name: EC2 Elastic IPs - query: '.Addresses | length' - source: ec2-describe-addresses -- name: EC2 Instances - query: '[.Reservations[]?.Instances[]?|select(.State.Name == "running")]|length' - source: ec2-describe-instances -- name: EC2 Network Interfaces - query: '.InternetGateways|length' - source: ec2-describe-network-interfaces -- name: EC2 Subnets - query: .Subnets|length - source: ec2-describe-subnets -- name: EC2 Volumes - query: .Volumes|length - source: ec2-describe-volumes -- name: EC2 VPCs - query: .Vpcs|length - source: ec2-describe-vpcs -- name: ELBs - query: .LoadBalancerDescriptions|length - source: elb-describe-load-balancers -- name: ELBv2s - query: .LoadBalancers|length - source: elbv2-describe-load-balancers -- name: RDS instances - query: .DBInstances|length - source: rds-describe-db-instances -- name: RDS Snapshots - query: .DBSnapshots|length - source: rds-describe-db-snapshots -- name: Redshift clusters - query: .Clusters|length - source: redshift-describe-clusters -- name: ElasticSearch domains - query: .DomainNames|length - source: es-list-domain-names -- name: Elasticache clusters - query: .CacheClusters|length - source: elasticache-describe-cache-clusters -- name: SNS topics - query: .Topics|length - source: sns-list-topics -- name: SQS queues - query: .QueueUrls|length - source: sqs-list-queues +- name: ACM Certificates + query: .CertificateSummaryList|length + source: acm-list-certificates + verbose: true +- name: Autoscaling groups + query: .AutoScalingGroups|length + source: autoscaling-describe-auto-scaling-groups +- name: CloudFormation stacks + query: .Stacks|length + source: cloudformation-describe-stacks + verbose: true - name: CloudFronts query: .DistributionList.Items|length source: cloudfront-list-distributions region: us-east-1 -- name: Autoscaling groups - query: .AutoScalingGroups|length - source: autoscaling-describe-auto-scaling-groups -- name: ElasticBeanstalks - query: .Applications|length - source: elasticbeanstalk-describe-applications -- name: Firehose streams - query: .DeliveryStreamNames|length - source: firehose-list-delivery-streams -- name: Glacier vaults - query: .VaultList|length - source: glacier-list-vaults -- name: KMS keys - query: .Keys|length - source: kms-list-keys -- name: Lambda functions - query: .Functions|length - source: lambda-list-functions -- name: Glue Jobs - query: .Jobs|length - source: glue-get-jobs -- name: Glue Triggers - query: .Triggers|length - source: glue-get-triggers -# - name: Kafka clusters -# query: .ClusterInfoList|length -# source: kafka-list-clusters -# Verbose resources -- name: Route53 hosted zones - query: .HostedZones|length - source: route53-list-hosted-zones +- name: CloudSearch domains + query: .DomainStatusList|length + source: cloudsearch-describe-domains verbose: true -- name: Route53 domains - query: .Domains|length - source: route53domains-list-domains +- name: Cloudwatch alarms + query: .MetricAlarms|length + source: cloudwatch-describe-alarms + verbose: true +- name: Config rules + query: .ConfigRules|length + source: config-describe-config-rules + verbose: true +- name: DirectConnects + query: .connections|length + source: directconnect-describe-connections + verbose: true +- name: DynamoDB Tables + query: .TableNames|length + source: dynamodb-list-tables verbose: true - name: EC2 AMIs query: .Images|length @@ -94,83 +41,139 @@ query: '.AvailabilityZones | length' source: ec2-describe-addresses verbose: true -- name: EC2 snapshots - query: .Snapshots|length - source: ec2-describe-snapshots - verbose: true -- name: EC2 Security Groups - query: .SecurityGroups|length - source: ec2-describe-security-groups - verbose: true -- name: EC2 VPCEndpoints - query: .VPCEndpoints|length - source: ec2-describe-vpc-endpoints - verbose: true +- name: EC2 Elastic IPs + query: '.Addresses | length' + source: ec2-describe-addresses +- name: EC2 Instances + query: '[.Reservations[]?.Instances[]?|select(.State.Name == "running")]|length' + source: ec2-describe-instances - name: EC2 Internet Gateways query: '[.InternetGateways[]?.Attachments[]?|select(.State == "available")]|length' source: describe-internet-gateways verbose: true -- name: Network ACLs +- name: EC2 Network ACLs query: .NetworkAcls|length source: ec2-describe-network-acls verbose: true -- name: Route tables +- name: EC2 Network Interfaces + query: '.InternetGateways|length' + source: ec2-describe-network-interfaces +- name: EC2 Route tables query: .RouteTables|length source: ec2-describe-route-tables verbose: true -- name: VPC endpoints +- name: EC2 Security Groups + query: .SecurityGroups|length + source: ec2-describe-security-groups + verbose: true +- name: EC2 snapshots + query: .Snapshots|length + source: ec2-describe-snapshots + verbose: true +- name: EC2 Subnets + query: .Subnets|length + source: ec2-describe-subnets +- name: EC2 Volumes + query: .Volumes|length + source: ec2-describe-volumes +- name: EC2 VPCs + query: .Vpcs|length + source: ec2-describe-vpcs +- name: EC2 VPCEndpoints + query: .VPCEndpoints|length + source: ec2-describe-vpc-endpoints + verbose: true +- name: EC2 VPCEndpoints con. query: .VpcEndpointConnections|length source: ec2-describe-vpc-endpoint-connections verbose: true -- name: VPN connections +- name: EC2 VPN connections query: .VpnConnections|length source: ec2-describe-vpn-connections verbose: true -- name: DirectConnects - query: .connections|length - source: directconnect-describe-connections - verbose: true -- name: CloudSearch domains - query: .DomainStatusList|length - source: cloudsearch-describe-domains - verbose: true - name: ECR repositories query: .repositories|length source: ecr-describe-repositories verbose: true -- name: CloudFormation stacks - query: .Stacks|length - source: cloudformation-describe-stacks - verbose: true - name: EFS query: .FileSystems|length source: efs-describe-file-systems verbose: true -- name: Cloudwatch alarms - query: .MetricAlarms|length - source: cloudwatch-describe-alarms - verbose: true -- name: Config rules - query: .ConfigRules|length - source: config-describe-config-rules +- name: ElasticBeanstalks + query: .Applications|length + source: elasticbeanstalk-describe-applications +- name: Elasticache clusters + query: .CacheClusters|length + source: elasticache-describe-cache-clusters +- name: ElasticSearch domains + query: .DomainNames|length + source: es-list-domain-names +- name: ELBs + query: .LoadBalancerDescriptions|length + source: elb-describe-load-balancers +- name: ELBs Policies + query: .PolicyDescriptions|length + source: elb-describe-load-balancer-policies verbose: true +- name: ELBv2s + query: .LoadBalancers|length + source: elbv2-describe-load-balancers - name: Event rules query: .Rules|length source: events-list-rules verbose: true +- name: Firehose streams + query: .DeliveryStreamNames|length + source: firehose-list-delivery-streams +- name: Glacier vaults + query: .VaultList|length + source: glacier-list-vaults +- name: Glue Jobs + query: .Jobs|length + source: glue-get-jobs +- name: Glue Triggers + query: .Triggers|length + source: glue-get-triggers +- name: KMS keys + query: .Keys|length + source: kms-list-keys +- name: Lambda functions + query: .Functions|length + source: lambda-list-functions - name: Log groups query: .logGroups|length source: logs-describe-log-groups verbose: true -- name: ACM Certificates - query: .CertificateSummaryList|length - source: acm-list-certificates - verbose: true -- name: DynamoDB Tables - query: .TableNames|length - source: dynamodb-list-tables +- name: RDS instances + query: .DBInstances|length + source: rds-describe-db-instances +- name: RDS Snapshots + query: .DBSnapshots|length + source: rds-describe-db-snapshots +- name: Redshift clusters + query: .Clusters|length + source: redshift-describe-clusters +- name: Route53 domains + query: .Domains|length + source: route53domains-list-domains verbose: true -- name: ELBs Policies - query: .PolicyDescriptions|length - source: elb-describe-load-balancer-policies +- name: Route53 hosted zones + query: .HostedZones|length + source: route53-list-hosted-zones verbose: true +- name: S3 buckets + query: .Buckets|length + source: s3-list-buckets +- name: Secret + query: .SecretList|length + source: secretsmanager-list-secrets +- name: SNS topics + query: .Topics|length + source: sns-list-topics +- name: SQS queues + query: .QueueUrls|length + source: sqs-list-queues +# - name: Kafka clusters +# query: .ClusterInfoList|length +# source: kafka-list-clusters +# Verbose resources