testing automation; unfinished

lifemapper · Sep 13, 2024 · 7884ff1 · 7884ff1
1 parent 50593d4
commit 7884ff1
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 31 deletions.
diff --git a/_sphinx_config/pages/aws/roles.rst b/_sphinx_config/pages/aws/roles.rst
@@ -6,7 +6,9 @@ Roles, Policies, Trust Relationships
 bison_redshift_s3_role
 ------------------------------
 
-* Trusted entity type: AWS Service
+Attach to BISON namespace (Redshift)
+* Regular role
+* Trust relationships: Service: "redshift.amazonaws.com"
 * for Redshift - Customizable
 
   * TODO: change to Redshift - Scheduler when automated
@@ -16,6 +18,7 @@ bison_redshift_s3_role
   * AmazonRedshiftAllCommandsFullAccess (AWS managed)
   * AmazonRedshiftDataFullAccess (AWS managed)
   * AmazonRedshiftFullAccess (AWS managed)
+  * bison_lambda_log_policy (write CloudWatch logs to bison log groups)
   * bison_s3_policy (read public/GBIF S3 data and read/write bison S3 data)
   * redshift_glue_policy.json (for Redshift interactions)
 
@@ -28,14 +31,12 @@ bison_redshift_s3_role
 bison_redshift_lambda_role
 ------------------------
 
-Attach to BISON lambda functions
+* Service role
+* Trust relationships: Services: ["lambda.amazonaws.com", "redshift.amazonaws.com"]
+* Attached to BISON lambda functions
+* Attach to BISON namespace (Redshift)
 
-  * AmazonRedshiftAllCommandsFullAccess (AWS managed)
-  * AmazonRedshiftDataFullAccess (AWS managed)
-  * AmazonRedshiftFullAccess (AWS managed)
-  * bison_lambda_log_policy (write CloudWatch logs to bison log groups)
-    TODO: add new log group for each lambda function
-  * bison_s3_policy (read public/GBIF S3 data and read/write bison S3 data)
+  * same as bison_redshift_s3_role
 
 .. _bison_ec2_s3_role:
 
@@ -52,3 +53,5 @@ bison_ec2_s3_role
 * Trust relationship:
 
   * ec2_s3_role_trust_policy.json edit trust policy for both ec2 and s3
+
+
diff --git a/aws/events/bison_mount_gbif_lambda.py b/aws/events/bison_mount_gbif_lambda.py
@@ -10,8 +10,7 @@
 region = "us-east-1"
 workgroup = "bison"
 database = "dev"
-dbuser = "IAM:aimee.stewart"
-dbuser = "arn:aws:iam::321942852011:role/service-role/bison_subset_gbif_lambda-role-9i5qvpux"
+iam_role = "arn:aws:iam::321942852011:role/service-role/bison_redshift_lambda_role"
 pub_schema = "public"
 external_schema = "redshift_spectrum"
 timeout = 900
@@ -27,6 +26,14 @@
 mounted_gbif_name = f"{external_schema}.occurrence_{bison_datestr}_parquet"
 subset_bison_name = f"{pub_schema}.bison_{bison_datestr}"
 
+create_schema_stmt = f"""
+    CREATE EXTERNAL SCHEMA IF NOT EXISTS {external_schema}
+        FROM data catalog
+        DATABASE 'dev'
+        IAM_ROLE DEFAULT
+        CREATE external database IF NOT EXISTS;
+"""
+
 mount_stmt = f"""
     CREATE EXTERNAL TABLE {mounted_gbif_name} (
         gbifid	VARCHAR(max),
@@ -131,7 +138,7 @@ def lambda_handler(event, context):
     try:
         submit_result = client_redshift.execute_statement(
             WorkgroupName=workgroup, Database=database, Sql=mount_stmt)
-        print(f"*** Mount command submitted")
+        print(f"*** Command submitted")
 
     except Exception as e:
         raise Exception(e)
@@ -163,21 +170,6 @@ def lambda_handler(event, context):
             print(f"Failed to describe_statement {e}")
             complete = True
 
-    # # # -------------------------------------
-    # # # IFF query, get statement output
-    # try:
-    #     stmt_result = client_redshift.get_statement_result(Id=submit_id)
-    # except Exception as e:
-    #     print(f"*** No get_statement_result {e}")
-    # else:
-    #     print("*** get_statement_result records")
-    #     try:
-    #         records = stmt_result["Records"]
-    #         for rec in records:
-    #             print(f"***     {rec}")
-    #     except Exception as e:
-    #         print(f"Failed to return records ({e})")
-
     return {
         'statusCode': 200,
         'body': json.dumps(f"Lambda result logged")

diff --git a/aws/redshift/queries.sql b/aws/redshift/queries.sql
@@ -19,3 +19,6 @@ ORDER BY database_name, SCHEMA_NAME;
 select current_user;
 SELECT * FROM PG_USER_INFO;
 
+SHOW DATABASES FROM DATA CATALOG IAM_ROLE default ;
+SHOW SCHEMAS FROM DATABASE dev;
+SHOW TABLES FROM SCHEMA dev.public;
diff --git a/aws/redshift/subset_to_bison.sql b/aws/redshift/subset_to_bison.sql
@@ -19,14 +19,21 @@ CREATE EXTERNAL SCHEMA IF NOT EXISTS redshift_spectrum
     FROM data catalog
     DATABASE 'dev'
     -- Same role as namespace
-    IAM_ROLE 'arn:aws:iam::321942852011:role/bison_redshift_s3_role'
+--    IAM_ROLE 'arn:aws:iam::321942852011:role/service-role/bison_redshift_lambda_role'
+    IAM_ROLE default
     CREATE external database if NOT exists;
 
--- If change IAM role, do this:
---GRANT USAGE TO redshift_spectrum to "IAMR:bison_subset_gbif_lambda-role-9i5qvpux";
-GRANT ALL ON ALL TABLES IN SCHEMA redshift_spectrum
-    TO ROLE 'arn:aws:iam::321942852011:role/service-role/bison_subset_gbif_lambda-role-9i5qvpux';
+GRANT ASSUMEROLE
+       ON 'arn:aws:iam::321942852011:role/service-role/bison_redshift_lambda_role'
+       TO ROLE IAMR:bison_redshift_lambda_role
+       FOR ALL;
 
+---- If change IAM role, do this:
+----GRANT USAGE TO redshift_spectrum to "IAMR:bison_subset_gbif_lambda-role-9i5qvpux";
+--GRANT ALL ON ALL TABLES IN SCHEMA redshift_spectrum
+--    TO ROLE 'arn:aws:iam::321942852011:role/service-role/bison_subset_gbif_lambda-role-9i5qvpux';
+--GRANT ALL ON ALL TABLES IN SCHEMA redshift_spectrum
+--    TO ROLE 'arn:aws:iam::321942852011:role/service-role/bison_redshift_lambda_role';
 
 
 -- Mount a table of current GBIF ODR data in S3