generalize summary and matrix operations for multiple dimensions

aws/ec2/ec2_tools.py

@@ -7,7 +7,7 @@
 import json
 import os
 import pandas
-from pprint import pp
+# from pprint import pp
 
 # --------------------------------------------------------------------------------------
 # Constants for GBIF data, local dev machine, EC2, S3
@@ -63,7 +63,6 @@
 }
 
 
-
 # --------------------------------------------------------------------------------------
 # Tools for experimentation
 # --------------------------------------------------------------------------------------
@@ -442,32 +441,31 @@ def get_instance(instance_id):
 # --------------------------------------------------------------------------------------
 # Main
 # --------------------------------------------------------------------------------------
-if __name__ == '__main__':
-
-# # -------  Find or create template -------
-# # Adds the script to the spot template
-# success = create_spot_launch_template(
-#         spot_template_name, security_group_id, script_filename, key_name)
-#
-# # -------  Run instance from template -------
-# # Runs the script on instantiation
-# instance_id = run_instance_spot(instance_basename, spot_template_name)
-#
-# instance = get_instance(instance_id)
-# ip = instance["PublicIpAddress"]
-
-    # Get an existing launch template
-    launch_template_file = "/tmp/bison_launch_template.json"
-    bison_dev_instance_id = "i-0cc438734534bb505"
-    launch_template = get_launch_template_from_instance(bison_dev_instance_id)
-    # Make easier to read
-    json_str = json.dumps(launch_template, indent=4)
-    with open(launch_template_file, "w") as outf:
-        outf.write(json_str)
-
-    # Edit this launch template
-
-    # Upload new launch template
-
-    create_spot_launch_template(
-        spot_template_name, security_group_id, script_filename, key_name)
+# if __name__ == '__main__':
+    # # -------  Find or create template -------
+    # # Adds the script to the spot template
+    # success = create_spot_launch_template(
+    #         spot_template_name, security_group_id, script_filename, key_name)
+    #
+    # # -------  Run instance from template -------
+    # # Runs the script on instantiation
+    # instance_id = run_instance_spot(instance_basename, spot_template_name)
+    #
+    # instance = get_instance(instance_id)
+    # ip = instance["PublicIpAddress"]
+    #
+    # # Get an existing launch template
+    # launch_template_file = "/tmp/bison_launch_template.json"
+    # bison_dev_instance_id = "i-0cc438734534bb505"
+    # launch_template = get_launch_template_from_instance(bison_dev_instance_id)
+    # # Make easier to read
+    # json_str = json.dumps(launch_template, indent=4)
+    # with open(launch_template_file, "w") as outf:
+    #     outf.write(json_str)
+    #
+    # # Edit this launch template
+    #
+    # # Upload new launch template
+    #
+    # create_spot_launch_template(
+    #     spot_template_name, security_group_id, script_filename, key_name)

bison/common/constants.py

@@ -183,7 +183,7 @@ def summaries(cls):
             meta1 = {
                 "code": name1,
                 "fname": f"{name1}{cls.sep}{cls.dt_token}",
-                "aggregate_type": "species_summary",
+                "aggregate_type": f"{SPECIES_DIM['name']}_summary",
                 # Axis 0, matches row (axis 0) in SPECIES_<dimension>_MATRIX
                 "row": SPECIES_DIM["key_fld"],
             }
@@ -244,6 +244,17 @@ def matrices(cls):
             mtxs[name] = meta
         return mtxs
 
+    # ...............................................
+    @classmethod
+    def parse_table_type(cls, table_type):
+        tbl = cls.get_table(table_type)
+        parts = table_type.split("_")
+        if len(parts) == 2:
+            datacontents, datetype = parts
+        else:
+            raise Exception(f"Failed to parse {table_type} into datacontents, datetype.")
+        return datacontents, datetype
+
     # ...............................................
     @classmethod
     def tables(cls, datestr=None):
@@ -291,6 +302,34 @@ def get_table(cls, table_type, datestr=None):
             cpy_table["fname"] = fname_tmpl.replace(cls.dt_token, datestr)
         return cpy_table
 
+    # ...............................................
+    @classmethod
+    def get_tabletype_from_filename_prefix(cls, datacontents, datatype):
+        """Get the table type from the file prefixes.
+
+        Args:
+            datacontents (str): first part of filename indicating data in table.
+            datatype (str): second part of filename indicating form of data in table
+                (records, lists, matrix, etc).
+
+        Returns:
+            table_type (SUMMARY_TABLE_TYPES type): type of table.
+
+        Raises:
+            Exception: on invalid file prefix.
+        """
+        tables = cls.tables()
+        table_type = None
+        for key, meta in tables.items():
+            fname = meta["fname"]
+            contents, dtp, _, _ = cls._parse_filename(fname)
+            if datacontents == contents and datatype == dtp:
+                table_type = key
+        if table_type is None:
+            raise Exception(
+                f"Table with filename prefix {datacontents}_{datatype} does not exist")
+        return table_type
+
     # ...............................................
     @classmethod
     def get_filename(cls, table_type, datestr):