From 6db95cea1a1cc34f5a11a9083ba7e5932d0daac0 Mon Sep 17 00:00:00 2001 From: Aimee Stewart Date: Wed, 25 Sep 2024 17:15:47 -0500 Subject: [PATCH] work on ec2/docker/lambda for initiating step 1 --- .env.conf | 2 +- Dockerfile | 46 +- _sphinx_config/pages/workflow.rst | 22 + aws/ec2/ec2_tools.py | 473 ++++++++++++++++++ aws/ec2/userdata_annotate_riis.sh | 18 + aws/events/bison_s0_resolve_riis_lambda.py | 135 +++++ .../bison_invoke_lambda_policy.json | 2 +- aws/permissions/bison_lambda_log_policy.json | 2 +- bison/tools/__init__.py | 2 +- bison/tools/annotate_riis.py | 11 +- docker-compose.development.yml | 14 +- docker-compose.yml | 12 +- .../src}/tools/calculate_pam_stats.py | 0 .../src}/tools/create_heatmap.py | 0 .../src}/tools/encode_points_to_pam.py | 0 .../src}/tools/write_geo_matrices.py | 0 16 files changed, 691 insertions(+), 48 deletions(-) create mode 100644 aws/ec2/ec2_tools.py create mode 100644 aws/ec2/userdata_annotate_riis.sh create mode 100644 aws/events/bison_s0_resolve_riis_lambda.py rename {bison => obsolete/src}/tools/calculate_pam_stats.py (100%) rename {bison => obsolete/src}/tools/create_heatmap.py (100%) rename {bison => obsolete/src}/tools/encode_points_to_pam.py (100%) rename {bison => obsolete/src}/tools/write_geo_matrices.py (100%) diff --git a/.env.conf b/.env.conf index 9b6c2246..e49b5476 100644 --- a/.env.conf +++ b/.env.conf @@ -3,4 +3,4 @@ WORKING_DIRECTORY=/scratch-path AWS_DATA_DIRECTORY=/volumes/aws_data FQDN=bison.localhost -PYTHONPATH=/home/bison/flask_app +PYTHONPATH=/home/bison/bison:/home/bison/flask_app diff --git a/Dockerfile b/Dockerfile index ee4bb9e3..8456aa34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,26 +61,26 @@ COPY --chown=bison:bison ./flask_app ./flask_app #ENV FLASK_ENV=production CMD venv/bin/python -m gunicorn -w 4 --bind 0.0.0.0:5000 ${FLASK_APP} -# ........................................................ -# Frontend base image (for development) -FROM node:16.10.0-buster as base-front-end - -LABEL maintainer="Specify Collections Consortium " - -USER node -WORKDIR /home/node - -COPY --chown=node:node bison/frontend/js_src/package*.json ./ -RUN npm install - -RUN mkdir dist \ - && chown node:node dist - -COPY --chown=node:node bison/frontend/js_src . - - -# ........................................................ -# Frontend image (for production) from base-front-end -FROM base-front-end as front-end - -RUN npm run build +## ........................................................ +## Frontend base image (for development) +#FROM node:16.10.0-buster as base-front-end +# +#LABEL maintainer="Specify Collections Consortium " +# +#USER node +#WORKDIR /home/node +# +#COPY --chown=node:node bison/frontend/js_src/package*.json ./ +#RUN npm install +# +#RUN mkdir dist \ +# && chown node:node dist +# +#COPY --chown=node:node bison/frontend/js_src . +# +# +## ........................................................ +## Frontend image (for production) from base-front-end +#FROM base-front-end as front-end +# +#RUN npm run build diff --git a/_sphinx_config/pages/workflow.rst b/_sphinx_config/pages/workflow.rst index 5382cb2a..3f944763 100644 --- a/_sphinx_config/pages/workflow.rst +++ b/_sphinx_config/pages/workflow.rst @@ -73,6 +73,28 @@ will be created in the same directory, and have the basename appended with This ensures that the RIIS data and the current GBIF data that have the same date appended and taxonomic resolutions are current. +Resolve RIIS data +======================= + +Add GBIF accepted taxa to RIIS data records of RIIS introduced/invasive status for +species by region (AK, HI, L48). This provides a field to match on, as all GBIF records +include the accepted taxon. + +This step is initiated by an EventBridge Schedule, and: + +* Creates and launches an EC2 instance +* The "userdata" in the EC2 instance runs a script which: + + * pulls a Docker image containing BISON code from the Specify Docker Hub + * starts the Docker instance + * initiates the "annotate_riis" command. + * when the annotation is complete, it uploads the annotated RIIS file to S3 + +* TODO: The update of the annotated RIIS file on S3 is an event which triggers: + + * stop the EC2 instance. + * initiate the next step. + Load ancillary data =================== diff --git a/aws/ec2/ec2_tools.py b/aws/ec2/ec2_tools.py new file mode 100644 index 00000000..66ba0436 --- /dev/null +++ b/aws/ec2/ec2_tools.py @@ -0,0 +1,473 @@ +"""Test workflow.""" +import base64 +import boto3 +import csv +import datetime +import io +import json +import os +import pandas +from pprint import pp + +# -------------------------------------------------------------------------------------- +# Constants for GBIF data, local dev machine, EC2, S3 +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------------------- +# On EC2: Create a trimmed dataframe from CSV and save to S3 in parquet format +# -------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------- +key_name = "aimee-aws-key" +aws_zone = "us-east-1" +# Allows KU Dyche hall +security_group_id = "sg-0b379fdb3e37389d1" + +# Local machine data +local_ec2_key = f"/home/astewart/.ssh/{key_name}.pem" +local_path = "/mnt/sata8/tank/testgbif/occ_2023_06_23" +test_basename = "occurrence_100k" +csv_fname = f"{test_basename}.csv" + +# DWCA data for download from GBIF +gbif_basename = "0042658-230530130749713" + +# S3 +dev_bucket = "specify-network-dev" +bucket_path = "gbif_test/gbif_dwc_extract" +# s3_csv_path = f"{bucket_path}/{test_basename}.csv" +# s3_parquet_path = f"{bucket_path}/{test_basename}.parquet" + +# EC2 persistent +ec2_user = "ubuntu" +ec2_dev_ip = "54.156.84.82" + +# EC2 Spot Instance +bison_ec2_template_name = "bison_launch_template" +instance_basename = "bison_dev" + +script_filename = "scripts/user_data_for_ec2spot.sh" + +target_capactity_spec = { + "TotalTargetCapacity": 1, + "SpotTargetCapacity": 1, + "DefaultTargetCapacityType": "spot" +} +spot_opts = { + "AllocationStrategy": "price-capacity-optimized", + "InstanceInterruptionBehavior": "terminate" +} +launch_template_config = { + "LaunchTemplateSpecification": { + "LaunchTemplateName": bison_ec2_template_name, + "Version": "1"} +} + + + +# -------------------------------------------------------------------------------------- +# Tools for experimentation +# -------------------------------------------------------------------------------------- +# ---------------------------------------------------- +def get_launch_template_from_instance(instance_id): + """Get or create an EC2 launch template from an EC2 instance identifier. + + Args: + instance_id: identifier for an EC2 instance to use as a template. + + Returns: + launch_template_data: metadata to be used as an EC2 launch template. + """ + ec2_client = boto3.client("ec2") + launch_template_data = ec2_client.get_launch_template_data(InstanceId=instance_id) + return launch_template_data + + +# ---------------------------------------------------- +def delete_instance(instance_id): + """Delete an EC2 instance. + + Args: + instance_id: identifier for an EC2 instance to delete. + + Returns: + response: response from the server. + """ + ec2_client = boto3.client("ec2") + response = ec2_client.delete_instance(InstanceId=instance_id) + return response + + +# ---------------------------------------------------- +def create_dataframe_from_gbifcsv_s3_bucket(bucket, csv_path): + """Get or create an EC2 launch template from an EC2 instance identifier. + + Args: + bucket: name for an S3 bucket. + csv_path: bucket path, including object name, of CSV data of interest. + + Returns: + df: pandas dataframe containing the tabular CSV data. + """ + # Read CSV file from S3 into a pandas DataFrame + s3_client = boto3.client("s3") + s3_obj = s3_client.get_object(Bucket=bucket, Key=csv_path) + df = pandas.read_csv( + s3_obj["Body"], delimiter="\t", encoding="utf-8", low_memory=False, + quoting=csv.QUOTE_NONE) + return df + + +# -------------------------------------------------------------------------------------- +# On local machine: +def find_instances(key_name, launch_template_name): + """Describe all instances with given key_name and/or launch_template_id. + + Args: + key_name: optional key_name assigned to an instance. + launch_template_name: name assigned to the template which created the instance + + Returns: + instances: list of metadata for instances + """ + ec2_client = boto3.client("ec2") + filters = [] + if launch_template_name is not None: + filters.append({"Name": "tag:TemplateName", "Values": [launch_template_name]}) + if key_name is not None: + filters.append({"Name": "key-name", "Values": [key_name]}) + response = ec2_client.describe_instances( + Filters=filters, + DryRun=False, + MaxResults=123, + # NextToken="string" + ) + instances = [] + try: + ress = response["Reservations"] + except Exception: + pass + else: + for res in ress: + _print_inst_info(res) + instances.extend(res["Instances"]) + return instances + + +# -------------------------------------------------------------------------------------- +# Hidden helper functions +# -------------------------------------------------------------------------------------- +# ---------------------------------------------------- +def _create_token(type): + token = f"{type}_{datetime.datetime.now().timestamp()}" + return token + + +# ---------------------------------------------------- +def _get_user_data(script_filename): + try: + with open(script_filename, "r") as infile: + script_text = infile.read() + except Exception: + return None + else: + text_bytes = script_text.encode("ascii") + text_base64_bytes = base64.b64encode(text_bytes) + base64_script_text = text_base64_bytes.decode("ascii") + return base64_script_text + + +# ---------------------------------------------------- +def _print_inst_info(reservation): + resid = reservation["ReservationId"] + inst = reservation["Instances"][0] + print(f"ReservationId: {resid}") + name = temp_id = None + try: + tags = inst["Tags"] + except Exception: + pass + else: + for t in tags: + if t["Key"] == "Name": + name = t["Value"] + if t["Key"] == "aws:ec2launchtemplate:id": + temp_id = t["Value"] + ip = inst["PublicIpAddress"] + state = inst["State"]["Name"] + print(f"Instance name: {name}, template: {temp_id}, IP: {ip}, state: {state}") + + +# ---------------------------------------------------- +def _define_spot_launch_template_data( + spot_template_name, security_group_id, script_filename, key_name): + user_data_64 = _get_user_data(script_filename) + launch_template_data = { + "EbsOptimized": True, + "IamInstanceProfile": {"Name": "AmazonEMR-InstanceProfile-20230404T163626"}, + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "Encrypted": False, + "DeleteOnTermination": True, + "VolumeSize": 50, "VolumeType": "gp2"} + } + ], + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": True, + "DeleteOnTermination": True, + "Description": "", + "DeviceIndex": 0, + "Groups": [security_group_id], + "InterfaceType": "interface", + "Ipv6Addresses": [], + } + ], + "ImageId": "ami-0a0c8eebcdd6dcbd0", + "InstanceType": "t4g.medium", + "KeyName": key_name, + "Monitoring": {"Enabled": False}, + "Placement": { + "AvailabilityZone": "us-east-1c", "GroupName": "", "Tenancy": "default" + }, + "DisableApiTermination": False, + "InstanceInitiatedShutdownBehavior": "terminate", + "UserData": user_data_64, + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [{"Key": "TemplateName", "Value": spot_template_name}] + } + ], + "InstanceMarketOptions": { + "MarketType": "spot", + "SpotOptions": { + "MaxPrice": "0.033600", + "SpotInstanceType": "one-time", + "InstanceInterruptionBehavior": "terminate" + } + }, + "CreditSpecification": {"CpuCredits": "unlimited"}, + "CpuOptions": {"CoreCount": 2, "ThreadsPerCore": 1}, + "CapacityReservationSpecification": {"CapacityReservationPreference": "open"}, + "HibernationOptions": {"Configured": False}, + "MetadataOptions": { + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": {"Enabled": False}, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": True, + "EnableResourceNameDnsAAAARecord": False}, + "MaintenanceOptions": {"AutoRecovery": "default"}, + "DisableApiStop": False + } + return launch_template_data + + +# -------------------------------------------------------------------------------------- +# On local machine: Describe the launch_template with the template_name +def _get_launch_template(template_name): + ec2_client = boto3.client("ec2") + lnch_temp = None + try: + response = ec2_client.describe_launch_templates( + LaunchTemplateNames=[template_name]) + except Exception: + pass + else: + # LaunchTemplateName is unique + try: + lnch_temp = response["LaunchTemplates"][0] + except Exception: + pass + return lnch_temp + + +# ---------------------------------------------------- +def create_spot_launch_template( + spot_template_name, security_group_id, script_filename, key_name): + """Create a launch template for an EC2 Spot instance. + + Args: + spot_template_name: Name to assign to this template. + security_group_id: Identifier for security group to assign to the + EC2 instance created from this template. + script_filename: filename for a script to include as userdata on an + EC2 instance created from this template. + key_name: key name assigned to a launch template. + + Returns: + success: boolean flag indicating success of the operation. + """ + template = _get_launch_template(spot_template_name) + if template is not None: + success = True + else: + spot_template_data = _define_spot_launch_template_data( + spot_template_name, security_group_id, script_filename, key_name) + template_token = _create_token("template") + ec2_client = boto3.client("ec2") + response = ec2_client.create_launch_template( + DryRun=False, + ClientToken=template_token, + LaunchTemplateName=spot_template_name, + VersionDescription="Spot for GBIF process", + LaunchTemplateData=spot_template_data + ) + success = (response["ResponseMetadata"]["HTTPStatusCode"] == 200) + return success + + +# ---------------------------------------------------- +def run_instance_spot(instance_basename, spot_template_name): + """Create a launch template for an EC2 Spot instance. + + Args: + instance_basename: Basename to assign to this EC2 Spot instance. + spot_template_name: Name assigned to a spot launch template to be used + creating this instance. + + Returns: + string: instance id of the successfully launched EC2 Spot instance. + """ + ec2_client = boto3.client("ec2") + spot_token = _create_token("spot") + instance_name = _create_token(instance_basename) + response = ec2_client.run_instances( + # KeyName=key_name, + ClientToken=spot_token, + MinCount=1, MaxCount=1, + LaunchTemplate={"LaunchTemplateName": spot_template_name, "Version": "1"}, + TagSpecifications=[{ + "ResourceType": "instance", + "Tags": [ + {"Key": "Name", "Value": instance_name}, + {"Key": "TemplateName", "Value": spot_template_name} + ] + }] + ) + try: + instance = response["Instances"][0] + except KeyError: + instance = None + print("No instance created") + return instance["InstanceId"] + + +# ---------------------------------------------------- +def write_dataframe_to_s3_parquet(df, bucket, parquet_path): + """Write DataFrame to Parquet format and upload to S3. + + Args: + df (pandas.DataFrame): tabular data to write on S3. + bucket (str): Bucket identifier on S3. + parquet_path (str): Destination path to the S3 parquet data. + """ + s3_client = boto3.client("s3") + parquet_buffer = io.BytesIO() + df.to_parquet(parquet_buffer, engine="pyarrow") + parquet_buffer.seek(0) + s3_client.upload_fileobj(parquet_buffer, bucket, parquet_path) + + +# -------------------------------------------------------------------------------------- +# Run on EC2 or local: Upload file to S3 +# -------------------------------------------------------------------------------------- +def upload_to_s3(local_path, filename, bucket, s3_path): + """Upload a local file to S3. + + Args: + local_path: path to data to upload. + filename: filename of the data to upload. + bucket (str): Bucket identifier on S3. + s3_path (str): Destination path to the S3 data. + + TODO: Use aws_tools method here + """ + s3_client = boto3.client("s3") + local_filename = os.path.join(local_path, filename) + s3_client.upload_file(local_filename, bucket, s3_path) + print(f"Successfully uploaded {filename} to s3://{bucket}/{s3_path}") + + +# # -------------------------------------------------------------------------------------- +# # On local machine: Trim CSV file to named columns and save in parquet format +# # -------------------------------------------------------------------------------------- +# def trim_gbifcsv_to_parquet(local_filename, parquet_filename): +# """Trim a GBIF CSV file to a subset of fields, then write to S3 in parquet format. +# +# Args: +# local_filename: path and filename of data to subset and upload. +# parquet_filename: filename for the destination parquet file on S3. +# """ +# gbif_dataframe = pandas.read_csv( +# local_filename, delimiter="\t", encoding="utf-8", low_memory=False, +# quoting=csv.QUOTE_NONE) +# # Trim the DataFrame to the subset of fieldnames +# trimmed_gbif_dataframe = gbif_dataframe[FIELD_SUBSET] +# # Write the trimmed DataFrame to Parquet file format +# trimmed_gbif_dataframe.to_parquet(parquet_filename) + + +# -------------------------------------------------------------------------------------- +# On local machine: Describe the instance with the instance_id +# -------------------------------------------------------------------------------------- +def get_instance(instance_id): + """Get an EC2 instance. + + Args: + instance_id: identifier for an EC2 instance to retrieve. + + Returns: + instance: metadata response from the server. + """ + ec2_client = boto3.client("ec2") + response = ec2_client.describe_instances( + InstanceIds=[instance_id], + DryRun=False, + ) + try: + instance = response["Reservations"][0]["Instances"][0] + except Exception: + instance = None + return instance + + +# -------------------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------------------- +if __name__ == '__main__': + +# # ------- Find or create template ------- +# # Adds the script to the spot template +# success = create_spot_launch_template( +# spot_template_name, security_group_id, script_filename, key_name) +# +# # ------- Run instance from template ------- +# # Runs the script on instantiation +# instance_id = run_instance_spot(instance_basename, spot_template_name) +# +# instance = get_instance(instance_id) +# ip = instance["PublicIpAddress"] + + # Get an existing launch template + launch_template_file = "/tmp/bison_launch_template.json" + bison_dev_instance_id = "i-0cc438734534bb505" + launch_template = get_launch_template_from_instance(bison_dev_instance_id) + # Make easier to read + json_str = json.dumps(launch_template, indent=4) + with open(launch_template_file, "w") as outf: + outf.write(json_str) + + # Edit this launch template + + # Upload new launch template + + create_spot_launch_template( + spot_template_name, security_group_id, script_filename, key_name) \ No newline at end of file diff --git a/aws/ec2/userdata_annotate_riis.sh b/aws/ec2/userdata_annotate_riis.sh new file mode 100644 index 00000000..e5306ce8 --- /dev/null +++ b/aws/ec2/userdata_annotate_riis.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# This is the user data script to be executed on an EC2 instance. + +aws configure set default.region us-east-1 && \ +aws configure set default.output json + +sudo apt update +sudo apt install apache2 certbot plocate unzip +sudo apt install docker.io +sudo apt install docker-compose-v2 + +# TODO: change this to pull a Docker image +git clone https://github.com/lifemapper/bison.git + +sudo docker compose up -d +#sudo docker compose -f docker-compose.development.yml -f docker-compose.yml up +# Executes from /home/bison directory, which contains bison code +sudo docker exec bison-bison-1 venv/bin/python bison/tools/annotate_riis.py \ No newline at end of file diff --git a/aws/events/bison_s0_resolve_riis_lambda.py b/aws/events/bison_s0_resolve_riis_lambda.py new file mode 100644 index 00000000..08c460f6 --- /dev/null +++ b/aws/events/bison_s0_resolve_riis_lambda.py @@ -0,0 +1,135 @@ +"""Lambda function to aggregate counts and lists by region.""" +# Set lambda timeout to 3 minutes. +import json +import boto3 +import botocore.session as bc +from botocore.client import Config +from datetime import datetime +import time + +print("*** Loading lambda function") + +region = "us-east-1" +workgroup = "bison" +database = "dev" +iam_role = "arn:aws:iam::321942852011:role/service-role/bison_redshift_lambda_role" +db_user = "IAMR:bison_redshift_lambda_role" +pub_schema = "public" +external_schema = "redshift_spectrum" +timeout = 900 +waittime = 1 + +# AMI = "ami-096ea6a12ea24a797" +# INSTANCE_TYPE = "t4g.micro" +# KEY_NAME = "aimee-aws-key" + +# Get previous data date +yr = datetime.now().year +mo = datetime.now().month +prev_yr = yr +prev_mo = mo - 1 +if mo == 1: + prev_mo = 12 + prev_yr = yr - 1 +bison_datestr = f"{yr}_{mo:02d}_01" +bison_tbl = f"bison_{bison_datestr}" + +# Define the bison bucket and table to create +bison_bucket = "bison-321942852011-us-east-1" +input_folder = "input" + +# List of tuples, (command for logfile, corresponding AWS command) +COMMANDS = [ + # Userdata script will pull + # bison repo (or bison docker image) + # run docker compose -f docker-compose.development.yml -f docker-compose.yml up + ("launch_ec2_annotate_riis"), + (""), + (), + ()] + +# Initialize Botocore session +session = boto3.session.Session() +bc_session = bc.get_session() +session = boto3.Session(botocore_session=bc_session, region_name=region) +# Initialize Redshift client +config = Config(connect_timeout=timeout, read_timeout=timeout) +client_ec2 = session.client("ec2", config=config) + + +# -------------------------------------------------------------------------------------- +def lambda_handler(event, context): + """Annotate BISON records with RIIS regions and status of species in record region. + + Args: + event: AWS event triggering this function. + context: AWS context of the event. + + Returns: + JSON object + + Raises: + Exception: on failure to execute Redshift command. + """ + # Execute the commmands in order + for (cmd, stmt) in COMMANDS: + # ------------------------------------- + try: + submit_result = client_ec2 + except Exception: + raise + + print("*** ---------------------------------------") + print(f"*** {cmd.upper()} command submitted") + print(f"*** {stmt}") + submit_id = submit_result['Id'] + + # ------------------------------------- + # Loop til complete, then describe result + elapsed_time = 0 + complete = False + while not complete: + try: + describe_result = client_ec2.describe_statement(Id=submit_id) + except Exception as e: + print(f"*** Failed to describe_statement in {elapsed_time} secs: {e}") + else: + status = describe_result["Status"] + if status in ("ABORTED", "FAILED", "FINISHED"): + print(f"*** Status - {status} after {elapsed_time} seconds") + complete = True + if status == "FAILED": + try: + err = describe_result["Error"] + except Exception: + err = "Unknown Error" + print(f"*** FAILED: {err}") + else: + time.sleep(waittime) + elapsed_time += waittime + + return { + 'statusCode': 200, + 'body': json.dumps("Lambda result logged") + } + + + +ec2 = boto3.client('ec2', region_name=REGION) + + +def lambda_handler(event, context): + + instance = ec2.run_instances( + ImageId=AMI, + InstanceType=INSTANCE_TYPE, + KeyName=KEY_NAME, + MaxCount=1, + MinCount=1 + ) + + print ("New instance created:") + instance_id = instance['Instances'][0]['InstanceId'] + print (instance_id) + + return instance_id \ No newline at end of file diff --git a/aws/permissions/bison_invoke_lambda_policy.json b/aws/permissions/bison_invoke_lambda_policy.json index 08fcae07..31b7cea5 100644 --- a/aws/permissions/bison_invoke_lambda_policy.json +++ b/aws/permissions/bison_invoke_lambda_policy.json @@ -12,4 +12,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/aws/permissions/bison_lambda_log_policy.json b/aws/permissions/bison_lambda_log_policy.json index 024420e4..3769a7f2 100644 --- a/aws/permissions/bison_lambda_log_policy.json +++ b/aws/permissions/bison_lambda_log_policy.json @@ -17,4 +17,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/bison/tools/__init__.py b/bison/tools/__init__.py index 23449daa..987aa7d3 100644 --- a/bison/tools/__init__.py +++ b/bison/tools/__init__.py @@ -1,4 +1,4 @@ -"""Data preparation and analyses tools.""" +"""Tools to be called from a Docker container in an EC2 instance.""" from . import annotate_riis __all__ = [ diff --git a/bison/tools/annotate_riis.py b/bison/tools/annotate_riis.py index d4c875f0..03cee662 100644 --- a/bison/tools/annotate_riis.py +++ b/bison/tools/annotate_riis.py @@ -3,14 +3,9 @@ from logging import INFO, ERROR import os -from bison.common.constants import ( - # PROJ_BUCKET, PROJ_INPUT_PATH, REGION, - REPORT) +from bison.common.constants import (PROJ_BUCKET, PROJ_INPUT_PATH, REGION, REPORT) from bison.common.log import Logger -from bison.common.util import ( - BisonNameOp, get_today_str, - # upload_to_s3 -) +from bison.common.util import (BisonNameOp, get_today_str, upload_to_s3) from bison.provider.constants import RIIS_FILENAME from bison.provider.riis_data import RIIS @@ -47,4 +42,4 @@ f"of total {report[REPORT.RIIS_IDENTIFIER]} from {RIIS_FILENAME} " f"to {report[REPORT.OUTFILE]}.", refname=script_name) - # upload_to_s3(annotated_filename, PROJ_BUCKET, PROJ_INPUT_PATH, region=REGION) + upload_to_s3(annotated_filename, PROJ_BUCKET, PROJ_INPUT_PATH, region=REGION) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index d76575af..7629322f 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -14,10 +14,10 @@ services: volumes: - "./flask_app/:/home/bison/flask_app/" - front-end: - build: - context: ./. - target: base-front-end - command: ["npm", "run", "watch"] - volumes: - - "./bison/frontend/js_src/lib/:/home/node/lib/" +# front-end: +# build: +# context: ./. +# target: base-front-end +# command: ["npm", "run", "watch"] +# volumes: +# - "./bison/frontend/js_src/lib/:/home/node/lib/" diff --git a/docker-compose.yml b/docker-compose.yml index ad110fa1..cf999094 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,12 +40,12 @@ services: - "./nginx.conf:/etc/nginx/conf.d/default.conf:ro" - "./bison/frontend/static/:/var/www/:ro" - front-end: - build: - context: ./. - target: front-end - volumes: - - "webpack-output:/home/node/dist" +# front-end: +# build: +# context: ./. +# target: front-end +# volumes: +# - "webpack-output:/home/node/dist" volumes: scratch-path: diff --git a/bison/tools/calculate_pam_stats.py b/obsolete/src/tools/calculate_pam_stats.py similarity index 100% rename from bison/tools/calculate_pam_stats.py rename to obsolete/src/tools/calculate_pam_stats.py diff --git a/bison/tools/create_heatmap.py b/obsolete/src/tools/create_heatmap.py similarity index 100% rename from bison/tools/create_heatmap.py rename to obsolete/src/tools/create_heatmap.py diff --git a/bison/tools/encode_points_to_pam.py b/obsolete/src/tools/encode_points_to_pam.py similarity index 100% rename from bison/tools/encode_points_to_pam.py rename to obsolete/src/tools/encode_points_to_pam.py diff --git a/bison/tools/write_geo_matrices.py b/obsolete/src/tools/write_geo_matrices.py similarity index 100% rename from bison/tools/write_geo_matrices.py rename to obsolete/src/tools/write_geo_matrices.py