diff --git a/api/archive_db.py b/api/archive_db.py index 7a6a15ae..11111c50 100644 --- a/api/archive_db.py +++ b/api/archive_db.py @@ -53,9 +53,9 @@ def delete_breeder_table(table_name=None): return query @staticmethod - def fetch_tables(breeder_name=None): + def fetch_tables(uuid=None): query = f""" - SELECT tablename FROM pg_tables WHERE tablename ~ '^{breeder_name}.*'; + SELECT tablename FROM pg_tables WHERE tablename ~ '^{uuid}.*'; """ return query @@ -82,9 +82,9 @@ def delete_trigger(trigger_name=None, table_name=None): return query @staticmethod - def fetch_triggers(breeder_name=None): + def fetch_triggers(breeder_id=None): query = f""" - SELECT tgname FROM pg_trigger WHERE tgname ~ '^{breeder_name}.*'; + SELECT tgname FROM pg_trigger WHERE tgname ~ '^{breeder_id}.*'; """ return query @@ -124,9 +124,9 @@ def delete_procedure(procedure_name=None): return query @staticmethod - def fetch_procedures(breeder_name=None): + def fetch_procedures(breeder_id=None): query = f""" - SELECT proname FROM pg_proc WHERE proname ~ '^{breeder_name}.*'; + SELECT proname FROM pg_proc WHERE proname ~ '^{breeder_id}.*'; """ return query diff --git a/api/controller.py b/api/controller.py index 3912595b..59a33d06 100644 --- a/api/controller.py +++ b/api/controller.py @@ -47,6 +47,8 @@ import logging import json +import uuid + AIRFLOW_API_BASE_URL = os.environ.get('AIRFLOW__URL') AIRFLOW_API_VERSION = "v1" @@ -75,7 +77,7 @@ ) -def breeders_name_delete(breeder_name): # noqa: E501 +def breeders_id_delete(breeder_id): # noqa: E501 """breeders_delete Purge a breeder # noqa: E501 @@ -96,10 +98,10 @@ def breeders_name_delete(breeder_name): # noqa: E501 db_config = ARCHIVE_DB_CONFIG.copy() db_config.update(dict(dbname="archive_db")) - __query = archive.queries.delete_breeder_table(table_name=breeder_name) + __query = archive.queries.delete_breeder_table(table_name=breeder_id) archive.archive_db.execute(db_info=db_config, query=__query) - __query = archive.queries.fetch_procedures(breeder_name=breeder_name) + __query = archive.queries.fetch_procedures(breeder_id=breeder_id) procedures = archive.archive_db.execute(db_info=db_config, query=__query, with_result=True) for procedure_name in procedures: @@ -108,7 +110,7 @@ def breeders_name_delete(breeder_name): # noqa: E501 __query = archive.queries.delete_procedure(procedure_name=procedure_name[0]) archive.archive_db.execute(db_info=db_config, query=__query) - __query = archive.queries.fetch_tables(breeder_name=breeder_name) + __query = archive.queries.fetch_tables(breeder_id=breeder_id) archive_tables = archive.archive_db.execute(db_info=db_config, query=__query, with_result=True) for table_name in archive_tables: @@ -123,10 +125,10 @@ def breeders_name_delete(breeder_name): # noqa: E501 db_table_name = 'breeder_meta_data' __query = meta_data.queries.remove_breeder_meta(table_name=db_table_name, - breeder_name=breeder_name) + breeder_id=breeder_id) archive.archive_db.execute(db_info=db_config, query=__query) - return Response(json.dumps(dict(message=f"Purged Breeder named {breeder_name}")), + return Response(json.dumps(dict(message=f"Purged Breeder named {breeder_id}")), status=200, mimetype='application/json') @@ -157,7 +159,7 @@ def breeders_get(): # noqa: E501 mimetype='application/json') -def breeders_name_get(breeder_name): # noqa: E501 +def breeders_id_get(breeder_uuid): # noqa: E501 """breeders_name_get Obtain information about breeder from its name # noqa: E501 @@ -169,7 +171,7 @@ def breeders_name_get(breeder_name): # noqa: E501 db_config.update(dict(dbname='meta_data')) db_table_name = 'breeder_meta_data' - __query = meta_data.queries.fetch_meta_data(table_name=db_table_name, breeder_name=breeder_name) + __query = meta_data.queries.fetch_meta_data(table_name=db_table_name, breeder_id=breeder_uuid) breeder_meta_data = archive.archive_db.execute(db_info=db_config, query=__query, with_result=True) breeder_meta_data_row = breeder_meta_data[0] @@ -190,7 +192,9 @@ def breeders_post(content): # noqa: E501 breeder_config_full = content breeder_config = dict(content.get('breeder')) - breeder_id = breeder_config.get('name') + breeder_name = breeder_config.get('name') + uuid = uuid.uuid4() + config.update(dict(uuid=uuid)) def create_breeder(api_client, content): @@ -218,13 +222,13 @@ def create_breeder(api_client, content): db_config = ARCHIVE_DB_CONFIG.copy() db_config.update(dict(dbname="archive_db")) - __query = archive.queries.create_breeder_table(table_name=dag_name) + __query = archive.queries.create_breeder_table(table_name=uuid) archive.archive_db.execute(db_info=db_config, query=__query) for target in targets: - identifier = hashlib.sha256(str.encode(target.get('address'))).hexdigest()[0:6] + hash_suffix = hashlib.sha256(str.encode(target.get('address'))).hexdigest()[0:6] for run_id in range(0, parallel_runs): - dag_id = f'{dag_name}_{run_id}_{identifier}' + dag_id = f'{uuid}_{run_id}_{hash_suffix}' __query = archive.queries.create_breeder_table(table_name=dag_id) archive.archive_db.execute(db_info=db_config, query=__query) @@ -249,6 +253,7 @@ def create_breeder(api_client, content): archive.archive_db.execute(db_info=db_config, query=__query) __query = meta_data.queries.insert_breeder_meta(table_name=db_table_name, + breeder_id=uuid, creation_ts=datetime.datetime.now(), meta_state=breeder_config) archive.archive_db.execute(db_info=db_config, query=__query) diff --git a/api/meta_data_db.py b/api/meta_data_db.py index b355d53b..49f3c07a 100644 --- a/api/meta_data_db.py +++ b/api/meta_data_db.py @@ -8,6 +8,7 @@ def create_meta_breeder_table(table_name=None): query = f""" CREATE TABLE IF NOT EXISTS {table_name} ( + id uuid PRIMARY KEY, creation_tsz TIMESTAMPTZ, definition jsonb NOT NULL ); @@ -16,34 +17,34 @@ def create_meta_breeder_table(table_name=None): return query @staticmethod - def insert_breeder_meta(table_name=None, creation_ts=None, meta_state=None): + def insert_breeder_meta(table_name=None, breeder_id=None, creation_ts=None, meta_state=None): json_string = json.dumps(meta_state) query = f""" INSERT INTO {table_name} (creation_tsz, definition) - VALUES('{creation_ts}', '{json_string}'); + VALUES('{breeder_id}', '{creation_ts}', '{json_string}'); """ return query - def remove_breeder_meta(table_name=None, breeder_name=None): + def remove_breeder_meta(table_name=None, breeder_id=None): query = f""" - DELETE FROM {table_name} WHERE definition->>'name' = '{breeder_name}'; + DELETE FROM {table_name} WHERE id = '{breeder_id}'; """ return query - def fetch_meta_data(table_name=None, breeder_name=None): + def fetch_meta_data(table_name=None, breeder_id=None): query = f""" - SELECT creation_tsz, definition FROM {table_name} WHERE definition->>'name' = '{breeder_name}'; + SELECT id, creation_tsz, definition FROM {table_name} WHERE id = '{breeder_id}'; """ return query def fetch_breeders_list(table_name=None): query = f""" - SELECT definition->>'name',creation_tsz FROM {table_name}; + SELECT id, definition->>'name', creation_tsz FROM {table_name}; """ return query diff --git a/api/openapi.yml b/api/openapi.yml index 7423e03f..50e01b04 100644 --- a/api/openapi.yml +++ b/api/openapi.yml @@ -39,9 +39,9 @@ paths: items: type: object required: - - name + - id properties: - name: + id: type: string '400': description: Invalid request @@ -61,7 +61,7 @@ paths: schema: type: object responses: - '204': + '200': description: Successfully created the breeder '400': description: Invalid request @@ -98,11 +98,11 @@ paths: message: type: string - /breeders/{name}: + /breeders/{id}: delete: description: Purge a breeder parameters: - - name: name + - name: id in: path required: true schema: @@ -127,9 +127,9 @@ paths: message: type: string get: - description: Obtain information about breeder from its name + description: Obtain information about breeder from its id(uuid) parameters: - - name: name + - name: id in: path required: true schema: @@ -144,9 +144,9 @@ paths: items: type: object required: - - name + - id properties: - name: + id: type: string '400': description: Invalid request diff --git a/breeder/linux_network_stack/objective.py b/breeder/linux_network_stack/objective.py index 37c6625a..47f1bb62 100644 --- a/breeder/linux_network_stack/objective.py +++ b/breeder/linux_network_stack/objective.py @@ -5,7 +5,7 @@ def objective(trial, identifier=None, archive_db_url=None, locking_db_url=None, - breeder_name=None): + breeder_id=None): ###--- definition coroutines ---### ### We have to keep to coroutines in the objective function, @@ -44,7 +44,7 @@ def objective(trial, logger.debug('fetching setting data') - breeder_table_name = f"{breeder_name}_{run}_{identifier}" + breeder_table_name = f"{breeder_id}_{run}_{identifier}" query = f"SELECT * FROM {breeder_table_name} WHERE {breeder_table_name}.setting_id = '{setting_id}';" archive_db_data = archive_db_engine.execute(query).fetchall() @@ -64,10 +64,10 @@ def objective(trial, # get lock to gate other objective runs locker = pals.Locker('network_breeder_effectuation', locking_db_url) - dlm_lock = locker.lock(f'{breeder_name}_{identifier}') + dlm_lock = locker.lock(f'{breeder_id}') if not dlm_lock.acquire(acquire_timeout=1200): - task_logger.debug("Could not aquire lock for {breeder_name}_{identifier}") + task_logger.debug("Could not aquire lock for {breeder_id}") asyncio.run(send_msg_via_nats(subject=f'effectuation_{identifier}', data_dict=settings_data)) diff --git a/breeder/linux_network_stack/optimization.py b/breeder/linux_network_stack/optimization.py index 27e19cd0..f7c1e183 100644 --- a/breeder/linux_network_stack/optimization.py +++ b/breeder/linux_network_stack/optimization.py @@ -24,7 +24,7 @@ def run_optimization(): locking_db_url=DLM_DB_CONNECTION, run=run, identifier=identifier, - breeder_name=config.get('name'), + breeder_id=config.get('uuid'), ) __directions = list() diff --git a/breeder/linux_network_stack/root_dag.py b/breeder/linux_network_stack/root_dag.py index e9314343..edbaa997 100644 --- a/breeder/linux_network_stack/root_dag.py +++ b/breeder/linux_network_stack/root_dag.py @@ -107,7 +107,7 @@ parallel_runs = config.get('run').get('parallel') targets = config.get('effectuation').get('targets') -dag_name = config.get('name') +dag_name = config.get('uuid') is_cooperative = config.get('cooperation').get('active') target_id = 0 @@ -137,13 +137,13 @@ def determine_config_shard(run_id=None, for target in targets: - identifier = hashlib.sha256(str.encode(target.get('address'))).hexdigest()[0:6] + hash_suffix = hashlib.sha256(str.encode(target.get('address'))).hexdigest()[0:6] for run_id in range(0, parallel_runs): dag_id = f'{dag_name}_{run_id}' if not is_cooperative: config = determine_config_shard() - globals()[f'{dag_id}_optimization_{identifier}'] = create_optimization_dag(f'{dag_id}_optimization_{identifier}', config, run_id, identifier) - globals()[f'{dag_id}_target_{identifier}'] = create_target_interaction_dag(f'{dag_id}_target_interaction_{identifier}', config, target, identifier) + globals()[f'{dag_id}_optimization_{hash_suffix}'] = create_optimization_dag(f'{dag_id}_optimization_{hash_suffix}', config, run_id, hash_suffix) + globals()[f'{dag_id}_target_{hash_suffix}'] = create_target_interaction_dag(f'{dag_id}_target_interaction_{hash_suffix}', config, target, hash_suffix) target_id += 1 diff --git a/cli/maskfile.md b/cli/maskfile.md index 7170fd3b..2089b099 100644 --- a/cli/maskfile.md +++ b/cli/maskfile.md @@ -88,9 +88,9 @@ curl --request POST \ **OPTIONS** * name - * flags: --name + * flags: --uuid * type: string - * desc: name of breeder to be purged + * desc: uuid of breeder to be purged * hostname * flags: --hostname * type: string @@ -112,7 +112,7 @@ set -eEux __api_version="${api_version:-v0}" curl --request DELETE \ - "http://${hostname}:${port}/${__api_version}/breeders/${name}" + "http://${hostname}:${port}/${__api_version}/breeders/${uuid}" ~~~ ### breeder update @@ -156,9 +156,9 @@ curl --request PUT \ **OPTIONS** * name - * flags: --name + * flags: --uuid * type: string - * desc: name of breeder to get details from + * desc: uuid of breeder to get details from * hostname * flags: --hostname * type: string @@ -181,6 +181,6 @@ __api_version="${api_version:-v0}" curl --request GET \ -H 'Content-Type: application/json' \ - --data "{ \"name\": \"${name}\" }" \ - "http://${hostname}:${port}/${__api_version}/breeders/${name}" + --data "{ \"id\": \"${uuid}\" }" \ + "http://${hostname}:${port}/${__api_version}/breeders/${uuid}" ~~~