From d2fc7baa28449871aa72944eb75fe96de5e240da Mon Sep 17 00:00:00 2001 From: brblacky <77108939+brblacky@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:25:52 +0530 Subject: [PATCH] new --- remove_duplicates.py | 30 ++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 31 insertions(+) create mode 100644 remove_duplicates.py create mode 100644 requirements.txt diff --git a/remove_duplicates.py b/remove_duplicates.py new file mode 100644 index 0000000..d22f0e6 --- /dev/null +++ b/remove_duplicates.py @@ -0,0 +1,30 @@ +import json + +# Load data from nodes.json +with open('nodes.json') as f: + data = json.load(f) + +# Function to remove duplicate nodes +def remove_duplicates(data): + seen_identifiers = set() + unique_data = [] + for node in data: + identifier = node['identifier'] + if identifier not in seen_identifiers: + seen_identifiers.add(identifier) + unique_data.append(node) + return unique_data + +# Remove duplicates +unique_data = remove_duplicates(data) + +# Check if any nodes were removed +if len(unique_data) < len(data): + print("Duplicate nodes removed.") + + # Save updated data back to nodes.json + with open('nodes.json', 'w') as f: + json.dump(unique_data, f, indent=4) +else: + print("No duplicate nodes found.") + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c87a0a0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +json \ No newline at end of file