From 058b8505792b0c81ef523673754caaf1be291ef5 Mon Sep 17 00:00:00 2001 From: Oussama BENGHECHOUA Date: Tue, 2 Apr 2024 15:59:20 +0200 Subject: [PATCH] Add an update script for CMS Add a script to update CMS hash files from a given database --- .../src_cms_files/list_files_drupal.txt | 30 ++++++ .../src_cms_files/list_files_joomla-cms.txt | 20 ++++ .../src_cms_files/list_files_prestashop.txt | 33 +++++++ .../attacks/src_cms_files/list_files_spip.txt | 27 ++++++ .../src_cms_files/list_files_wordpress.txt | 30 ++++++ wapitiCore/data/attacks/update_hash_files.py | 97 +++++++++++++++++++ 6 files changed, 237 insertions(+) create mode 100644 wapitiCore/data/attacks/src_cms_files/list_files_drupal.txt create mode 100644 wapitiCore/data/attacks/src_cms_files/list_files_joomla-cms.txt create mode 100644 wapitiCore/data/attacks/src_cms_files/list_files_prestashop.txt create mode 100644 wapitiCore/data/attacks/src_cms_files/list_files_spip.txt create mode 100644 wapitiCore/data/attacks/src_cms_files/list_files_wordpress.txt create mode 100755 wapitiCore/data/attacks/update_hash_files.py diff --git a/wapitiCore/data/attacks/src_cms_files/list_files_drupal.txt b/wapitiCore/data/attacks/src_cms_files/list_files_drupal.txt new file mode 100644 index 000000000..bd7cc3c63 --- /dev/null +++ b/wapitiCore/data/attacks/src_cms_files/list_files_drupal.txt @@ -0,0 +1,30 @@ +misc/tabledrag.js +misc/tableheader.js +misc/drupal.js +misc/ajax.js +core/misc/drupal.js +core/misc/tabledrag.js +core/misc/tableheader.js +core/misc/ajax.js +CHANGELOG.txt +core/MAINTAINERS.txt +README.txt +core/assets/vendor/ckeditor/skins/moono-lisa/editor_gecko.css +core/modules/system/js/system.date.js +core/scripts/css/compile.js +core/themes/claro/css/theme/media-library.css +core/themes/seven/css/theme/media-library.css +misc/jquery.js +misc/states.js +modules/contextual/contextual.css +core/misc/cspell/dictionary.txt +core/misc/dialog/off-canvas.reset.css +core/profiles/demo_umami/themes/umami/css/components/navigation/tabs/tabs.css +core/INSTALL.txt +core/assets/vendor/ckeditor/build-config.js +core/themes/claro/css/components/views-ui.pcss.css +core/misc/vertical-tabs.js +core/modules/ckeditor5/js/ckeditor5.admin.js +core/misc/dialog/off-canvas.js +core/themes/claro/css/components/views-ui.css +core/themes/claro/css/components/tables.css diff --git a/wapitiCore/data/attacks/src_cms_files/list_files_joomla-cms.txt b/wapitiCore/data/attacks/src_cms_files/list_files_joomla-cms.txt new file mode 100644 index 000000000..20568dcc7 --- /dev/null +++ b/wapitiCore/data/attacks/src_cms_files/list_files_joomla-cms.txt @@ -0,0 +1,20 @@ +LICENSE.txt +administrator/manifests/files/joomla.xml +administrator/templates/isis/js/template.js +libraries/simplepie/README.txt +media/com_finder/js/indexer.js +media/editors/codemirror/addon/search/matchesonscrollbar.js +media/editors/codemirror/keymap/sublime.js +media/plg_system_stats/js/stats.js +media/system/css/calendar-jos.css +media/system/js/helpsite.js +media/system/js/mootools-core.js +media/system/js/mootree-uncompressed.js +media/system/js/tabs.js +media/system/js/validate.js +plugins/finder/content/content.xml +plugins/system/cache/cache.xml +templates/system/css/error.css +templates/system/css/general.css +templates/system/css/offline.css +templates/system/css/system.css diff --git a/wapitiCore/data/attacks/src_cms_files/list_files_prestashop.txt b/wapitiCore/data/attacks/src_cms_files/list_files_prestashop.txt new file mode 100644 index 000000000..a51ff5713 --- /dev/null +++ b/wapitiCore/data/attacks/src_cms_files/list_files_prestashop.txt @@ -0,0 +1,33 @@ +admin/filemanager/css/style.css +admin/themes/new-theme/js/app/store/index.js +admin/themes/new-theme/js/app/utils/datepicker.js +js/admin.js +js/admin/carrier_wizard.js +js/admin/dashboard.js +js/admin/dnd.js +js/admin/modules-position.js +js/admin/notifications.js +js/cropper/prototype.js +modules/ps_facetedsearch/_dev/back/index.js +modules/ps_linklist/views/public/form.bundle.js +modules/ps_themecusto/views/css/controllers/configuration/back.css +modules/ps_themecusto/views/css/general.css +modules/psgdpr/views/css/back.css +modules/psgdpr/views/css/faq.css +modules/psgdpr/views/js/back.js +modules/psgdpr/views/js/vfs_fonts.js +themes/classic/assets/css/theme.css +admin/themes/new-theme/public/product.css +css/tabpane.css +LICENSES +INSTALL.txt +admin-dev/ajaxfilemanager/jscripts/ajaxfilemanager_c.js +admin-dev/ajaxfilemanager/jscripts/ajaximageeditor.js +admin-dev/filemanager/img/dimension.png +admin-dev/themes/default/css/admin-theme.css +admin-dev/themes/default/css/admin-theme_rtl.css +admin-dev/themes/default/css/schemes/admin-theme-blue.css +admin-dev/themes/default/css/schemes/admin-theme-contrast.css +admin-dev/themes/default/js/bundle/product/form.js +js/jquery-google-reviews.js +js/tools.js diff --git a/wapitiCore/data/attacks/src_cms_files/list_files_spip.txt b/wapitiCore/data/attacks/src_cms_files/list_files_spip.txt new file mode 100644 index 000000000..f6e0b5cd3 --- /dev/null +++ b/wapitiCore/data/attacks/src_cms_files/list_files_spip.txt @@ -0,0 +1,27 @@ +ecrire/polices/dustismo-license.txt +ecrire/safehtml/license.txt +ecrire/safehtml/readme-SPIP.txt +ecrire/safehtml/readme.txt +COPYING.txt +htaccess.txt +CHANGELOG.txt +INSTALL.txt +plugins-dist.json +prive/formulaires/dateur/jquery.dateur.js.html +prive/formulaires/selecteur/jquery.picker.js +prive/javascript/SearchHighlight.js +prive/javascript/ajaxCallback.js +prive/javascript/gadgets.js +prive/javascript/jquery.autosave.js +prive/javascript/jquery.form.js +prive/javascript/jquery.js +prive/javascript/layer.js +prive/javascript/login.js +prive/aide_body.css +prive/spip_admin.css +prive/spip_style.css +prive/themes/spip/content.css.html +prive/themes/spip/forms.css.html +ecrire/paquet.xml +prive/objets/liste/articles.html +prive/javascript/ajaxCallback.js diff --git a/wapitiCore/data/attacks/src_cms_files/list_files_wordpress.txt b/wapitiCore/data/attacks/src_cms_files/list_files_wordpress.txt new file mode 100644 index 000000000..95b25f6b6 --- /dev/null +++ b/wapitiCore/data/attacks/src_cms_files/list_files_wordpress.txt @@ -0,0 +1,30 @@ +wp-admin/css/about.min.css +wp-admin/css/common-rtl.css +wp-admin/css/common.css +wp-admin/css/customize-controls.css +wp-admin/css/dashboard.css +wp-admin/css/list-tables-rtl.css +wp-admin/js/customize-controls.js +wp-admin/js/editor.min.js +wp-admin/js/inline-edit-post.min.js +wp-admin/js/media-gallery.min.js +wp-content/themes/twentynineteen/style-rtl.css +wp-content/themes/twentyeleven/style.css +wp-includes/css/dist/block-library/style.min.css +wp-includes/css/editor-rtl.css +wp-includes/css/media-views-rtl.min.css +wp-includes/css/media-views.css +wp-includes/js/dist/a11y.js +wp-includes/js/dist/block-editor.js +wp-includes/js/dist/block-editor.min.js +wp-includes/js/dist/block-library.js +wp-includes/js/dist/components.js +wp-includes/js/dist/core-data.js +wp-includes/js/dist/element.js +wp-includes/js/dist/data.js +wp-includes/js/dist/private-apis.js +wp-includes/js/dist/url.js +wp-includes/js/mce-view.js +wp-includes/js/mce-view.min.js +wp-includes/js/wp-ajax-response.js +wp-includes/theme.json diff --git a/wapitiCore/data/attacks/update_hash_files.py b/wapitiCore/data/attacks/update_hash_files.py new file mode 100755 index 000000000..2ea5df18c --- /dev/null +++ b/wapitiCore/data/attacks/update_hash_files.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import argparse +import json +import logging +from pathlib import Path +import re +import sqlite3 + +# Set up basic configuration for logging +logging.basicConfig(level=logging.INFO, + format='%(levelname)s - %(message)s') + +parser = argparse.ArgumentParser( + description=( + "To execute this script correctly, ensure the following:\n\n" + "1. The database file must be specified using the '--db' option.\n" + "2. The 'src_cms_files' directory must be specified using the '--source-file' option.\n" + "3. The 'src_cms_files' directory must contain one .txt file per CMS.\n" + "4. Each .txt file must contain specific paths for each CMS, with one path per line.\n" + ), + formatter_class=argparse.RawTextHelpFormatter +) + +# Add arguments for database path and source files directory +parser.add_argument('--db', required=True, help="Path to the database file") +parser.add_argument('--source-file', required=True, help="Path to the 'src_cms_files' directory") + +# Parse the arguments +args = parser.parse_args() + +tech_list = [] +file_lists = {} +src_dir = Path(args.source_file) +if not src_dir.is_dir(): + raise FileNotFoundError(f"Directory '{src_dir}' does not exist") + + +txt_pattern = r".*list_files_(.*)\.txt" +pattern = re.compile(txt_pattern) + +with sqlite3.connect(args.db) as conn: + cursor = conn.cursor() + # Extract technology names from the database + sql_tech = "SELECT DISTINCT technology FROM hash;" + cursor.execute(sql_tech) + for row in cursor.fetchall(): + tech = row[0].lower() + if tech not in tech_list: + tech_list.append(tech) + + # Create the list object of the source files + list_files = [file for file in src_dir.glob("list_files_*.txt")] + if not list_files: + raise AssertionError(f"The directory '{src_dir}' is empty or no file match the pattern.") + for file in list_files: + match = re.match(pattern, str(file)) + if match: + tool = match.group(1) + else: + logging.warning(f"Warning! Filename {file} does not match pattern {txt_pattern}.") + continue + file_lists[tool] = [] + # Read the file contents and extract file_names + with open(file, "r") as file_buffer: + for line in file_buffer: + file_lists[tool].append(line.strip()) + + # Create an empty JSON for each technology + json_objects = {} + + # Check if each technology exists in the database + for tech_name in file_lists.keys(): + logging.info(f"Updating {tech_name} :") + if tech_name not in tech_list: + logging.warning(f"Technology '{tech_name}' does not exist in the 'hash' table.") + continue + json_objects[tech_name] = {} + logging.info("DONE!") + + for file_name in file_lists[tech_name]: + cursor = conn.cursor() + # Retrieve data from the hash table for the current file name + cursor.execute("SELECT file, hash, versions FROM hash WHERE file = ? AND UPPER(technology) = UPPER(?)", (file_name, tech_name)) + data = cursor.fetchall() + # Convert data to a list of dictionaries + for row in data: + file_name, hash_value, versions = row + if file_name not in json_objects[tech_name]: + json_objects[tech_name][file_name] = {} + versions_str = json.loads(versions) + versions_json = json.loads(versions_str) + json_objects[tech_name][file_name][hash_value] = versions_json["versions"] + + # Save the JSON data to a file + for key, value in json_objects.items(): + with open(f"{key}_hash_files.json", "w") as json_file: + json.dump(value, json_file, indent=4)