first commit

.gitattributes

@@ -0,0 +1 @@
+*.js linguist-detectable=false

.gitignore

@@ -0,0 +1,21 @@
+node_modules/
+.pnp/
+.pnp.js
+coverage/
+build/
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.vscode
+bin/
+volumes/data/*
+!volumes/data/.gitkeep
+volumes/uploads/*
+!volumes/uploads/.gitkeep
+volumes/logs/*
+!volumes/logs/.gitkeep

LICENSE.md

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2021 r7wx
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,62 @@
+<p align="center">
+    <img src="assets/logo.svg">
+</p>
+<h2 align="center">dump-hub</h2>
+<p align="center">
+Self hosted search engine for data leaks and password dumps
+</p>
+
+---
+
+Upload and parse multiple files, then quickly search through all stored items with the power of Elasticsearch.
+
+**Disclaimer:** _This project does not include, and will never include, any data. Data must be uploaded by end users on their own instances of Dump Hub. I take no responsability for the nature of uploaded data._
+
+Dump Hub currenlty supports dumps in csv/combo-list format, the parser is not strict, so if, for instance, one of the lines is not correctly formatted it will still be indexed as a single document and the parsing process will continue through the entire file.
+
+## Docker Compose
+
+Dump Hub can be executed by using docker-compose:
+
+```
+git clone https://github.com/r7wx/dump-hub.git
+cd dump-hub
+docker-compose up --build
+```
+
+Dump Hub will bind port 443 on 0.0.0.0 by default.
+
+**Warning:** _Do not expose Dump Hub on public networks! Please edit the **docker-compose.yml** file to fit your needs, evaluate to use your own SSL certificate and evaluate the usage of basic auth on nginx._
+
+## Usage
+
+To start using Dump Hub open a web browser on https://[dump-hub-ip].
+
+**Warning:** _When you upload a file on Dump Hub, the entries will be parsed and indexed on Elasticsearch. You will be able to perform search queries on every field really quickly, but this comes at a cost: **bigger disk usage**. Keep that in mind if you need to work with a lot of data!_
+
+The upload of a new file can be executed by following 2 steps:
+
+**Multiple File Upload**
+
+To upload files on Dump Hub use the upload page and select the desired files. Once one or more files are selected the upload process will begin. You will find a list of already uploaded files on the first section of the same page. **Warning:** The web uploader has a maximum filesize of **15GB**, if you need to upload bigger files you can upload them directly inside **dump-hub/volumes/uploads** on your dump-hub server (maybe via ssh or ftp).
+
+**Analyze**
+
+In order to add entries on Elasticsearch use the analyze page. From the analyze page you are able to select one of the already uploaded file. Select one file by clicking on it. Once one file is selected, a preview of the file content will be displayed on the box below. You can edit two settings by using the form above the preview box:
+
+- **Starting Line:** The parsing will start from this line. When editing this value the preview will update accordingly.
+- **Separator:** This is the separator character. This char will be used to split entries on line (just like a standard csv).
+
+If the parser is correctly configured you will be able to see parsed items as columns in the table at the bottom of the page. From this table you can select which columns will be parsed and included in the final document (highlighted in green). Each of those fields will be indexed and fully searchable. When the desired result appears in the table you can start the analyze process by clicking on **Analyze File** button.
+
+**Data**
+
+From the Data page you are able to view the list of:
+
+- Entries in processing status (Files that are currently being analyzed and uploaded to Elasticsearch).
+- Entries in deleting status (Files that are currently being deleted from Elasticsearch).
+- Entries in pending status (Files that are waiting to be analyzed or deleted. Only one file can be analyzed or deleted at one time).
+- Entries in error status.
+- Entries in completed status.
+
+From this page you are able to delete entries in completed or error status.

configs/dump-hub.conf

@@ -0,0 +1,33 @@
+server {
+    listen 443 ssl;
+    gzip on;
+
+    ssl_certificate dump-hub.crt;
+    ssl_certificate_key dump-hub.key;
+
+    server_tokens off;
+    add_header Strict-Transport-Security 'max-age=31536000; includeSubdomains';
+    add_header Content-Security-Policy "default-src 'self'; font-src 'self' data:; img-src 'self' data:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline';";
+    add_header X-XSS-Protection "1; mode=block";
+    add_header X-Frame-Options "SAMEORIGIN";
+    add_header X-Content-Type-Options "nosniff";
+    add_header Referrer-Policy "strict-origin";
+    add_header X-Permitted-Cross-Domain-Policies "none";
+
+    client_max_body_size 30M;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+        try_files $uri$args $uri$args/ $uri/ /index.html;
+    }
+
+    location /api {
+        proxy_pass http://dump-hub-be:8080;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}

docker-compose.yml

@@ -0,0 +1,46 @@
+version: "3.9"
+
+services:
+  elasticsearch:
+    image: "elasticsearch:7.11.1"
+    container_name: "dump-hub-elastic"
+    restart: always
+    networks:
+      - dh-net
+    expose:
+      - 9200
+    environment:
+      discovery.type: single-node
+    volumes:
+      - "./volumes/data:/usr/share/elasticsearch/data"
+
+  dump-hub-be:
+    build: ./dump-hub
+    container_name: "dump-hub-be"
+    restart: always
+    networks:
+      - dh-net
+    expose:
+      - 8080
+    depends_on:
+      - elasticsearch
+    volumes:
+      - "./volumes/uploads:/opt/uploads"
+      - "./volumes/logs:/var/log/dump-hub"
+
+  dump-hub-app:
+    build: ./dump-hub-app
+    container_name: "dump-hub-app"
+    restart: always
+    networks:
+      - dh-net
+    ports:
+      - "443:443"
+    depends_on:
+      - dump-hub-be
+    volumes:
+      - "./configs/dump-hub.conf:/etc/nginx/conf.d/default.conf"
+
+networks:
+  dh-net:
+    driver: bridge

dump-hub-app/.dockerignore

@@ -0,0 +1,2 @@
+node_modules/
+build/

dump-hub-app/.env

@@ -0,0 +1 @@
+REACT_APP_BASE_API=/api/

dump-hub-app/Dockerfile

@@ -0,0 +1,13 @@
+FROM node AS react-builder
+WORKDIR /opt/dump-hub-app
+COPY . .
+RUN npm install
+RUN npm run build
+
+FROM nginx AS dump-hub-web
+WORKDIR /usr/share/nginx/html
+COPY --from=react-builder /opt/dump-hub-app/build .
+WORKDIR /etc/nginx
+RUN openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
+    -subj "/C=AU/ST=None/L=None/O=r7wx/CN=dump-hub.priv" \
+    -keyout dump-hub.key  -out dump-hub.crt