Refactor seeding of demo data and changes keys skip list update

Makefile

@@ -18,18 +18,20 @@ docs:
 local: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/localhost:8443' docker compose up --build -d
+	sleep 60
+	cd ./elk && ./seed.sh add_mappings add_configs add_tokens add_users
 
 demo: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/demo.paradrop.io' docker compose up --build -d
 	sleep 60
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 develop: npm mkcert docs
 	sudo docker compose down --remove-orphans
 	sudo URL='https:\/\/develop.paradrop.io' docker compose up --build -d
 	sleep 60
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 ui: npm
 	./ui/http_server.py
@@ -45,7 +47,7 @@ up:
 	sudo URL='https:\/\/127.0.0.1' docker compose up --build -d
 
 seed:
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 elk:
 	sudo docker rm -f opensearch
@@ -63,7 +65,7 @@ elk:
 		-v "${PWD}"/elk/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml \
 		opensearchproject/opensearch-dashboards:2.11.1
 
-	cd ./elk && ./seed.sh
+	cd ./elk && ./seed.sh demo
 
 cbuilds: npm mkcert
 	cd ./api && sudo docker build -t $(API_NAME) .

README.md

@@ -18,7 +18,7 @@ paradrop is an open source, cybersecurity asset management tool that collects op
 
 ```bash
 make local
-# Optional: Load test / example data
+# Optional: Load test / example demo data
 make seed
 ```
 

Vagrantfile

@@ -43,7 +43,7 @@ ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get update -y
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get remove -y whoopsie apport apport-gtk ubuntu-report unattended-upgrades kerneloops plymouth thunderbird transmission-common cheese aisleriot gnome-mahjongg gnome-mines gnome-sudoku remmina mlocate
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get autoremove -y
 ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
-ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq vim net-tools dnsutils screen nodejs python3-pip python3-dev make unzip htop clamav libopenscap8 dmidecode shellcheck
+ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq vim net-tools dnsutils screen nodejs python3-pip python3-dev make unzip htop clamav libopenscap8 dmidecode shellcheck wget apt-transport-https gnupg lsb-release
 
 # Setup Docker
 curl -fsSL https://get.docker.com -o ./get-docker.sh
@@ -54,14 +54,13 @@ systemctl enable docker
 systemctl restart docker
 
 # Install Trivy Scanner
-apt-get install wget apt-transport-https gnupg lsb-release
 wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
 echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
 apt-get update
 apt-get install trivy
 
 # Install OpenScap Content Guides
-wget -c https://github.com/ComplianceAsCode/content/releases/download/v0.1.72/scap-security-guide-0.1.72.zip
+curl -OLs https://github.com/ComplianceAsCode/content/releases/download/v0.1.72/scap-security-guide-0.1.72.zip
 unzip scap-security-guide-0.1.72.zip
 mkdir -p /usr/share/scap-security-guide
 cp -rf scap-security-guide-0.1.72/* /usr/share/scap-security-guide/
@@ -75,7 +74,7 @@ pip3 install flake8
 # Setup Golang
 GOVER="1.22.3"
 curl -OLs "https://golang.org/dl/go$GOVER.linux-amd64.tar.gz"
-tar -zxvf ./"go$GOVER.linux-amd64.tar.gz"
+tar -zxf ./"go$GOVER.linux-amd64.tar.gz"
 mv -f ./go /usr/local/
 rm -f ./"go$GOVER.linux-amd64.tar.gz"
 ln -s /usr/local/go/bin/go /usr/bin/go
@@ -107,9 +106,7 @@ EOF
 
 cp -f /paradrop/agent/paradrop-agent /usr/bin/
 
-sleep 30
-
-timeout 120 paradrop-agent -d
+timeout 200 paradrop-agent -d
 
 # Restart
 systemctl reboot

api/db/changes.py

@@ -19,13 +19,30 @@ async def db_add_changes(original_host_data: dict, new_host_data: dict) -> dict:
 
         # List of all keys that we don't want to include in the comparison.
         keys_to_skip: list = [
-            "dmesg_errors",
-            "users_loggedin",
-            "last_run",
-            "failed_logins",
+            "diskfree_gb",
+            "diskused_gb",
+            "diskused_pct",
+            "docker_containers",
+            "docker_images_count",
+            "docker_images",
+            "docker_labels",
             "id",
-            "processes",
             "network_interfaces",
+            "journalctl_logs",
+            "last_run",
+            "load1",
+            "load5",
+            "load15",
+            "memoryfree_gb",
+            "memoryused_gb",
+            "memoryused_pct",
+            "processes",
+            "sysctl",
+            "systemd_timers",
+            "trivy",
+            "uptime_days",
+            "users_loggedin",
+            "dmesg_errors",
         ]
 
         changes: dict = {}

elk/es_settings.json

@@ -6,5 +6,9 @@
         "filter": ["lowercase"]
       }
     }
+  },
+  "index": {
+    "max_result_window": 100000,
+    "number_of_replicas": 0
   }
 }