Merge pull request #109 from punk-security/add-js-include-takovers

Allow multiple files on ingest

README.md

@@ -103,7 +103,7 @@ options:
   --out OUT             Output file (default: results) - use 'stdout' to stream out
   --out-format {csv,json}
   --resolver RESOLVER
-                        Provide a custom DNS resolver, otherwise it is autodetected
+                        Provide a custom DNS resolver (or multiple seperated by commas)
   --parallelism PARALLELISM
                         Number of domains to test in parallel - too high and you may see odd DNS results (default: 30)
   --disable-probable    Do not check for probable conditions

argparsing.py

@@ -116,7 +116,7 @@ def error(self, message):
     "--resolver",
     type=str,
     default="",
-    help="Provide a custom DNS resolver, otherwise it is autodetected",
+    help="Provide a custom DNS resolver (or multiple seperated by commas)",
 )
 
 

finding.py

@@ -1,6 +1,31 @@
+from os import linesep
+
+
 class Finding(object):
-    def __init__(self, domain, signature, info, confidence):
-        self.domain = domain
+    def __init__(
+        self,
+        domain,
+        signature,
+        info,
+        confidence,
+    ):
+        self.domain = domain.domain
         self.signature = signature
         self.info = info.replace("\n", " ").replace("\r", "").rstrip()
         self.confidence = confidence.name
+        self.a_records = domain.A
+        self.aaaa_records = domain.AAAA
+        self.cname_records = domain.CNAME
+        self.ns_records = domain.NS
+
+    def populated_records(self):
+        resp = ""
+        if self.a_records:
+            resp += f"A: {self.a_records},"
+        if self.aaaa_records:
+            resp += f"AAAA: {self.aaaa_records},"
+        if self.cname_records:
+            resp += f"CNAME: {self.cname_records},"
+        if self.ns_records:
+            resp += f"NS: {self.ns_records},"
+        return resp.rstrip(",")

main.py

@@ -95,7 +95,7 @@
         output_handler=o,
         lock=lock,
         findings=findings,
-        name_server=args.resolver,
+        name_servers=args.resolver.replace(" ", "").split(","),
     )
     pool = ThreadPool(processes=args.parallelism)
     pool.map(scan, domains)

providers/file.py

@@ -1,14 +1,40 @@
 import logging
 from domain import Domain
+from os import listdir
+from os.path import isfile, join
 
-description = "Read domains from a file, one per line"
+description = "Read domains from a file (or folder of files), one per line"
 
 
 def fetch_domains(filename, **args):
-    with open(filename) as file:
-        try:
-            lines = file.readlines()
-            logging.warn(f"Ingested {len(lines)} domains from file '{filename}'")
-        except Exception as e:
-            logging.error(f"Could not read any domains from file {filename} -- {e}")
-    return [Domain(line.rstrip()) for line in lines]
+    if isfile(filename):
+        with open(filename) as file:
+            try:
+                domains = file.readlines()
+                logging.warn(f"Ingested {len(domains)} domains from file '{filename}'")
+            except Exception as e:
+                logging.error(f"Could not read any domains from file {filename} -- {e}")
+                exit(-1)
+    else:
+        domains = []
+        files = fetch_nested_files(filename)
+        for f in files:
+            try:
+                with open(f) as file:
+                    domains += file.readlines()
+                logging.debug(f"Ingested domains from file '{file}'")
+            except:
+                logging.debug(f"Could not read file '{file}'")
+        logging.warn(f"Ingested {len(domains)} domains from folder '{filename}'")
+    return [Domain(domain.rstrip()) for domain in domains]
+
+
+def fetch_nested_files(dir):
+    files = []
+    for item in listdir(dir):
+        path = join(dir, item)
+        if isfile(path):
+            files.append(path)
+        else:
+            files = [*files, *fetch_nested_files(path)]
+    return files

scan.py

@@ -2,10 +2,12 @@
 
 import logging
 
+import random
 
-def scan_domain(domain, signatures, lock, findings, output_handler, name_server=""):
-    if name_server:
-        domain.set_custom_NS(name_server)
+
+def scan_domain(domain, signatures, lock, findings, output_handler, name_servers: list):
+    if name_servers and name_servers != [""]:
+        domain.set_custom_NS(random.choice(name_servers))
     if domain.should_fetch_std_records:
         domain.fetch_std_records()
     for signature in signatures: