New feature and API edits to the hunter API.

Merge branch 'capnspacehook-master'

Common/SimplyEmail.ini

@@ -18,7 +18,7 @@ QueryJitter = 10
 # API keys will be labeld
 # By the service
 [APIKeys]
-Canario: 
+Hunter: 
 
 # Settings for HTML Scrapping module
 # Save can add in a path - default is the SimpleEmail folder with domain name
@@ -31,6 +31,17 @@ Maxfilesize:
 Save: 
 RemoveHTML: Yes
 
+# Settings for Hunter API module
+# Request Limit: sets the maximum number of requests SimplyEmail will make of Hunter API every time its run. Every
+# 10 emails counts as 1 request. Set to 0 for no limit
+# Quota Limit: Hunter gives you 100 or 150 searches on the free plan, depending on whether you registered with a work email or not.
+# This setting will stop making requests when you hit the specified limit
+# EmailType: choose whether to only return generic (ex. [email protected]) or personal (ex. [email protected]) emails or both
+[Hunter]
+RequestLimit: 10
+QuotaLimit: 100
+EmailType: Personal
+
 # You can use a few diffrent Key Servers so a config may be a good idea for this
 [SearchPGP]
 KeyServer: pgp.rediris.es:11371
@@ -47,12 +58,12 @@ QueryStart: 0
 Hostname: flickr.com
 
 #GitHub Code Scraping settigns
-#Page Depth: WARNING every page can contain upto 30 users and multiple links to scrape, this can slow down the results obtain very fast 
+#Page Depth: WARNING every page can contain up to 30 users and multiple links to scrape, this can slow down the results obtain very fast 
 [GitHubSearch]
 PageDepth: 3
 QueryStart: 1
 
-#StartPAge Search engine settings
+#StartPage Search engine settings
 [StartPageSearch]
 StartQuantity: 100
 QueryLimit: 1000

Common/TaskController.py

@@ -136,7 +136,7 @@ def ExecuteModule(self, Task_queue, Results_queue, Html_queue, Json_queue, domai
                 try:
                     # Check for API key to ensure its in .ini
                     if Module.apikey:
-                        if self._execute_api_module(Module):
+                        if self._execute_api_module(Module) == False:
                             break
                     # Emails will be returned as a list
                     Emails, HtmlResults, JsonResults = Module.execute()

Helpers/EmailFormat.py

@@ -2,6 +2,7 @@
 import helpers
 import requests
 import configparser
+import Download
 
 # Email layouts supported:
 # {first}.{last} = [email protected]
@@ -24,12 +25,23 @@ def __init__(self, domain, Verbose=False):
         config = configparser.ConfigParser()
         try:
             config.read('Common/SimplyEmail.ini')
-            self.UserAgent = {
-                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
+            self.UserAgent = str(config['GlobalSettings']['UserAgent'])
+            self.apikeyv = str(config['APIKeys']['Hunter'])
+            self.EmailType = str(config['Hunter']['EmailType'])
             self.domain = domain
             # self.email = email
             self.FinalAnswer = ''
             self.verbose = Verbose
+            # If you send the exact same query twice, Hunter won't count the second request
+            if self.EmailType == "Both":
+                self.type = ""
+                self.etype = "total"
+            elif self.EmailType == "Personal":
+                self.type = "&type=personal"
+                self.etype = "personal_emails"
+            elif self.EmailType == "Generic":
+                self.type = "&type=generic"
+                self.etype = "generic_emails"
         except Exception as e:
             print e
 
@@ -40,28 +52,22 @@ def EmailHunterDetect(self):
         '''
         try:
             # This returns a JSON object
-            url = "https://emailhunter.co/trial/v1/search?offset=0&domain=" + \
-                self.domain + "&format=json"
-            r = requests.get(url)
-        except Exception as e:
-            error = "[!] Major issue with EmailHunter Search:" + str(e)
-            print helpers.color(error, warning=True)
-        try:
+            dl = Download.Download(self.verbose)
+            url = "https://api.hunter.io/v2/domain-search?domain=" + \
+                self.domain + self.type + "&limit=100&offset=0" + "&api_key=" + self.apikeyv
+            r = dl.requesturl(url, useragent=self.UserAgent, raw=True)
             results = r.json()
-            # pprint(results)
-            # Check to make sure we got data back from the API
-            if results['status'] == "success":
-                if results['pattern']:
-                    pattern = results['pattern']
-                    if pattern:
-                        return pattern
+            pattern = str(results['data']['pattern'])
+            if pattern:
+                return pattern
             else:
                 if self.verbose:
                     e = ' [!] No pattern detected via EmailHunter API'
                     print helpers.color(e, firewall=True)
                     return False
         except:
-            pass
+            error = "[!] Major issue with EmailHunter Search:" + str(e)
+            print helpers.color(error, warning=True)
 
     def BuildName(self, CleanName, Format, Raw=False):
         '''

Modules/Hunter.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+import configparser
+import logging
+from Helpers import Download
+from Helpers import Parser
+from Helpers import helpers
+
+# Class will have the following properties:
+# 1) name / description
+# 2) main name called "ClassName"
+# 3) execute function (calls everything it needs)
+# 4) places the findings into a queue
+
+# https://api.hunter.io/v2/domain-search?domain=any.com&type=personal&limit=100&offset=0&api_key=your_api_key
+
+
+class ClassName(object):
+
+    def __init__(self, domain, verbose=False):
+        self.apikey = True
+        self.name = "Hunter API"
+        self.description = "Search the Hunter DB for potential emails"
+        self.domain = domain
+        config = configparser.ConfigParser()
+        self.results = []
+        self.verbose = verbose
+        try:
+            self.logger = logging.getLogger("SimplyEmail.Hunter")
+            config.read('Common/SimplyEmail.ini')
+            self.UserAgent = str(config['GlobalSettings']['UserAgent'])
+            self.apikeyv = str(config['APIKeys']['Hunter'])
+            self.RequestLimit = int(config['Hunter']['RequestLimit'])
+            self.QuotaLimit = int(config['Hunter']['QuotaLimit'])
+            self.EmailType = str(config['Hunter']['EmailType'])
+
+            if self.EmailType == "Both":
+                self.type = ""
+                self.etype = "total"
+            elif self.EmailType == "Personal":
+                self.type = "&type=personal"
+                self.etype = "personal_emails"
+            elif self.EmailType == "Generic":
+                self.type = "&type=generic"
+                self.etype = "generic_emails"
+            else:
+                raise Exception("Email Type setting invalid")
+        except Exception as e:
+            self.logger.critical("Hunter module failed to __init__: " + str(e))
+            print helpers.color(" [*] Error in Hunter settings: " + str(e) + "\n", warning=True)
+
+    def execute(self):
+        self.logger.debug("Hunter module started")
+        self.process()
+        FinalOutput, HtmlResults, JsonResults = self.get_emails()
+        return FinalOutput, HtmlResults, JsonResults
+
+    def process(self):
+        dl = Download.Download(self.verbose)
+        try:
+            # We will check to see that we have enough requests left to make a search
+            url = "https://api.hunter.io/v2/account?api_key=" + self.apikeyv
+            r = dl.requesturl(url, useragent=self.UserAgent, raw=True)
+            accountInfo = r.json()
+            quota = int(accountInfo['data']['calls']['available'])
+            quotaUsed = int(accountInfo['data']['calls']['used'])
+            if quotaUsed >= self.QuotaLimit:
+                overQuotaLimit = True
+            else:
+                overQuotaLimit = False
+        except Exception as e:
+            error = " [!] Hunter API error: " + str(accountInfo['errors'][0]['details'])
+            print helpers.color(error, warning=True)      
+        try:
+            # Hunter's API only allows 100 emails per request, so we check the number of emails Hunter has 
+            # on our specified domain, and if it's over 100 we need to make multiple requests to get all of the emails
+            url = "https://api.hunter.io/v2/email-count?domain=" + self.domain
+            r = dl.requesturl(url, useragent=self.UserAgent, raw=True)
+            response = r.json()
+            totalEmails = int(response['data'][self.etype])
+            emailsLeft = totalEmails
+            offset = 0
+        except Exception as e:
+            error = "[!] Major issue with Hunter Search: " + str(e)
+            print helpers.color(error, warning=True)
+        requestsMade = 0
+        # Main loop to keep requesting the Hunter API until we get all of the emails they have
+        while emailsLeft > 0:
+            try:
+                if overQuotaLimit or requestsMade + quotaUsed >= self.QuotaLimit:
+                    if self.verbose:
+                        print helpers.color(" [*] You are over your set Quota Limit: " + \
+                            str(quotaUsed) + "/" + str(self.QuotaLimit) + " stopping search", firewall=True)
+                    break
+                elif self.RequestLimit != 0 and requestsMade >= self.RequestLimit:
+                    if self.verbose:
+                        print helpers.color(" [*] Stopping search due to user set Request Limit", firewall=True)
+                    break
+
+                # This returns a JSON object
+                url = "https://api.hunter.io/v2/domain-search?domain=" + \
+                    self.domain + self.type + "&limit=100&offset=" + str(offset) + "&api_key=" + self.apikeyv
+                r = dl.requesturl(url, useragent=self.UserAgent, raw=True)
+                results = r.json()
+                emailCount = int(results['meta']['results'])
+            except Exception as e:
+                error = " [!] Hunter API error: " + str(results['errors'][0]['details']) + " QUITTING!"
+                print helpers.color(error, warning=True)
+                break
+            try:
+                # Make sure we don't exceed the index for the 'emails' array in the 'results' Json object
+                if emailsLeft < 100:
+                    emailCount = emailsLeft
+                if emailCount > 100:
+                    emailCount = 100
+                # 1 request is every 10 emails delivered
+                requestsMade += emailCount // 10
+                if emailCount % 10 != 0:
+                    requestsMade += 1
+                # The API starts at 0 for the first value
+                x = 0
+                # We will itirate of the Json object for the index objects
+                while x < emailCount:
+                    self.results.append(results['data']['emails'][int(x)]['value'])
+                    x += 1
+                emailsLeft -= emailCount
+                if emailsLeft > 100:
+                    offset += 100
+                else:
+                    offset += emailsLeft
+            except Exception as e:
+                error = " [!] Major issue with search parsing: " + str(e)
+                print helpers.color(error, warning=True)
+                break
+        if self.verbose:
+            # Print the avalible requests user has if verbose
+            print helpers.color(' [*] Hunter has completed JSON request', firewall=True)
+            requestsUsed = requestsMade + quotaUsed
+            if quota - requestsUsed <= 0:
+                print helpers.color(" [*] You have no Hunter requests left." \
+                    + "They will refill in about a month", firewall=True)
+            else:
+                print helpers.color(" [*] You have " + str(requestsUsed) \
+                    + "/" + str(quota) + " Hunter requests left", firewall=True) 
+
+    def get_emails(self):
+        # Make sure you remove any newlines
+        Parse = Parser.Parser(self.results)
+        FinalOutput = Parse.CleanListOutput()
+        HtmlResults = Parse.BuildResults(FinalOutput, self.name)
+        JsonResults = Parse.BuildJson(FinalOutput, self.name)
+        self.logger.debug('Hunter completed search')
+        return FinalOutput, HtmlResults, JsonResults

README.md

@@ -10,7 +10,7 @@ SimplyEmail was built arround the concept that tools should do somthing, and do
 
 [INSTALL / FAQ] https://simplysecurity.github.io/SimplyEmail/
 
-[CHANGELOG] https://github.com/SimplySecurity/SimplyEmail/blob/master/setup/changelog
+[CHANGELOG] https://github.com/SimplySecurity/SimplyEmail/blob/master/CHANGELOG.md
 
 ## TL;DR 
 Supported Platforms / Tested with CI: