mongodb+srv:// seed-list support (#289)

* add mongodb+srv:// support

* Cleanup

* Remove test logging

* Move raise -> log+raise

* Fix URI check + build.sh

* Fix missing 'e' in except-line

* revert 3.6 tools

* 1.4.0 -> 1.4.1 version

* Fix flake8 warn

* Add --no-cache back

VERSION

@@ -1 +1 @@
-1.4.0
+1.4.1

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -126,20 +126,24 @@ def run(self):
 
         # backup a secondary from each shard:
         for shard in self.replsets:
-            secondary = self.replsets[shard].find_secondary()
-            mongo_uri = secondary['uri']
-            self.states[shard] = OplogState(self.manager, mongo_uri)
-            thread = MongodumpThread(
-                self.states[shard],
-                mongo_uri,
-                self.timer,
-                self.config,
-                self.backup_dir,
-                self.version,
-                self.threads(),
-                self.do_gzip()
-            )
-            self.dump_threads.append(thread)
+            try:
+                secondary = self.replsets[shard].find_secondary()
+                mongo_uri = secondary['uri']
+                self.states[shard] = OplogState(self.manager, mongo_uri)
+                thread = MongodumpThread(
+                    self.states[shard],
+                    mongo_uri,
+                    self.timer,
+                    self.config,
+                    self.backup_dir,
+                    self.version,
+                    self.threads(),
+                    self.do_gzip()
+                )
+                self.dump_threads.append(thread)
+            except Exception, e:
+                logging.error("Failed to get secondary for shard %s: %s" % (shard, e))
+                raise e
 
         if not len(self.dump_threads) > 0:
             raise OperationError('No backup threads started!')

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -138,12 +138,24 @@ def wait(self):
     def mongodump_cmd(self):
         mongodump_uri   = self.uri.get()
         mongodump_cmd   = [self.binary]
-        mongodump_flags = [
-            "--host=%s" % mongodump_uri.host,
-            "--port=%s" % str(mongodump_uri.port),
+        mongodump_flags = []
+
+        # --host/--port (suport mongodb+srv:// too)
+        if self.uri.srv:
+            if not self.is_version_gte("3.6.0"):
+                logging.fatal("Mongodump must be >= 3.6.0 to use mongodb+srv:// URIs")
+                sys.exit(1)
+            mongodump_flags.append("--host=%s" % self.uri.url)
+        else:
+            mongodump_flags.extend([
+                "--host=%s" % mongodump_uri.host,
+                "--port=%s" % str(mongodump_uri.port)
+            ])
+
+        mongodump_flags.extend([
             "--oplog",
             "--out=%s/dump" % self.backup_dir
-        ]
+        ])
 
         # --numParallelCollections
         if self.threads > 0:

mongodb_consistent_backup/Common/DB.py

@@ -3,7 +3,7 @@
 from bson.codec_options import CodecOptions
 from inspect import currentframe, getframeinfo
 from pymongo import DESCENDING, CursorType, MongoClient
-from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError
+from pymongo.errors import ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError
 from ssl import CERT_REQUIRED, CERT_NONE
 from time import sleep
 
@@ -113,7 +113,7 @@ def connect(self):
             conn = MongoClient(**self.client_opts())
             if self.do_connect:
                 conn['admin'].command({"ping": 1})
-        except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
+        except (ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
             logging.error("Unable to connect to %s! Error: %s" % (self.uri, e))
             raise DBConnectionError(e)
         if conn is not None:

mongodb_consistent_backup/Common/MongoUri.py

@@ -1,4 +1,7 @@
+import re
+
 from Util import validate_hostname
+from mongodb_consistent_backup.Errors import OperationError
 
 
 class MongoAddr:
@@ -22,19 +25,24 @@ def __init__(self, url, default_port=27017, replset=None):
         self.default_port = default_port
         self.replset      = replset
 
+        self.srv      = False
         self.addrs    = []
         self.addr_idx = 0
 
         self.parse()
 
     def hosts(self):
+        if self.srv:
+            return self.url
         if len(self.addrs) > 0:
             hosts = []
             for addr in self.addrs:
                 hosts.append(str(addr))
             return ",".join(hosts)
 
     def str(self):
+        if self.srv:
+            return self.url
         string = self.hosts()
         if self.replset:
             string = "%s/%s" % (self.replset, string)
@@ -44,6 +52,15 @@ def __str__(self):
         return self.str()
 
     def parse(self):
+        # allow mongodb+srv:// URI
+        if self.url.startswith("mongodb+srv://"):
+            rsSearch = re.search(r'replicaSet=(\S+)(&.+)?$', self.url)
+            if not rsSearch:
+                raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI")
+            self.replset = rsSearch.group(1)
+            self.srv = True
+            return True
+
         if "/" in self.url:
             self.replset, self.url = self.url.split("/")
         for url in self.url.split(","):

mongodb_consistent_backup/Pipeline/Stage.py

@@ -84,6 +84,7 @@ def run(self):
                 data = self._task.run()
                 self.stopped = True
             except Exception, e:
+                logging.error("State %s returned error: %s" % (self.stage, e))
                 raise OperationError(e)
             finally:
                 self.running = False

requirements.txt

@@ -10,3 +10,4 @@ yconf==0.3.4
 google_compute_engine==2.7.2
 progress==1.3
 py-zabbix==1.1.3
+dnspython==1.15

scripts/build.sh

@@ -103,7 +103,14 @@ if [ -d ${srcdir} ]; then
 	pip_flags="--download-cache=${pipdir}"
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pip --help | grep -q '\-\-cache\-dir'
 	[ $? = 0 ] && pip_flags="--cache-dir=${pipdir}"
-	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex requests
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "requests"
+	if [ $? -gt 0 ]; then
+		echo "Failed to install 'requests'!"
+		exit 1
+	fi
+
+	# build fails on Pex 1.5+
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "pex<=1.4"
 	if [ $? -gt 0 ]; then
 		echo "Failed to install pex utility for building!"
 		exit 1