ZeroMQ Island

CMakeLists.txt

@@ -68,6 +68,9 @@ OPTION(ENABLE_WORHP "Enable support for WORHP minimiser." OFF)
 # Build Option: support for MPI clustering.
 OPTION(ENABLE_MPI "Enable support for the Message Passage Interface (MPI)." OFF)
 
+# Build Option: ZeroMQ islands.
+OPTION(ENABLE_ZMQ "Enable support for ZeroMQ islands." OFF)
+
 # Build option: enable test set.
 OPTION(ENABLE_TESTS "Build test set." OFF)
 
@@ -199,6 +202,12 @@ IF(ENABLE_MPI)
  ADD_DEFINITIONS(-DPAGMO_ENABLE_MPI)
 ENDIF(ENABLE_MPI)
 
+IF(ENABLE_ZMQ)
+ ADD_DEFINITIONS(-DPAGMO_ENABLE_ZMQ)
+ SET(MANDATORY_LIBRARIES ${MANDATORY_LIBRARIES} zmq redox ev hiredis)
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") # workaround for void perform_evolution() const 
+ENDIF(ENABLE_ZMQ)
+
 # If GSL support is requested, look for the library.
 IF(ENABLE_GSL)
  FIND_LIBRARY(GSL_GSL_LIBRARY NAMES gsl)

PyGMO/core/__init__.py

@@ -18,6 +18,9 @@
  'population',
  'py_island']
 
+if '_zmq_island' in dir(_core):
+ __all__.append('zmq_island')
+
 _orig_signal = _signal.getsignal(_signal.SIGINT)
 _main_pid = _os.getpid()
 
@@ -136,6 +139,84 @@ def _generic_island_ctor(self, *args, **kwargs):
 local_island.__original_init__ = local_island.__init__
 local_island.__init__ = _generic_island_ctor
 
+if '_zmq_island' in dir(_core):
+# Raw C++ ZMQ island class.
+ _zmq_island = _core._zmq_island
+
+ class zmq_island(_core._zmq_island):
+ def __init__(self, *args, **kwargs):
+ """ZMQ Island. Unnamed arguments:
+
+ #. algorithm
+ #. problem or population
+ #. number of individuals (optional and valid only if the second argument is a problem, defaults to 0 if not specified)
+
+ Keyword arguments:
+
+ * *s_policy* -- migration selection policy (defaults to 'best selection' policy)
+ * *r_policy* -- migration replacement policy (defaults to 'fair replacement' policy)
+
+ """
+ if len(args) == 0:
+ raise ValueError(
+ "Cannot initialise ZeroMQ island without parameters for the constructor.")
+
+ from PyGMO.algorithm._algorithm import _base as _base_algorithm
+ from PyGMO.algorithm import base as base_algorithm
+ from PyGMO.problem._problem import _base as _base_problem
+ from PyGMO.problem._problem import _base_stochastic as _base_problem_stochastic
+ from PyGMO.problem import base as base_problem
+ from PyGMO.problem import base_stochastic as base_problem_stochastic
+ from PyGMO.migration._migration import best_s_policy, fair_r_policy, _base_s_policy, _base_r_policy
+
+ if len(args) < 2 or len(args) > 3:
+ raise ValueError(
+ "Unnamed arguments list must have either 2 or three elements, but %d elements were found instead." %
+ (len(args),))
+ if not isinstance(args[0], _base_algorithm):
+ raise TypeError("The first unnamed argument must be an algorithm.")
+ ctor_args = [args[0]]
+ if isinstance(args[1], _base_problem) or isinstance(args[1], _base_problem_stochastic):
+ ctor_args.append(args[1])
+ if len(args) == 3:
+ if not isinstance(args[2], int):
+ raise TypeError(
+ "Please provide an integer for the number of individuals in the island.")
+ ctor_args.append(args[2])
+ else:
+ ctor_args.append(0)
+ elif isinstance(args[1], population):
+ if len(args) == 3:
+ raise ValueError(
+ "When the second unnamed argument is a population, there cannot be a third unnamed argument.")
+ ctor_args.append(args[1])
+ else:
+ raise TypeError(
+ "The second unnamed argument must be either a problem or a population.")
+
+ if 's_policy' in kwargs:
+ ctor_args.append(kwargs['s_policy'])
+ else:
+ ctor_args.append(best_s_policy())
+ if not isinstance(ctor_args[-1], _base_s_policy):
+ raise TypeError("s_policy must be a migration selection policy.")
+
+ if 'r_policy' in kwargs:
+ ctor_args.append(kwargs['r_policy'])
+ else:
+ ctor_args.append(fair_r_policy())
+ if not isinstance(ctor_args[-1], _base_r_policy):
+ raise TypeError("r_policy must be a migration replacement policy.")
+
+ super(type(self), self).__init__(*ctor_args)
+
+ def get_name(self):
+ return str(type(self))
+
+ def __get_deepcopy__(self):
+ from copy import deepcopy
+ return deepcopy(self)
+
 # This is the function that will be called by the separate process
 # spawned from py_island.
 

PyGMO/core/core.cpp

@@ -49,6 +49,9 @@
 #include "../../src/algorithm/base.h"
 #include "../../src/archipelago.h"
 #include "../../src/base_island.h"
+#ifdef PAGMO_ENABLE_ZMQ
+ #include "../../src/zmq_island.h"
+#endif
 #include "../../src/config.h"
 #include "../../src/exceptions.h"
 #include "../../src/migration/base_r_policy.h"
@@ -380,6 +383,20 @@ BOOST_PYTHON_MODULE(_core)
  // Register to_python conversion from smart pointer.
  register_ptr_to_python<base_island_ptr>();
 
+#ifdef PAGMO_ENABLE_ZMQ
+ // ZMQ island class.
+ class_<zmq_island,bases<base_island> >("_zmq_island", "ZMQ island class.",init<const algorithm::base &, const problem::base &, optional<int,const migration::base_s_policy &,const migration::base_r_policy &> >())
+ .def(init<const algorithm::base &, const population &, optional<const migration::base_s_policy &,const migration::base_r_policy &> >())
+ .def(init<const zmq_island &>())
+ .def("__copy__", &Py_copy_from_ctor<zmq_island>)
+ .def("__deepcopy__", &Py_deepcopy_from_ctor<zmq_island>)
+ .def("set_broker_details", &zmq_island::set_broker_details)
+ .def("set_token", &zmq_island::set_token)
+ .def("set_ip", &zmq_island::set_ip)
+ .def("set_evolve", &zmq_island::set_evolve)
+ .def("connect", &zmq_island::connect);
+#endif
+
  // Expose archipelago class.
  class_<archipelago>("archipelago", "Archipelago class.", init<const algorithm::base &, const problem::base &,
  int,int,optional<const topology::base &,archipelago::distribution_type,archipelago::migration_direction> >())

PyGMO/utils.h

@@ -90,7 +90,7 @@ struct python_class_pickle_suite: boost::python::pickle_suite
  throw_error_already_set();
  }
  // Restore the object's __dict__.
- dict d = extract<dict>(obj.attr("__dict__"))();
+ boost::python::dict d = extract<boost::python::dict>(obj.attr("__dict__"))();
  d.update(state[0]);
  // Restore the internal state of the C++ object.
  const std::string str = extract<std::string>(state[1]);

doc/notebooks/tutorials/first_contact_zmq.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First contact\n",
+ "===\n",
+ "\n",
+ "Now that we have a Redis server running and we have compiled PaGMO and PyGMO with ZeroMQ support (see The Setup (TODO: link)), we can run an example program that leverages the functionality of this type of island.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "from PyGMO import *\n",
+ "import time\n",
+ "\n",
+ "prob = problem.schwefel(10)\n",
+ "algo = algorithm.de(10)\n",
+ "pop = population(prob, 20)\n",
+ "\n",
+ "isl = zmq_island(algo, pop)\n",
+ "isl.set_broker_details(\"127.0.0.1\", 6379)\n",
+ "isl.set_token(\"schwefel10_de10_pop20\")\n",
+ "isl.set_ip(\"127.0.0.1\")\n",
+ "\n",
+ "def evolve():\n",
+ " isl.evolve(10)\n",
+ " print(\"Best: \", isl.population.champion.x)\n",
+ "\n",
+ " time.sleep(1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-12 set up the island itself. \n",
+ "\n",
+ "In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost.\n",
+ "\n",
+ "The channel token is set to `schwefel10_de10_pop20`, which can be any string, but choosing a token relevant to the actual properties of the computation will help you determine which group of islands is working on what problem. If we start this program, it will connect to the broker and advertise itself as an island working on that token, but because there are no other peers it will simply perform the evolution locally. This is a typical output:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Best: (424.4920569537634, 440.7206390429582, 399.43102290788795, -299.52078183929916, 424.4274999225018, 395.96019766970727, 396.61431073790675, 414.7987356103403, -293.15767979233385, -152.01178644903854)\n"
+ ]
+ }
+ ],
+ "source": [
+ "evolve()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Note that because the island is not actually connected to the ZeroMQ network yet, it's not exchanging solutions with peers."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, if we call the `connect()` function on this island, we'll be ready to accept migrants and send our own as well."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "isl.connect()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Because IPython doesn't have an easy way of executing two threads and displaying the result easily, I'll start up another program just like this outside IPython and call evolve() here -- it should receive the population from the other program, and display it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n"
+ ]
+ }
+ ],
+ "source": [
+ "evolve()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As you can see, our current solution is much better than the original and because the program in the background had been running for a little while before that, and it had already converged on the optimum for the Schwefel function."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here's a snippet of the output on the other side: \n",
+ "\n",
+ " Best: (420.96880347940544, 420.9687194811639, 420.96880839006843, 420.9687540217262, 420.9687810359755, 420.9687609702579, 420.968730572192, 420.96875630500364, 420.9687648447646, 420.96880903640675)\n",
+ " Best: (420.9687351908159, 420.96872163933637, 420.9687399639167, 420.9687069931382, 420.9686997709129, 420.9687976850297, 420.9687312413354, 420.9687104977846, 420.9687392971252, 420.9687677879039)\n",
+ " DEBUG: Opening connection to 127.0.0.1:2609\n",
+ " Best: (420.9687701356386, 420.9687781629428, 420.9687691217583, 420.9687483448487, 420.96878827364174, 420.96875668743473, 420.96870707941497, 420.9687079359776, 420.96874266075383, 420.96872310305565)\n",
+ " Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n",
+ " Best: (420.9687533680281, 420.96876243183203, 420.9687590126924, 420.9687372408334, 420.96875017480545, 420.96877891439186, 420.9687558066539, 420.9687460272027, 420.9687420838824, 420.9687647092545)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Note how it opens a connection to our island as soon as we call `isl.connect()`. Then it sent its population, which had already converged on the answer and was better than our original value."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}