[repo] Add a publish/subscribe mechanism for inter-instance communication using zmq
authorJulien Cristau <julien.cristau@logilab.fr>, Quentin Roquefort <quentin@kpsule.me>
Fri, 10 Feb 2012 16:20:35 +0100
changeset 8211 543e1579ba0d
parent 8210 1d1cfc97f6b9
child 8214 ce9556358dbd
[repo] Add a publish/subscribe mechanism for inter-instance communication using zmq Each repo can have a publishing and any number of subscribing sockets whose addresses are specified in the instance's configuration. An application or cube can subscribe to some 'topics', and give a callback that gets called when a message matching that topic is received. As a proof of concept, this introduces a hook to clean up the caches associated with the repository when an entity is deleted. A subscription is added using Repository::zmq::add_subscription; the callback receives a list representing the received multi-part message as argument (the first element of the message is its topic).
__pkginfo__.py
debian/control
doc/book/en/devrepo/repo/hooks.rst
hooks/zmq.py
server/cwzmq.py
server/hook.py
server/repository.py
server/serverconfig.py
--- a/__pkginfo__.py	Fri Feb 10 10:45:17 2012 +0100
+++ b/__pkginfo__.py	Fri Feb 10 16:20:35 2012 +0100
@@ -63,6 +63,7 @@
     'fyzz': '>= 0.1.0',         # for sparql
     'vobject': '>= 0.6.0',      # for ical view
     'rdflib': None,             #
+    'pyzmq': None,
     #'Products.FCKeditor':'',
     #'SimpleTAL':'>= 4.1.6',
     }
--- a/debian/control	Fri Feb 10 10:45:17 2012 +0100
+++ b/debian/control	Fri Feb 10 16:20:35 2012 +0100
@@ -37,6 +37,7 @@
 Provides: cubicweb-multisources
 Depends: ${misc:Depends}, ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.8.2), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
 Recommends: pyro (<< 4.0.0), cubicweb-documentation (= ${source:Version})
+Suggests: python-zmq
 Description: server part of the CubicWeb framework
  CubicWeb is a semantic web application framework.
  .
--- a/doc/book/en/devrepo/repo/hooks.rst	Fri Feb 10 10:45:17 2012 +0100
+++ b/doc/book/en/devrepo/repo/hooks.rst	Fri Feb 10 16:20:35 2012 +0100
@@ -162,6 +162,44 @@
 :ref:`adv_tuto_security_propagation`.
 
 
+Inter-instance communication
+----------------------------
+
+If your application consists of several instances, you may need some means to
+communicate between them.  Cubicweb provides a publish/subscribe mechanism
+using ØMQ_.  In order to use it, use
+:meth:`~cubicweb.server.cwzmq.ZMQComm.add_subscription` on the
+`repo.app_instances_bus` object.  The `callback` will get the message (as a
+list).  A message can be sent by calling
+:meth:`~cubicweb.server.cwzmq.ZMQComm.publish` on `repo.app_instances_bus`.
+The first element of the message is the topic which is used for filtering and
+dispatching messages.
+
+.. _ØMQ: http://www.zeromq.org/
+
+.. sourcecode:: python
+
+  class FooHook(hook.Hook):
+      events = ('server_startup',)
+      __regid__ = 'foo_startup'
+
+      def __call__(self):
+          def callback(msg):
+              self.info('received message: %s', ' '.join(msg))
+          self.repo.app_instances_bus.subscribe('hello', callback)
+
+.. sourcecode:: python
+
+  def do_foo(self):
+      actually_do_foo()
+      self._cw.repo.app_instances_bus.publish(['hello', 'world'])
+
+The `zmq-address-pub` configuration variable contains the address used
+by the instance for sending messages, e.g. `tcp://*:1234`.  The
+`zmq-address-sub` variable contains a comma-separated list of addresses
+to listen on, e.g. `tcp://localhost:1234, tcp://192.168.1.1:2345`.
+
+
 Hooks writing tips
 ------------------
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hooks/zmq.py	Fri Feb 10 16:20:35 2012 +0100
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# copyright 2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
+
+from cubicweb.server import hook
+
+class ZMQStopHook(hook.Hook):
+    __regid__ = 'zmqstop'
+    events = ('server_shutdown',)
+
+    def __call__(self):
+        self.repo.app_instances_bus.stop()
+
+class ZMQStartHook(hook.Hook):
+    __regid__ = 'zmqstart'
+    events = ('server_startup',)
+
+    def __call__(self):
+        config = self.repo.config
+        address_pub = config.get('zmq-address-pub')
+        if not address_pub:
+            return
+        from cubicweb.server import cwzmq
+        self.repo.app_instances_bus = cwzmq.ZMQComm()
+        self.repo.app_instances_bus.add_publisher(address_pub)
+        def clear_cache_callback(msg):
+            self.debug('clear_caches: %s', ' '.join(msg))
+            self.repo.clear_caches(msg[1:])
+        self.repo.app_instances_bus.add_subscription('delete', clear_cache_callback)
+        for address in config.get('zmq-address-sub'):
+            self.repo.app_instances_bus.add_subscriber(address)
+        self.repo.app_instances_bus.start()
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/server/cwzmq.py	Fri Feb 10 16:20:35 2012 +0100
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# copyright 2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
+
+from threading import Thread
+import zmq
+from zmq.eventloop import ioloop
+import zmq.eventloop.zmqstream
+
+from logging import getLogger
+from cubicweb import set_log_methods
+
+ctx = zmq.Context()
+
+class ZMQComm(object):
+    def __init__(self):
+        self.ioloop = ioloop.IOLoop()
+        self._topics = {}
+        self._subscribers = []
+        self.publisher = None
+
+    def add_publisher(self, address):
+        assert self.publisher is None, "more than one publisher is not supported"
+        self.publisher = Publisher(self.ioloop, address)
+
+    def add_subscription(self, topic, callback):
+        for subscriber in self._subscribers:
+            subscriber.subscribe(topic, callback)
+        self._topics[topic] = callback
+
+    def add_subscriber(self, address):
+        subscriber = Subscriber(self.ioloop, address)
+        for topic, callback in self._topics.iteritems():
+            subscriber.subscribe(topic, callback)
+        self._subscribers.append(subscriber)
+
+    def publish(self, msg):
+        assert self.publisher is not None, "can't publish without a publisher"
+        self.publisher.send(msg)
+
+    def start(self):
+        Thread(target=self.ioloop.start).start()
+
+    def stop(self):
+        self.ioloop.add_callback(self.ioloop.stop)
+
+    def __del__(self):
+        self.ioloop.close()
+
+
+class Publisher(object):
+    def __init__(self, ioloop, address):
+        self.address = address
+        self._topics = {}
+        self._subscribers = []
+        self.ioloop = ioloop
+        def callback():
+            s = ctx.socket(zmq.PUB)
+            self.stream = zmq.eventloop.zmqstream.ZMQStream(s, io_loop=ioloop)
+            self.stream.bind(self.address)
+            self.debug('start publisher on %s', self.address)
+        ioloop.add_callback(callback)
+
+    def send(self, msg):
+        self.ioloop.add_callback(lambda:self.stream.send_multipart(msg))
+
+
+class Subscriber(object):
+    def __init__(self, ioloop, address):
+        self.address = address
+        self.dispatch_table = {}
+        self.ioloop = ioloop
+        def callback():
+            s = ctx.socket(zmq.SUB)
+            self.stream = zmq.eventloop.zmqstream.ZMQStream(s, io_loop=ioloop)
+            self.stream.on_recv(self.dispatch)
+            self.stream.connect(self.address)
+            self.debug('start subscriber on %s', self.address)
+        ioloop.add_callback(callback)
+
+    def dispatch(self, msg):
+        try:
+            f = self.dispatch_table[msg[0]]
+        except KeyError:
+            return
+        f(msg)
+
+    def subscribe(self, topic, callback):
+        self.dispatch_table[topic] = callback
+        self.ioloop.add_callback(lambda: self.stream.setsockopt(zmq.SUBSCRIBE, topic))
+
+
+set_log_methods(Publisher, getLogger('cubicweb.zmq.pub'))
+set_log_methods(Subscriber, getLogger('cubicweb.zmq.sub'))
--- a/server/hook.py	Fri Feb 10 10:45:17 2012 +0100
+++ b/server/hook.py	Fri Feb 10 16:20:35 2012 +0100
@@ -1064,6 +1064,8 @@
         remove inserted eid from repository type/source cache
         """
         try:
-            self.session.repo.clear_caches(self.get_data())
+            eids = self.get_data()
+            self.session.repo.clear_caches(eids)
+            self.session.repo.app_instances_bus.publish(['delete'] + list(str(eid) for eid in eids))
         except KeyError:
             pass
--- a/server/repository.py	Fri Feb 10 10:45:17 2012 +0100
+++ b/server/repository.py	Fri Feb 10 16:20:35 2012 +0100
@@ -120,6 +120,20 @@
                             {'x': eidfrom, 'y': eidto})
 
 
+class NullEventBus(object):
+    def send(self, msg):
+        pass
+
+    def add_subscription(self, topic, callback):
+        pass
+
+    def start(self):
+        pass
+
+    def stop(self):
+        pass
+
+
 class Repository(object):
     """a repository provides access to a set of persistent storages for
     entities and relations
@@ -134,6 +148,7 @@
         self.vreg = vreg
         self.pyro_registered = False
         self.pyro_uri = None
+        self.app_instances_bus = NullEventBus()
         self.info('starting repository from %s', self.config.apphome)
         # dictionary of opened sessions
         self._sessions = {}
--- a/server/serverconfig.py	Fri Feb 10 10:45:17 2012 +0100
+++ b/server/serverconfig.py	Fri Feb 10 16:20:35 2012 +0100
@@ -207,6 +207,19 @@
 and if not set, it will be choosen randomly',
           'group': 'pyro', 'level': 3,
           }),
+
+         ('zmq-address-sub',
+          {'type' : 'csv',
+           'default' : None,
+           'help': ('List of ZMQ addresses to subscribe to (requires pyzmq)'),
+           'group': 'zmq', 'level': 1,
+           }),
+         ('zmq-address-pub',
+          {'type' : 'string',
+           'default' : None,
+           'help': ('ZMQ address to use for publishing (requires pyzmq)'),
+           'group': 'zmq', 'level': 1,
+           }),
         ) + CubicWebConfiguration.options)
 
     # should we init the connections pool (eg connect to sources). This is