[cwctl] fix cubicweb-ctl shell command (closes #2583919)
authorAurelien Campeas <aurelien.campeas@logilab.fr>
Fri, 25 Jan 2013 13:28:23 +0100
changeset 8682 20bd1cdf86ae
parent 8677 a75bb09d6d19
child 8683 d537786e52b8
[cwctl] fix cubicweb-ctl shell command (closes #2583919) * add an uri parsing utility that documents a bit the uri cubicweb supports out of the box * cwctl: use this utility and refactor a bit for clarity (the novelty here being real support for old style 'myapp', or: * 'inmemory://myapp' * 'pyro://pyro-ns-host:pyro-ns-port/myapp' (pyro access through a pyro concentrator) * 'pyroloc://host:port/pyro-instance-id' (direct access to a pyro repository) * 'zmqpickle-tcp://host:port' * dbapi: refactor to sort out some of the complexity and restore functionality lost in 62213a34726e
cwctl.py
dbapi.py
test/unittest_utils.py
utils.py
--- a/cwctl.py	Wed Jan 23 17:58:49 2013 +0100
+++ b/cwctl.py	Fri Jan 25 13:28:23 2013 +0100
@@ -28,6 +28,8 @@
 from warnings import warn
 from os import remove, listdir, system, pathsep
 from os.path import exists, join, isfile, isdir, dirname, abspath
+from urlparse import urlparse
+
 try:
     from os import kill, getpgid
 except ImportError:
@@ -878,48 +880,59 @@
           }),
         )
 
+    def _handle_inmemory(self, appid):
+        """ returns migration context handler & shutdown function """
+        config = cwcfg.config_for(appid)
+        if self.config.ext_sources:
+            assert not self.config.system_only
+            sources = self.config.ext_sources
+        elif self.config.system_only:
+            sources = ('system',)
+        else:
+            sources = ('all',)
+        config.set_sources_mode(sources)
+        config.repairing = self.config.force
+        mih = config.migration_handler()
+        return mih, lambda: mih.shutdown()
+
+    def _handle_networked(self, appuri):
+        """ returns migration context handler & shutdown function """
+        from cubicweb import AuthenticationError
+        from cubicweb.dbapi import connect
+        from cubicweb.server.utils import manager_userpasswd
+        from cubicweb.server.migractions import ServerMigrationHelper
+        while True:
+            try:
+                login, pwd = manager_userpasswd(msg=None)
+                cnx = connect(appuri, login=login, password=pwd, mulcnx=False)
+            except AuthenticationError, ex:
+                print ex
+            except (KeyboardInterrupt, EOFError):
+                print
+                sys.exit(0)
+            else:
+                break
+        cnx.load_appobjects()
+        repo = cnx._repo
+        mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0,
+                                    # hack so it don't try to load fs schema
+                                    schema=1)
+        return mih, lambda: cnx.close()
+
     def run(self, args):
-        from urlparse import urlparse
         appuri = args.pop(0)
         if self.config.repo_uri:
             warn('[3.16] --repo-uri option is deprecated, directly give the URI as instance id',
                  DeprecationWarning)
             if urlparse(self.config.repo_uri).scheme in ('pyro', 'inmemory'):
                 appuri = '%s/%s' % (self.config.repo_uri.rstrip('/'), appuri)
-        scheme = urlparse(self.config.repo_uri).scheme
-        if scheme not in ('', 'inmemory'):
-            from cubicweb import AuthenticationError
-            from cubicweb.dbapi import connect
-            from cubicweb.server.utils import manager_userpasswd
-            from cubicweb.server.migractions import ServerMigrationHelper
-            while True:
-                try:
-                    login, pwd = manager_userpasswd(msg=None)
-                    cnx = connect(appuri, login=login, password=pwd, mulcnx=False)
-                except AuthenticationError, ex:
-                    print ex
-                except (KeyboardInterrupt, EOFError):
-                    print
-                    sys.exit(0)
-                else:
-                    break
-            cnx.load_appobjects()
-            repo = cnx._repo
-            mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0,
-                                         # hack so it don't try to load fs schema
-                                        schema=1)
+
+        from cubicweb.utils import parse_repo_uri
+        protocol, hostport, appid = parse_repo_uri(appuri)
+        if protocol == 'inmemory':
+            mih, shutdown_callback = self._handle_inmemory(appid)
         else:
-            config = cwcfg.config_for(appid)
-            if self.config.ext_sources:
-                assert not self.config.system_only
-                sources = self.config.ext_sources
-            elif self.config.system_only:
-                sources = ('system',)
-            else:
-                sources = ('all',)
-            config.set_sources_mode(sources)
-            config.repairing = self.config.force
-            mih = config.migration_handler()
+            mih, shutdown_callback = self._handle_networked(appuri)
         try:
             if args:
                 # use cmdline parser to access left/right attributes only
@@ -931,10 +944,7 @@
             else:
                 mih.interactive_shell()
         finally:
-            if scheme in ('', 'inmemory'): # shutdown in-memory repo
-                mih.shutdown()
-            else:
-                cnx.close()
+            shutdown_callback()
 
 
 class RecompileInstanceCatalogsCommand(InstanceCommand):
--- a/dbapi.py	Wed Jan 23 17:58:49 2013 +0100
+++ b/dbapi.py	Fri Jan 25 13:28:23 2013 +0100
@@ -40,6 +40,7 @@
 from cubicweb import ETYPE_NAME_MAP, ConnectionError, AuthenticationError,\
      cwvreg, cwconfig
 from cubicweb.req import RequestSessionBase
+from cubicweb.utils import parse_repo_uri
 
 
 _MARKER = object()
@@ -90,6 +91,11 @@
         self.close_on_del = close
 
 
+def _get_inmemory_repo(config, vreg=None):
+    from cubicweb.server.repository import Repository
+    from cubicweb.server.utils import TasksManager
+    return Repository(config, TasksManager(), vreg=vreg)
+
 def get_repository(uri=None, config=None, vreg=None):
     """get a repository for the given URI or config/vregistry (in case we're
     loading the repository for a client, eg web server, configuration).
@@ -97,39 +103,48 @@
     The returned repository may be an in-memory repository or a proxy object
     using a specific RPC method, depending on the given URI (pyro or zmq).
     """
+    try:
+        return _get_repository(uri, config, vreg)
+    except ConnectionError:
+        raise
+    except Exception, exc:
+        raise ConnectionError('cause: %r' % exc)
+
+def _get_repository(uri=None, config=None, vreg=None):
+    """ implements get_repository (see above) """
     if uri is None:
-        uri = config['repository-uri'] or config.appid
-    puri = urlparse(uri)
-    method = puri.scheme.lower() or 'inmemory'
-    if method == 'inmemory':
-        # get local access to the repository
-        from cubicweb.server.repository import Repository
-        from cubicweb.server.utils import TasksManager
-        return Repository(config, TasksManager(), vreg=vreg)
-    elif method in ('pyro', 'pyroloc'):
-        # resolve the Pyro object
-        from logilab.common.pyro_ext import ns_get_proxy, get_proxy
-        try:
-            if puri.scheme == 'pyroloc':
-                return get_proxy(uri)
-            path = puri.path.rstrip('/')
-            if not path:
-                raise ConnectionError(
-                    "can't find instance name in %s (expected to be the path component)"
-                    % uri)
-            if '.' in path:
-                nsgroup, nsid = path.rsplit('.', 1)
-            else:
-                nsgroup = 'cubicweb'
-                nsid = path
-            return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=puri.netloc)
-        except Exception, ex:
-            raise ConnectionError(str(ex))
-    elif method.startswith('zmqpickle-'):
+        return _get_inmemory_repo(config, vreg)
+
+    protocol, hostport, appid = parse_repo_uri(uri)
+
+    if protocol == 'inmemory':
+        # me may have been called with a dummy 'inmemory://' uri ...
+        return _get_inmemory_repo(config, vreg)
+
+    if protocol == 'pyroloc': # direct connection to the instance
+        from logilab.common.pyro_ext import get_proxy
+        uri = uri.replace('pyroloc', 'PYRO')
+        return get_proxy(uri)
+
+    if protocol == 'pyro': # connection mediated through the pyro ns
+        from logilab.common.pyro_ext import ns_get_proxy
+        path = appid.strip('/')
+        if not path:
+            raise ConnectionError(
+                "can't find instance name in %s (expected to be the path component)"
+                % uri)
+        if '.' in path:
+            nsgroup, nsid = path.rsplit('.', 1)
+        else:
+            nsgroup = 'cubicweb'
+            nsid = path
+        return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=hostport)
+
+    if protocol.startswith('zmqpickle-'):
         from cubicweb.zmqclient import ZMQRepositoryClient
         return ZMQRepositoryClient(uri)
     else:
-        raise ConnectionError('unknown protocol: `%s`' % method)
+        raise ConnectionError('unknown protocol: `%s`' % protocol)
 
 
 def _repo_connect(repo, login, **kwargs):
@@ -159,7 +174,7 @@
     * a simple instance id for in-memory connection
 
     * an uri like scheme://host:port/instanceid where scheme may be one of
-      'pyro', 'pyroloc', 'inmemory' or a schema supported by ZMQ
+      'pyro', 'inmemory' or 'zmqpickle'
 
       * if scheme is 'pyro', <host:port> determine the name server address. If
         not specified (e.g. 'pyro:///instanceid'), it will be detected through a
@@ -167,8 +182,6 @@
         server and may be prefixed by a group (e.g.
         'pyro:///:cubicweb.instanceid')
 
-      * if scheme is 'pyroloc', it's expected to be a bare pyro location URI
-
       * if scheme is handled by ZMQ (eg 'tcp'), you should not specify an
         instance id
 
@@ -215,7 +228,7 @@
     puri = urlparse(database)
     method = puri.scheme.lower()
     if method == 'inmemory':
-        config = cwconfig.instance_configuration(puuri.path)
+        config = cwconfig.instance_configuration(puri.path)
     else:
         config = cwconfig.CubicWebNoAppConfiguration()
     repo = get_repository(database, config=config)
--- a/test/unittest_utils.py	Wed Jan 23 17:58:49 2013 +0100
+++ b/test/unittest_utils.py	Fri Jan 25 13:28:23 2013 +0100
@@ -26,7 +26,7 @@
 
 from cubicweb.devtools.testlib import CubicWebTC
 from cubicweb.utils import (make_uid, UStringIO, SizeConstrainedList,
-                            RepeatList, HTMLHead, QueryCache)
+                            RepeatList, HTMLHead, QueryCache, parse_repo_uri)
 from cubicweb.entity import Entity
 
 try:
@@ -50,6 +50,25 @@
                           'some numeric character, got %s' % uid)
             d.add(uid)
 
+
+class TestParseRepoUri(TestCase):
+
+    def test_parse_repo_uri(self):
+        self.assertEqual(('inmemory', None, 'myapp'),
+                         parse_repo_uri('myapp'))
+        self.assertEqual(('inmemory', None, 'myapp'),
+                         parse_repo_uri('inmemory://myapp'))
+        self.assertEqual(('pyro', 'pyro-ns-host:pyro-ns-port', '/myapp'),
+                         parse_repo_uri('pyro://pyro-ns-host:pyro-ns-port/myapp'))
+        self.assertEqual(('pyroloc', 'host:port', '/appkey'),
+                         parse_repo_uri('pyroloc://host:port/appkey'))
+        self.assertEqual(('zmqpickle-tcp', '127.0.0.1:666', ''),
+                         parse_repo_uri('zmqpickle-tcp://127.0.0.1:666'))
+        with self.assertRaises(NotImplementedError):
+            parse_repo_uri('foo://bar')
+
+
+
 class TestQueryCache(TestCase):
     def test_querycache(self):
         c = QueryCache(ceiling=20)
--- a/utils.py	Wed Jan 23 17:58:49 2013 +0100
+++ b/utils.py	Fri Jan 25 13:28:23 2013 +0100
@@ -33,6 +33,7 @@
 from uuid import uuid4
 from warnings import warn
 from threading import Lock
+from urlparse import urlparse
 
 from logging import getLogger
 
@@ -574,6 +575,25 @@
     return dict1
 
 
+def parse_repo_uri(uri):
+    """ transform a command line uri into a (protocol, hostport, appid), e.g:
+    <myapp>                      -> 'inmemory', None, '<myapp>'
+    inmemory://<myapp>           -> 'inmemory', None, '<myapp>'
+    pyro://[host][:port]         -> 'pyro', 'host:port', None
+    zmqpickle://[host][:port]    -> 'zmqpickle', 'host:port', None
+    """
+    parseduri = urlparse(uri)
+    scheme = parseduri.scheme
+    if scheme == '':
+        return ('inmemory', None, parseduri.path)
+    if scheme == 'inmemory':
+        return (scheme, None, parseduri.netloc)
+    if scheme in ('pyro', 'pyroloc') or scheme.startswith('zmqpickle-'):
+        return (scheme, parseduri.netloc, parseduri.path)
+    raise NotImplementedError('URI protocol not implemented for `%s`' % uri)
+
+
+
 logger = getLogger('cubicweb.utils')
 
 class QueryCache(object):