# HG changeset patch # User Aurelien Campeas # Date 1359116903 -3600 # Node ID 20bd1cdf86ae691979824d572c12588416c7cab3 # Parent a75bb09d6d19e5b7709ebf7af6bd386fe70619e4 [cwctl] fix cubicweb-ctl shell command (closes #2583919) * add an uri parsing utility that documents a bit the uri cubicweb supports out of the box * cwctl: use this utility and refactor a bit for clarity (the novelty here being real support for old style 'myapp', or: * 'inmemory://myapp' * 'pyro://pyro-ns-host:pyro-ns-port/myapp' (pyro access through a pyro concentrator) * 'pyroloc://host:port/pyro-instance-id' (direct access to a pyro repository) * 'zmqpickle-tcp://host:port' * dbapi: refactor to sort out some of the complexity and restore functionality lost in 62213a34726e diff -r a75bb09d6d19 -r 20bd1cdf86ae cwctl.py --- a/cwctl.py Wed Jan 23 17:58:49 2013 +0100 +++ b/cwctl.py Fri Jan 25 13:28:23 2013 +0100 @@ -28,6 +28,8 @@ from warnings import warn from os import remove, listdir, system, pathsep from os.path import exists, join, isfile, isdir, dirname, abspath +from urlparse import urlparse + try: from os import kill, getpgid except ImportError: @@ -878,48 +880,59 @@ }), ) + def _handle_inmemory(self, appid): + """ returns migration context handler & shutdown function """ + config = cwcfg.config_for(appid) + if self.config.ext_sources: + assert not self.config.system_only + sources = self.config.ext_sources + elif self.config.system_only: + sources = ('system',) + else: + sources = ('all',) + config.set_sources_mode(sources) + config.repairing = self.config.force + mih = config.migration_handler() + return mih, lambda: mih.shutdown() + + def _handle_networked(self, appuri): + """ returns migration context handler & shutdown function """ + from cubicweb import AuthenticationError + from cubicweb.dbapi import connect + from cubicweb.server.utils import manager_userpasswd + from cubicweb.server.migractions import ServerMigrationHelper + while True: + try: + login, pwd = manager_userpasswd(msg=None) + cnx = connect(appuri, login=login, password=pwd, mulcnx=False) + except AuthenticationError, ex: + print ex + except (KeyboardInterrupt, EOFError): + print + sys.exit(0) + else: + break + cnx.load_appobjects() + repo = cnx._repo + mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0, + # hack so it don't try to load fs schema + schema=1) + return mih, lambda: cnx.close() + def run(self, args): - from urlparse import urlparse appuri = args.pop(0) if self.config.repo_uri: warn('[3.16] --repo-uri option is deprecated, directly give the URI as instance id', DeprecationWarning) if urlparse(self.config.repo_uri).scheme in ('pyro', 'inmemory'): appuri = '%s/%s' % (self.config.repo_uri.rstrip('/'), appuri) - scheme = urlparse(self.config.repo_uri).scheme - if scheme not in ('', 'inmemory'): - from cubicweb import AuthenticationError - from cubicweb.dbapi import connect - from cubicweb.server.utils import manager_userpasswd - from cubicweb.server.migractions import ServerMigrationHelper - while True: - try: - login, pwd = manager_userpasswd(msg=None) - cnx = connect(appuri, login=login, password=pwd, mulcnx=False) - except AuthenticationError, ex: - print ex - except (KeyboardInterrupt, EOFError): - print - sys.exit(0) - else: - break - cnx.load_appobjects() - repo = cnx._repo - mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0, - # hack so it don't try to load fs schema - schema=1) + + from cubicweb.utils import parse_repo_uri + protocol, hostport, appid = parse_repo_uri(appuri) + if protocol == 'inmemory': + mih, shutdown_callback = self._handle_inmemory(appid) else: - config = cwcfg.config_for(appid) - if self.config.ext_sources: - assert not self.config.system_only - sources = self.config.ext_sources - elif self.config.system_only: - sources = ('system',) - else: - sources = ('all',) - config.set_sources_mode(sources) - config.repairing = self.config.force - mih = config.migration_handler() + mih, shutdown_callback = self._handle_networked(appuri) try: if args: # use cmdline parser to access left/right attributes only @@ -931,10 +944,7 @@ else: mih.interactive_shell() finally: - if scheme in ('', 'inmemory'): # shutdown in-memory repo - mih.shutdown() - else: - cnx.close() + shutdown_callback() class RecompileInstanceCatalogsCommand(InstanceCommand): diff -r a75bb09d6d19 -r 20bd1cdf86ae dbapi.py --- a/dbapi.py Wed Jan 23 17:58:49 2013 +0100 +++ b/dbapi.py Fri Jan 25 13:28:23 2013 +0100 @@ -40,6 +40,7 @@ from cubicweb import ETYPE_NAME_MAP, ConnectionError, AuthenticationError,\ cwvreg, cwconfig from cubicweb.req import RequestSessionBase +from cubicweb.utils import parse_repo_uri _MARKER = object() @@ -90,6 +91,11 @@ self.close_on_del = close +def _get_inmemory_repo(config, vreg=None): + from cubicweb.server.repository import Repository + from cubicweb.server.utils import TasksManager + return Repository(config, TasksManager(), vreg=vreg) + def get_repository(uri=None, config=None, vreg=None): """get a repository for the given URI or config/vregistry (in case we're loading the repository for a client, eg web server, configuration). @@ -97,39 +103,48 @@ The returned repository may be an in-memory repository or a proxy object using a specific RPC method, depending on the given URI (pyro or zmq). """ + try: + return _get_repository(uri, config, vreg) + except ConnectionError: + raise + except Exception, exc: + raise ConnectionError('cause: %r' % exc) + +def _get_repository(uri=None, config=None, vreg=None): + """ implements get_repository (see above) """ if uri is None: - uri = config['repository-uri'] or config.appid - puri = urlparse(uri) - method = puri.scheme.lower() or 'inmemory' - if method == 'inmemory': - # get local access to the repository - from cubicweb.server.repository import Repository - from cubicweb.server.utils import TasksManager - return Repository(config, TasksManager(), vreg=vreg) - elif method in ('pyro', 'pyroloc'): - # resolve the Pyro object - from logilab.common.pyro_ext import ns_get_proxy, get_proxy - try: - if puri.scheme == 'pyroloc': - return get_proxy(uri) - path = puri.path.rstrip('/') - if not path: - raise ConnectionError( - "can't find instance name in %s (expected to be the path component)" - % uri) - if '.' in path: - nsgroup, nsid = path.rsplit('.', 1) - else: - nsgroup = 'cubicweb' - nsid = path - return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=puri.netloc) - except Exception, ex: - raise ConnectionError(str(ex)) - elif method.startswith('zmqpickle-'): + return _get_inmemory_repo(config, vreg) + + protocol, hostport, appid = parse_repo_uri(uri) + + if protocol == 'inmemory': + # me may have been called with a dummy 'inmemory://' uri ... + return _get_inmemory_repo(config, vreg) + + if protocol == 'pyroloc': # direct connection to the instance + from logilab.common.pyro_ext import get_proxy + uri = uri.replace('pyroloc', 'PYRO') + return get_proxy(uri) + + if protocol == 'pyro': # connection mediated through the pyro ns + from logilab.common.pyro_ext import ns_get_proxy + path = appid.strip('/') + if not path: + raise ConnectionError( + "can't find instance name in %s (expected to be the path component)" + % uri) + if '.' in path: + nsgroup, nsid = path.rsplit('.', 1) + else: + nsgroup = 'cubicweb' + nsid = path + return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=hostport) + + if protocol.startswith('zmqpickle-'): from cubicweb.zmqclient import ZMQRepositoryClient return ZMQRepositoryClient(uri) else: - raise ConnectionError('unknown protocol: `%s`' % method) + raise ConnectionError('unknown protocol: `%s`' % protocol) def _repo_connect(repo, login, **kwargs): @@ -159,7 +174,7 @@ * a simple instance id for in-memory connection * an uri like scheme://host:port/instanceid where scheme may be one of - 'pyro', 'pyroloc', 'inmemory' or a schema supported by ZMQ + 'pyro', 'inmemory' or 'zmqpickle' * if scheme is 'pyro', determine the name server address. If not specified (e.g. 'pyro:///instanceid'), it will be detected through a @@ -167,8 +182,6 @@ server and may be prefixed by a group (e.g. 'pyro:///:cubicweb.instanceid') - * if scheme is 'pyroloc', it's expected to be a bare pyro location URI - * if scheme is handled by ZMQ (eg 'tcp'), you should not specify an instance id @@ -215,7 +228,7 @@ puri = urlparse(database) method = puri.scheme.lower() if method == 'inmemory': - config = cwconfig.instance_configuration(puuri.path) + config = cwconfig.instance_configuration(puri.path) else: config = cwconfig.CubicWebNoAppConfiguration() repo = get_repository(database, config=config) diff -r a75bb09d6d19 -r 20bd1cdf86ae test/unittest_utils.py --- a/test/unittest_utils.py Wed Jan 23 17:58:49 2013 +0100 +++ b/test/unittest_utils.py Fri Jan 25 13:28:23 2013 +0100 @@ -26,7 +26,7 @@ from cubicweb.devtools.testlib import CubicWebTC from cubicweb.utils import (make_uid, UStringIO, SizeConstrainedList, - RepeatList, HTMLHead, QueryCache) + RepeatList, HTMLHead, QueryCache, parse_repo_uri) from cubicweb.entity import Entity try: @@ -50,6 +50,25 @@ 'some numeric character, got %s' % uid) d.add(uid) + +class TestParseRepoUri(TestCase): + + def test_parse_repo_uri(self): + self.assertEqual(('inmemory', None, 'myapp'), + parse_repo_uri('myapp')) + self.assertEqual(('inmemory', None, 'myapp'), + parse_repo_uri('inmemory://myapp')) + self.assertEqual(('pyro', 'pyro-ns-host:pyro-ns-port', '/myapp'), + parse_repo_uri('pyro://pyro-ns-host:pyro-ns-port/myapp')) + self.assertEqual(('pyroloc', 'host:port', '/appkey'), + parse_repo_uri('pyroloc://host:port/appkey')) + self.assertEqual(('zmqpickle-tcp', '127.0.0.1:666', ''), + parse_repo_uri('zmqpickle-tcp://127.0.0.1:666')) + with self.assertRaises(NotImplementedError): + parse_repo_uri('foo://bar') + + + class TestQueryCache(TestCase): def test_querycache(self): c = QueryCache(ceiling=20) diff -r a75bb09d6d19 -r 20bd1cdf86ae utils.py --- a/utils.py Wed Jan 23 17:58:49 2013 +0100 +++ b/utils.py Fri Jan 25 13:28:23 2013 +0100 @@ -33,6 +33,7 @@ from uuid import uuid4 from warnings import warn from threading import Lock +from urlparse import urlparse from logging import getLogger @@ -574,6 +575,25 @@ return dict1 +def parse_repo_uri(uri): + """ transform a command line uri into a (protocol, hostport, appid), e.g: + -> 'inmemory', None, '' + inmemory:// -> 'inmemory', None, '' + pyro://[host][:port] -> 'pyro', 'host:port', None + zmqpickle://[host][:port] -> 'zmqpickle', 'host:port', None + """ + parseduri = urlparse(uri) + scheme = parseduri.scheme + if scheme == '': + return ('inmemory', None, parseduri.path) + if scheme == 'inmemory': + return (scheme, None, parseduri.netloc) + if scheme in ('pyro', 'pyroloc') or scheme.startswith('zmqpickle-'): + return (scheme, parseduri.netloc, parseduri.path) + raise NotImplementedError('URI protocol not implemented for `%s`' % uri) + + + logger = getLogger('cubicweb.utils') class QueryCache(object):