[cwctl] fix cubicweb-ctl shell command (closes #2583919)
* add an uri parsing utility that documents a bit the uri cubicweb
supports out of the box
* cwctl: use this utility and refactor a bit for clarity (the novelty
here being real support for old style 'myapp', or:
* 'inmemory://myapp'
* 'pyro://pyro-ns-host:pyro-ns-port/myapp' (pyro access through a
pyro concentrator)
* 'pyroloc://host:port/pyro-instance-id' (direct access to a pyro
repository)
* 'zmqpickle-tcp://host:port'
* dbapi: refactor to sort out some of the complexity and restore
functionality lost in 62213a34726e
--- a/cwctl.py Wed Jan 23 17:58:49 2013 +0100
+++ b/cwctl.py Fri Jan 25 13:28:23 2013 +0100
@@ -28,6 +28,8 @@
from warnings import warn
from os import remove, listdir, system, pathsep
from os.path import exists, join, isfile, isdir, dirname, abspath
+from urlparse import urlparse
+
try:
from os import kill, getpgid
except ImportError:
@@ -878,48 +880,59 @@
}),
)
+ def _handle_inmemory(self, appid):
+ """ returns migration context handler & shutdown function """
+ config = cwcfg.config_for(appid)
+ if self.config.ext_sources:
+ assert not self.config.system_only
+ sources = self.config.ext_sources
+ elif self.config.system_only:
+ sources = ('system',)
+ else:
+ sources = ('all',)
+ config.set_sources_mode(sources)
+ config.repairing = self.config.force
+ mih = config.migration_handler()
+ return mih, lambda: mih.shutdown()
+
+ def _handle_networked(self, appuri):
+ """ returns migration context handler & shutdown function """
+ from cubicweb import AuthenticationError
+ from cubicweb.dbapi import connect
+ from cubicweb.server.utils import manager_userpasswd
+ from cubicweb.server.migractions import ServerMigrationHelper
+ while True:
+ try:
+ login, pwd = manager_userpasswd(msg=None)
+ cnx = connect(appuri, login=login, password=pwd, mulcnx=False)
+ except AuthenticationError, ex:
+ print ex
+ except (KeyboardInterrupt, EOFError):
+ print
+ sys.exit(0)
+ else:
+ break
+ cnx.load_appobjects()
+ repo = cnx._repo
+ mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0,
+ # hack so it don't try to load fs schema
+ schema=1)
+ return mih, lambda: cnx.close()
+
def run(self, args):
- from urlparse import urlparse
appuri = args.pop(0)
if self.config.repo_uri:
warn('[3.16] --repo-uri option is deprecated, directly give the URI as instance id',
DeprecationWarning)
if urlparse(self.config.repo_uri).scheme in ('pyro', 'inmemory'):
appuri = '%s/%s' % (self.config.repo_uri.rstrip('/'), appuri)
- scheme = urlparse(self.config.repo_uri).scheme
- if scheme not in ('', 'inmemory'):
- from cubicweb import AuthenticationError
- from cubicweb.dbapi import connect
- from cubicweb.server.utils import manager_userpasswd
- from cubicweb.server.migractions import ServerMigrationHelper
- while True:
- try:
- login, pwd = manager_userpasswd(msg=None)
- cnx = connect(appuri, login=login, password=pwd, mulcnx=False)
- except AuthenticationError, ex:
- print ex
- except (KeyboardInterrupt, EOFError):
- print
- sys.exit(0)
- else:
- break
- cnx.load_appobjects()
- repo = cnx._repo
- mih = ServerMigrationHelper(None, repo=repo, cnx=cnx, verbosity=0,
- # hack so it don't try to load fs schema
- schema=1)
+
+ from cubicweb.utils import parse_repo_uri
+ protocol, hostport, appid = parse_repo_uri(appuri)
+ if protocol == 'inmemory':
+ mih, shutdown_callback = self._handle_inmemory(appid)
else:
- config = cwcfg.config_for(appid)
- if self.config.ext_sources:
- assert not self.config.system_only
- sources = self.config.ext_sources
- elif self.config.system_only:
- sources = ('system',)
- else:
- sources = ('all',)
- config.set_sources_mode(sources)
- config.repairing = self.config.force
- mih = config.migration_handler()
+ mih, shutdown_callback = self._handle_networked(appuri)
try:
if args:
# use cmdline parser to access left/right attributes only
@@ -931,10 +944,7 @@
else:
mih.interactive_shell()
finally:
- if scheme in ('', 'inmemory'): # shutdown in-memory repo
- mih.shutdown()
- else:
- cnx.close()
+ shutdown_callback()
class RecompileInstanceCatalogsCommand(InstanceCommand):
--- a/dbapi.py Wed Jan 23 17:58:49 2013 +0100
+++ b/dbapi.py Fri Jan 25 13:28:23 2013 +0100
@@ -40,6 +40,7 @@
from cubicweb import ETYPE_NAME_MAP, ConnectionError, AuthenticationError,\
cwvreg, cwconfig
from cubicweb.req import RequestSessionBase
+from cubicweb.utils import parse_repo_uri
_MARKER = object()
@@ -90,6 +91,11 @@
self.close_on_del = close
+def _get_inmemory_repo(config, vreg=None):
+ from cubicweb.server.repository import Repository
+ from cubicweb.server.utils import TasksManager
+ return Repository(config, TasksManager(), vreg=vreg)
+
def get_repository(uri=None, config=None, vreg=None):
"""get a repository for the given URI or config/vregistry (in case we're
loading the repository for a client, eg web server, configuration).
@@ -97,39 +103,48 @@
The returned repository may be an in-memory repository or a proxy object
using a specific RPC method, depending on the given URI (pyro or zmq).
"""
+ try:
+ return _get_repository(uri, config, vreg)
+ except ConnectionError:
+ raise
+ except Exception, exc:
+ raise ConnectionError('cause: %r' % exc)
+
+def _get_repository(uri=None, config=None, vreg=None):
+ """ implements get_repository (see above) """
if uri is None:
- uri = config['repository-uri'] or config.appid
- puri = urlparse(uri)
- method = puri.scheme.lower() or 'inmemory'
- if method == 'inmemory':
- # get local access to the repository
- from cubicweb.server.repository import Repository
- from cubicweb.server.utils import TasksManager
- return Repository(config, TasksManager(), vreg=vreg)
- elif method in ('pyro', 'pyroloc'):
- # resolve the Pyro object
- from logilab.common.pyro_ext import ns_get_proxy, get_proxy
- try:
- if puri.scheme == 'pyroloc':
- return get_proxy(uri)
- path = puri.path.rstrip('/')
- if not path:
- raise ConnectionError(
- "can't find instance name in %s (expected to be the path component)"
- % uri)
- if '.' in path:
- nsgroup, nsid = path.rsplit('.', 1)
- else:
- nsgroup = 'cubicweb'
- nsid = path
- return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=puri.netloc)
- except Exception, ex:
- raise ConnectionError(str(ex))
- elif method.startswith('zmqpickle-'):
+ return _get_inmemory_repo(config, vreg)
+
+ protocol, hostport, appid = parse_repo_uri(uri)
+
+ if protocol == 'inmemory':
+ # me may have been called with a dummy 'inmemory://' uri ...
+ return _get_inmemory_repo(config, vreg)
+
+ if protocol == 'pyroloc': # direct connection to the instance
+ from logilab.common.pyro_ext import get_proxy
+ uri = uri.replace('pyroloc', 'PYRO')
+ return get_proxy(uri)
+
+ if protocol == 'pyro': # connection mediated through the pyro ns
+ from logilab.common.pyro_ext import ns_get_proxy
+ path = appid.strip('/')
+ if not path:
+ raise ConnectionError(
+ "can't find instance name in %s (expected to be the path component)"
+ % uri)
+ if '.' in path:
+ nsgroup, nsid = path.rsplit('.', 1)
+ else:
+ nsgroup = 'cubicweb'
+ nsid = path
+ return ns_get_proxy(nsid, defaultnsgroup=nsgroup, nshost=hostport)
+
+ if protocol.startswith('zmqpickle-'):
from cubicweb.zmqclient import ZMQRepositoryClient
return ZMQRepositoryClient(uri)
else:
- raise ConnectionError('unknown protocol: `%s`' % method)
+ raise ConnectionError('unknown protocol: `%s`' % protocol)
def _repo_connect(repo, login, **kwargs):
@@ -159,7 +174,7 @@
* a simple instance id for in-memory connection
* an uri like scheme://host:port/instanceid where scheme may be one of
- 'pyro', 'pyroloc', 'inmemory' or a schema supported by ZMQ
+ 'pyro', 'inmemory' or 'zmqpickle'
* if scheme is 'pyro', <host:port> determine the name server address. If
not specified (e.g. 'pyro:///instanceid'), it will be detected through a
@@ -167,8 +182,6 @@
server and may be prefixed by a group (e.g.
'pyro:///:cubicweb.instanceid')
- * if scheme is 'pyroloc', it's expected to be a bare pyro location URI
-
* if scheme is handled by ZMQ (eg 'tcp'), you should not specify an
instance id
@@ -215,7 +228,7 @@
puri = urlparse(database)
method = puri.scheme.lower()
if method == 'inmemory':
- config = cwconfig.instance_configuration(puuri.path)
+ config = cwconfig.instance_configuration(puri.path)
else:
config = cwconfig.CubicWebNoAppConfiguration()
repo = get_repository(database, config=config)
--- a/test/unittest_utils.py Wed Jan 23 17:58:49 2013 +0100
+++ b/test/unittest_utils.py Fri Jan 25 13:28:23 2013 +0100
@@ -26,7 +26,7 @@
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.utils import (make_uid, UStringIO, SizeConstrainedList,
- RepeatList, HTMLHead, QueryCache)
+ RepeatList, HTMLHead, QueryCache, parse_repo_uri)
from cubicweb.entity import Entity
try:
@@ -50,6 +50,25 @@
'some numeric character, got %s' % uid)
d.add(uid)
+
+class TestParseRepoUri(TestCase):
+
+ def test_parse_repo_uri(self):
+ self.assertEqual(('inmemory', None, 'myapp'),
+ parse_repo_uri('myapp'))
+ self.assertEqual(('inmemory', None, 'myapp'),
+ parse_repo_uri('inmemory://myapp'))
+ self.assertEqual(('pyro', 'pyro-ns-host:pyro-ns-port', '/myapp'),
+ parse_repo_uri('pyro://pyro-ns-host:pyro-ns-port/myapp'))
+ self.assertEqual(('pyroloc', 'host:port', '/appkey'),
+ parse_repo_uri('pyroloc://host:port/appkey'))
+ self.assertEqual(('zmqpickle-tcp', '127.0.0.1:666', ''),
+ parse_repo_uri('zmqpickle-tcp://127.0.0.1:666'))
+ with self.assertRaises(NotImplementedError):
+ parse_repo_uri('foo://bar')
+
+
+
class TestQueryCache(TestCase):
def test_querycache(self):
c = QueryCache(ceiling=20)
--- a/utils.py Wed Jan 23 17:58:49 2013 +0100
+++ b/utils.py Fri Jan 25 13:28:23 2013 +0100
@@ -33,6 +33,7 @@
from uuid import uuid4
from warnings import warn
from threading import Lock
+from urlparse import urlparse
from logging import getLogger
@@ -574,6 +575,25 @@
return dict1
+def parse_repo_uri(uri):
+ """ transform a command line uri into a (protocol, hostport, appid), e.g:
+ <myapp> -> 'inmemory', None, '<myapp>'
+ inmemory://<myapp> -> 'inmemory', None, '<myapp>'
+ pyro://[host][:port] -> 'pyro', 'host:port', None
+ zmqpickle://[host][:port] -> 'zmqpickle', 'host:port', None
+ """
+ parseduri = urlparse(uri)
+ scheme = parseduri.scheme
+ if scheme == '':
+ return ('inmemory', None, parseduri.path)
+ if scheme == 'inmemory':
+ return (scheme, None, parseduri.netloc)
+ if scheme in ('pyro', 'pyroloc') or scheme.startswith('zmqpickle-'):
+ return (scheme, parseduri.netloc, parseduri.path)
+ raise NotImplementedError('URI protocol not implemented for `%s`' % uri)
+
+
+
logger = getLogger('cubicweb.utils')
class QueryCache(object):